You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2014/10/10 04:51:30 UTC
[01/52] [abbrv] SQOOP-1374: From/To: Metadata upgrade
Repository: sqoop
Updated Branches:
refs/heads/SQOOP-1367 0cfcce5aa -> d7e049043 (forced update)
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
index 2260a45..e658c11 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
@@ -17,6 +17,7 @@
*/
package org.apache.sqoop.repository.derby;
+import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MJob;
@@ -32,242 +33,249 @@ import java.util.Map;
*/
public class TestJobHandling extends DerbyTestCase {
-// DerbyRepositoryHandler handler;
-//
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// handler = new DerbyRepositoryHandler();
-//
-// // We always needs schema for this test case
-// createSchema();
-//
-// // We always needs connector and framework structures in place
-// loadConnectorAndFramework();
-//
-// // We always needs connection metadata in place
-// loadConnections();
-// }
-//
-// public void testFindJob() throws Exception {
-// // Let's try to find non existing job
-// try {
-// handler.findJob(1, getDerbyConnection());
-// fail();
-// } catch(SqoopException ex) {
-// assertEquals(DerbyRepoError.DERBYREPO_0030, ex.getErrorCode());
-// }
-//
-// // Load prepared connections into database
-// loadJobs();
-//
-// MJob jobImport = handler.findJob(1, getDerbyConnection());
-// assertNotNull(jobImport);
-// assertEquals(1, jobImport.getPersistenceId());
-// assertEquals("JA", jobImport.getName());
-// assertEquals(MJob.Type.IMPORT, jobImport.getType());
-//
-// List<MForm> forms;
-//
-// // Check connector part
-// forms = jobImport.getFromPart().getForms();
-// assertEquals("Value5", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Value7", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-//
-// // Check framework part
-// forms = jobImport.getFrameworkPart().getForms();
-// assertEquals("Value17", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Value19", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-// }
-//
-// public void testFindJobs() throws Exception {
-// List<MJob> list;
-//
-// // Load empty list on empty repository
-// list = handler.findJobs(getDerbyConnection());
-// assertEquals(0, list.size());
-//
-// loadJobs();
-//
-// // Load all two connections on loaded repository
-// list = handler.findJobs(getDerbyConnection());
-// assertEquals(4, list.size());
-//
-// assertEquals("JA", list.get(0).getName());
-// assertEquals(MJob.Type.IMPORT, list.get(0).getType());
-//
-// assertEquals("JB", list.get(1).getName());
-// assertEquals(MJob.Type.IMPORT, list.get(1).getType());
-//
-// assertEquals("JA", list.get(2).getName());
-// assertEquals(MJob.Type.EXPORT, list.get(2).getType());
-//
-// assertEquals("JB", list.get(3).getName());
-// assertEquals(MJob.Type.EXPORT, list.get(3).getType());
-// }
-//
-// public void testExistsJob() throws Exception {
-// // There shouldn't be anything on empty repository
-// assertFalse(handler.existsJob(1, getDerbyConnection()));
-// assertFalse(handler.existsJob(2, getDerbyConnection()));
-// assertFalse(handler.existsJob(3, getDerbyConnection()));
-// assertFalse(handler.existsJob(4, getDerbyConnection()));
-// assertFalse(handler.existsJob(5, getDerbyConnection()));
-//
-// loadJobs();
-//
-// assertTrue(handler.existsJob(1, getDerbyConnection()));
-// assertTrue(handler.existsJob(2, getDerbyConnection()));
-// assertTrue(handler.existsJob(3, getDerbyConnection()));
-// assertTrue(handler.existsJob(4, getDerbyConnection()));
-// assertFalse(handler.existsJob(5, getDerbyConnection()));
-// }
-//
-// public void testInUseJob() throws Exception {
-// loadJobs();
-// loadSubmissions();
-//
-// assertTrue(handler.inUseJob(1, getDerbyConnection()));
-// assertFalse(handler.inUseJob(2, getDerbyConnection()));
-// assertFalse(handler.inUseJob(3, getDerbyConnection()));
-// assertFalse(handler.inUseJob(4, getDerbyConnection()));
-// }
-//
-// public void testCreateJob() throws Exception {
-// MJob job = getJob();
-//
-// // Load some data
-// fillJob(job);
-//
-// handler.createJob(job, getDerbyConnection());
-//
-// assertEquals(1, job.getPersistenceId());
-// assertCountForTable("SQOOP.SQ_JOB", 1);
-// assertCountForTable("SQOOP.SQ_JOB_INPUT", 4);
-//
-// MJob retrieved = handler.findJob(1, getDerbyConnection());
-// assertEquals(1, retrieved.getPersistenceId());
-//
-// List<MForm> forms;
-// forms = job.getFromPart().getForms();
-// assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Value2", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-//
-// forms = job.getFrameworkPart().getForms();
-// assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-//
-// // Let's create second job
-// job = getJob();
-// fillJob(job);
-//
-// handler.createJob(job, getDerbyConnection());
-//
-// assertEquals(2, job.getPersistenceId());
-// assertCountForTable("SQOOP.SQ_JOB", 2);
-// assertCountForTable("SQOOP.SQ_JOB_INPUT", 8);
-// }
-//
-// public void testUpdateJob() throws Exception {
-// loadJobs();
-//
-// MJob job = handler.findJob(1, getDerbyConnection());
-//
-// List<MForm> forms;
-//
-// forms = job.getFromPart().getForms();
-// ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
-// ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
-// ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
-// ((MMapInput)forms.get(1).getInputs().get(1)).setValue(null);
-//
-// forms = job.getFrameworkPart().getForms();
-// ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
-// ((MMapInput)forms.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
-// ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
-// ((MMapInput)forms.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
-//
-// job.setName("name");
-//
-// handler.updateJob(job, getDerbyConnection());
-//
-// assertEquals(1, job.getPersistenceId());
-// assertCountForTable("SQOOP.SQ_JOB", 4);
-// assertCountForTable("SQOOP.SQ_JOB_INPUT", 18);
-//
-// MJob retrieved = handler.findJob(1, getDerbyConnection());
-// assertEquals("name", retrieved.getName());
-//
-// forms = retrieved.getFromPart().getForms();
-// assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-//
-// forms = retrieved.getFrameworkPart().getForms();
-// assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
-// assertNotNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
-// assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
-// assertNotNull(forms.get(1).getInputs().get(1).getValue());
-// assertEquals(((Map)forms.get(1).getInputs().get(1).getValue()).size(), 0);
-// }
-//
-// public void testEnableAndDisableJob() throws Exception {
-// loadJobs();
-//
-// // disable job 1
-// handler.enableJob(1, false, getDerbyConnection());
-//
-// MJob retrieved = handler.findJob(1, getDerbyConnection());
-// assertNotNull(retrieved);
-// assertEquals(false, retrieved.getEnabled());
-//
-// // enable job 1
-// handler.enableJob(1, true, getDerbyConnection());
-//
-// retrieved = handler.findJob(1, getDerbyConnection());
-// assertNotNull(retrieved);
-// assertEquals(true, retrieved.getEnabled());
-// }
-//
-// public void testDeleteJob() throws Exception {
-// loadJobs();
-//
-// handler.deleteJob(1, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_JOB", 3);
-// assertCountForTable("SQOOP.SQ_JOB_INPUT", 12);
-//
-// handler.deleteJob(2, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_JOB", 2);
-// assertCountForTable("SQOOP.SQ_JOB_INPUT", 8);
-//
-// handler.deleteJob(3, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_JOB", 1);
-// assertCountForTable("SQOOP.SQ_JOB_INPUT", 4);
-//
-// handler.deleteJob(4, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_JOB", 0);
-// assertCountForTable("SQOOP.SQ_JOB_INPUT", 0);
-// }
-//
-// public MJob getJob() {
-// return new MJob(1, 1, MJob.Type.IMPORT,
-// handler.findConnector("A",
-// getDerbyConnection()).getJobForms(MJob.Type.IMPORT
-// ),
-// handler.findFramework(
-// getDerbyConnection()).getJobForms(MJob.Type.IMPORT
-// )
-// );
-// }
+ DerbyRepositoryHandler handler;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+
+ // We always needs connector and framework structures in place
+ loadConnectorAndFramework();
+
+ // We always needs connection metadata in place
+ loadConnections();
+ }
+
+ public void testFindJob() throws Exception {
+ // Let's try to find non existing job
+ try {
+ handler.findJob(1, getDerbyConnection());
+ fail();
+ } catch(SqoopException ex) {
+ assertEquals(DerbyRepoError.DERBYREPO_0030, ex.getErrorCode());
+ }
+
+ // Load prepared connections into database
+ loadJobs();
+
+ MJob jobImport = handler.findJob(1, getDerbyConnection());
+ assertNotNull(jobImport);
+ assertEquals(1, jobImport.getPersistenceId());
+ assertEquals("JA", jobImport.getName());
+
+ List<MForm> forms;
+
+ // Check connector parts
+ forms = jobImport.getConnectorPart(Direction.FROM).getForms();
+ assertEquals(2, forms.size());
+ assertEquals("Value5", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value5", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ forms = jobImport.getConnectorPart(Direction.TO).getForms();
+ assertEquals(2, forms.size());
+ assertEquals("Value9", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value9", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ // Check framework part
+ forms = jobImport.getFrameworkPart().getForms();
+ assertEquals(2, forms.size());
+ assertEquals("Value17", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value19", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+ }
+
+ public void testFindJobs() throws Exception {
+ List<MJob> list;
+
+ // Load empty list on empty repository
+ list = handler.findJobs(getDerbyConnection());
+ assertEquals(0, list.size());
+
+ loadJobs();
+
+ // Load all two connections on loaded repository
+ list = handler.findJobs(getDerbyConnection());
+ assertEquals(4, list.size());
+
+ assertEquals("JA", list.get(0).getName());
+
+ assertEquals("JB", list.get(1).getName());
+
+ assertEquals("JC", list.get(2).getName());
+
+ assertEquals("JD", list.get(3).getName());
+ }
+
+ public void testExistsJob() throws Exception {
+ // There shouldn't be anything on empty repository
+ assertFalse(handler.existsJob(1, getDerbyConnection()));
+ assertFalse(handler.existsJob(2, getDerbyConnection()));
+ assertFalse(handler.existsJob(3, getDerbyConnection()));
+ assertFalse(handler.existsJob(4, getDerbyConnection()));
+ assertFalse(handler.existsJob(5, getDerbyConnection()));
+
+ loadJobs();
+
+ assertTrue(handler.existsJob(1, getDerbyConnection()));
+ assertTrue(handler.existsJob(2, getDerbyConnection()));
+ assertTrue(handler.existsJob(3, getDerbyConnection()));
+ assertTrue(handler.existsJob(4, getDerbyConnection()));
+ assertFalse(handler.existsJob(5, getDerbyConnection()));
+ }
+
+ public void testInUseJob() throws Exception {
+ loadJobs();
+ loadSubmissions();
+
+ assertTrue(handler.inUseJob(1, getDerbyConnection()));
+ assertFalse(handler.inUseJob(2, getDerbyConnection()));
+ assertFalse(handler.inUseJob(3, getDerbyConnection()));
+ assertFalse(handler.inUseJob(4, getDerbyConnection()));
+ }
+
+ public void testCreateJob() throws Exception {
+ MJob job = getJob();
+
+ // Load some data
+ fillJob(job);
+
+ handler.createJob(job, getDerbyConnection());
+
+ assertEquals(1, job.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_JOB", 1);
+ assertCountForTable("SQOOP.SQ_JOB_INPUT", 6);
+
+ MJob retrieved = handler.findJob(1, getDerbyConnection());
+ assertEquals(1, retrieved.getPersistenceId());
+
+ List<MForm> forms;
+ forms = job.getConnectorPart(Direction.FROM).getForms();
+ assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ forms = job.getConnectorPart(Direction.TO).getForms();
+ assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+
+ forms = job.getFrameworkPart().getForms();
+ assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ // Let's create second job
+ job = getJob();
+ fillJob(job);
+
+ handler.createJob(job, getDerbyConnection());
+
+ assertEquals(2, job.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_JOB", 2);
+ assertCountForTable("SQOOP.SQ_JOB_INPUT", 12);
+ }
+
+ public void testUpdateJob() throws Exception {
+ loadJobs();
+
+ assertCountForTable("SQOOP.SQ_JOB", 4);
+ assertCountForTable("SQOOP.SQ_JOB_INPUT", 24);
+
+ MJob job = handler.findJob(1, getDerbyConnection());
+
+ List<MForm> forms;
+
+ forms = job.getConnectorPart(Direction.FROM).getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
+ forms = job.getConnectorPart(Direction.TO).getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
+
+ forms = job.getFrameworkPart().getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+
+ job.setName("name");
+
+ handler.updateJob(job, getDerbyConnection());
+
+ assertEquals(1, job.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_JOB", 4);
+ assertCountForTable("SQOOP.SQ_JOB_INPUT", 26);
+
+ MJob retrieved = handler.findJob(1, getDerbyConnection());
+ assertEquals("name", retrieved.getName());
+
+ forms = job.getConnectorPart(Direction.FROM).getForms();
+ assertEquals(2, forms.size());
+ assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ forms = job.getConnectorPart(Direction.TO).getForms();
+ assertEquals(2, forms.size());
+ assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+
+ forms = retrieved.getFrameworkPart().getForms();
+ assertEquals(2, forms.size());
+ assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
+ assertNotNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
+ }
+
+ public void testEnableAndDisableJob() throws Exception {
+ loadJobs();
+
+ // disable job 1
+ handler.enableJob(1, false, getDerbyConnection());
+
+ MJob retrieved = handler.findJob(1, getDerbyConnection());
+ assertNotNull(retrieved);
+ assertEquals(false, retrieved.getEnabled());
+
+ // enable job 1
+ handler.enableJob(1, true, getDerbyConnection());
+
+ retrieved = handler.findJob(1, getDerbyConnection());
+ assertNotNull(retrieved);
+ assertEquals(true, retrieved.getEnabled());
+ }
+
+ public void testDeleteJob() throws Exception {
+ loadJobs();
+
+ handler.deleteJob(1, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_JOB", 3);
+ assertCountForTable("SQOOP.SQ_JOB_INPUT", 18);
+
+ handler.deleteJob(2, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_JOB", 2);
+ assertCountForTable("SQOOP.SQ_JOB_INPUT", 12);
+
+ handler.deleteJob(3, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_JOB", 1);
+ assertCountForTable("SQOOP.SQ_JOB_INPUT", 6);
+
+ handler.deleteJob(4, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_JOB", 0);
+ assertCountForTable("SQOOP.SQ_JOB_INPUT", 0);
+ }
+
+ public MJob getJob() {
+ return new MJob(1, 1, 1, 1,
+ handler.findConnector("A", getDerbyConnection()).getJobForms(Direction.FROM),
+ handler.findConnector("A", getDerbyConnection()).getJobForms(Direction.TO),
+ handler.findFramework(getDerbyConnection()).getJobForms()
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
index 8cfe076..8fce0dd 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
@@ -32,214 +32,214 @@ import java.util.List;
*/
public class TestSubmissionHandling extends DerbyTestCase {
-// DerbyRepositoryHandler handler;
-//
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// handler = new DerbyRepositoryHandler();
-//
-// // We always needs schema for this test case
-// createSchema();
-//
-// // We always needs connector and framework structures in place
-// loadConnectorAndFramework();
-//
-// // We also always needs connection metadata in place
-// loadConnections();
-//
-// // And finally we always needs job metadata in place
-// loadJobs();
-// }
-//
-// public void testFindSubmissionsUnfinished() throws Exception {
-// List<MSubmission> submissions;
-//
-// submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(0, submissions.size());
-//
-// loadSubmissions();
-//
-// submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(2, submissions.size());
-// }
-//
-// public void testExistsSubmission() throws Exception {
-// // There shouldn't be anything on empty repository
-// assertFalse(handler.existsSubmission(1, getDerbyConnection()));
-// assertFalse(handler.existsSubmission(2, getDerbyConnection()));
-// assertFalse(handler.existsSubmission(3, getDerbyConnection()));
-// assertFalse(handler.existsSubmission(4, getDerbyConnection()));
-// assertFalse(handler.existsSubmission(5, getDerbyConnection()));
-// assertFalse(handler.existsSubmission(6, getDerbyConnection()));
-//
-// loadSubmissions();
-//
-// assertTrue(handler.existsSubmission(1, getDerbyConnection()));
-// assertTrue(handler.existsSubmission(2, getDerbyConnection()));
-// assertTrue(handler.existsSubmission(3, getDerbyConnection()));
-// assertTrue(handler.existsSubmission(4, getDerbyConnection()));
-// assertTrue(handler.existsSubmission(5, getDerbyConnection()));
-// assertFalse(handler.existsSubmission(6, getDerbyConnection()));
-// }
-//
-// public void testCreateSubmission() throws Exception {
-// Date creationDate = new Date();
-// Date updateDate = new Date();
-//
-// CounterGroup firstGroup = new CounterGroup("ga");
-// CounterGroup secondGroup = new CounterGroup("gb");
-// firstGroup.addCounter(new Counter("ca", 100));
-// firstGroup.addCounter(new Counter("cb", 200));
-// secondGroup.addCounter(new Counter("ca", 300));
-// secondGroup.addCounter(new Counter("cd", 400));
-// Counters counters = new Counters();
-// counters.addCounterGroup(firstGroup);
-// counters.addCounterGroup(secondGroup);
-//
-// MSubmission submission = new MSubmission();
-// submission.setJobId(1);
-// submission.setStatus(SubmissionStatus.RUNNING);
-// submission.setCreationDate(creationDate);
-// submission.setLastUpdateDate(updateDate);
-// submission.setExternalId("job-x");
-// submission.setExternalLink("http://somewhere");
-// submission.setExceptionInfo("RuntimeException");
-// submission.setExceptionStackTrace("Yeah it happens");
-// submission.setCounters(counters);
-//
-// handler.createSubmission(submission, getDerbyConnection());
-//
-// assertEquals(1, submission.getPersistenceId());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 1);
-//
-// List<MSubmission> submissions =
-// handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(1, submissions.size());
-//
-// submission = submissions.get(0);
-//
-// assertEquals(1, submission.getJobId());
-// assertEquals(SubmissionStatus.RUNNING, submission.getStatus());
-// assertEquals(creationDate, submission.getCreationDate());
-// assertEquals(updateDate, submission.getLastUpdateDate());
-// assertEquals("job-x", submission.getExternalId());
-// assertEquals("http://somewhere", submission.getExternalLink());
-// assertEquals("RuntimeException", submission.getExceptionInfo());
-// assertEquals("Yeah it happens", submission.getExceptionStackTrace());
-//
-// CounterGroup group;
-// Counter counter;
-// Counters retrievedCounters = submission.getCounters();
-// assertNotNull(retrievedCounters);
-//
-// group = counters.getCounterGroup("ga");
-// assertNotNull(group);
-//
-// counter = group.getCounter("ca");
-// assertNotNull(counter);
-// assertEquals(100, counter.getValue());
-//
-// counter = group.getCounter("cb");
-// assertNotNull(counter);
-// assertEquals(200, counter.getValue());
-//
-// group = counters.getCounterGroup("gb");
-// assertNotNull(group);
-//
-// counter = group.getCounter("ca");
-// assertNotNull(counter);
-// assertEquals(300, counter.getValue());
-//
-// counter = group.getCounter("cd");
-// assertNotNull(counter);
-// assertEquals(400, counter.getValue());
-//
-// // Let's create second (simpler) connection
-// submission =
-// new MSubmission(1, new Date(), SubmissionStatus.SUCCEEDED, "job-x");
-// handler.createSubmission(submission, getDerbyConnection());
-//
-// assertEquals(2, submission.getPersistenceId());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
-// }
-//
-// public void testUpdateConnection() throws Exception {
-// loadSubmissions();
-//
-// List<MSubmission> submissions =
-// handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(2, submissions.size());
-//
-// MSubmission submission = submissions.get(0);
-// submission.setStatus(SubmissionStatus.SUCCEEDED);
-//
-// handler.updateSubmission(submission, getDerbyConnection());
-//
-// submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(1, submissions.size());
-// }
-//
-// public void testPurgeSubmissions() throws Exception {
-// loadSubmissions();
-// List<MSubmission> submissions;
-//
-// submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(2, submissions.size());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 5);
-//
-// Calendar calendar = Calendar.getInstance();
-// // 2012-01-03 05:05:05
-// calendar.set(2012, Calendar.JANUARY, 3, 5, 5, 5);
-// handler.purgeSubmissions(calendar.getTime(), getDerbyConnection());
-//
-// submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(1, submissions.size());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
-//
-// handler.purgeSubmissions(new Date(), getDerbyConnection());
-//
-// submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(0, submissions.size());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
-//
-// handler.purgeSubmissions(new Date(), getDerbyConnection());
-//
-// submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
-// assertNotNull(submissions);
-// assertEquals(0, submissions.size());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
-// }
-//
-// /**
-// * Test that by directly removing jobs we will also remove associated
-// * submissions and counters.
-// *
-// * @throws Exception
-// */
-// public void testDeleteJobs() throws Exception {
-// loadSubmissions();
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 5);
-//
-// handler.deleteJob(1, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 3);
-//
-// handler.deleteJob(2, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
-//
-// handler.deleteJob(3, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 1);
-//
-// handler.deleteJob(4, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
-// }
+ DerbyRepositoryHandler handler;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+
+ // We always needs connector and framework structures in place
+ loadConnectorAndFramework();
+
+ // We also always needs connection metadata in place
+ loadConnections();
+
+ // And finally we always needs job metadata in place
+ loadJobs();
+ }
+
+ public void testFindSubmissionsUnfinished() throws Exception {
+ List<MSubmission> submissions;
+
+ submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(0, submissions.size());
+
+ loadSubmissions();
+
+ submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(2, submissions.size());
+ }
+
+ public void testExistsSubmission() throws Exception {
+ // There shouldn't be anything on empty repository
+ assertFalse(handler.existsSubmission(1, getDerbyConnection()));
+ assertFalse(handler.existsSubmission(2, getDerbyConnection()));
+ assertFalse(handler.existsSubmission(3, getDerbyConnection()));
+ assertFalse(handler.existsSubmission(4, getDerbyConnection()));
+ assertFalse(handler.existsSubmission(5, getDerbyConnection()));
+ assertFalse(handler.existsSubmission(6, getDerbyConnection()));
+
+ loadSubmissions();
+
+ assertTrue(handler.existsSubmission(1, getDerbyConnection()));
+ assertTrue(handler.existsSubmission(2, getDerbyConnection()));
+ assertTrue(handler.existsSubmission(3, getDerbyConnection()));
+ assertTrue(handler.existsSubmission(4, getDerbyConnection()));
+ assertTrue(handler.existsSubmission(5, getDerbyConnection()));
+ assertFalse(handler.existsSubmission(6, getDerbyConnection()));
+ }
+
+ public void testCreateSubmission() throws Exception {
+ Date creationDate = new Date();
+ Date updateDate = new Date();
+
+ CounterGroup firstGroup = new CounterGroup("ga");
+ CounterGroup secondGroup = new CounterGroup("gb");
+ firstGroup.addCounter(new Counter("ca", 100));
+ firstGroup.addCounter(new Counter("cb", 200));
+ secondGroup.addCounter(new Counter("ca", 300));
+ secondGroup.addCounter(new Counter("cd", 400));
+ Counters counters = new Counters();
+ counters.addCounterGroup(firstGroup);
+ counters.addCounterGroup(secondGroup);
+
+ MSubmission submission = new MSubmission();
+ submission.setJobId(1);
+ submission.setStatus(SubmissionStatus.RUNNING);
+ submission.setCreationDate(creationDate);
+ submission.setLastUpdateDate(updateDate);
+ submission.setExternalId("job-x");
+ submission.setExternalLink("http://somewhere");
+ submission.setExceptionInfo("RuntimeException");
+ submission.setExceptionStackTrace("Yeah it happens");
+ submission.setCounters(counters);
+
+ handler.createSubmission(submission, getDerbyConnection());
+
+ assertEquals(1, submission.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 1);
+
+ List<MSubmission> submissions =
+ handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(1, submissions.size());
+
+ submission = submissions.get(0);
+
+ assertEquals(1, submission.getJobId());
+ assertEquals(SubmissionStatus.RUNNING, submission.getStatus());
+ assertEquals(creationDate, submission.getCreationDate());
+ assertEquals(updateDate, submission.getLastUpdateDate());
+ assertEquals("job-x", submission.getExternalId());
+ assertEquals("http://somewhere", submission.getExternalLink());
+ assertEquals("RuntimeException", submission.getExceptionInfo());
+ assertEquals("Yeah it happens", submission.getExceptionStackTrace());
+
+ CounterGroup group;
+ Counter counter;
+ Counters retrievedCounters = submission.getCounters();
+ assertNotNull(retrievedCounters);
+
+ group = counters.getCounterGroup("ga");
+ assertNotNull(group);
+
+ counter = group.getCounter("ca");
+ assertNotNull(counter);
+ assertEquals(100, counter.getValue());
+
+ counter = group.getCounter("cb");
+ assertNotNull(counter);
+ assertEquals(200, counter.getValue());
+
+ group = counters.getCounterGroup("gb");
+ assertNotNull(group);
+
+ counter = group.getCounter("ca");
+ assertNotNull(counter);
+ assertEquals(300, counter.getValue());
+
+ counter = group.getCounter("cd");
+ assertNotNull(counter);
+ assertEquals(400, counter.getValue());
+
+ // Let's create second (simpler) connection
+ submission =
+ new MSubmission(1, new Date(), SubmissionStatus.SUCCEEDED, "job-x");
+ handler.createSubmission(submission, getDerbyConnection());
+
+ assertEquals(2, submission.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
+ }
+
+ public void testUpdateConnection() throws Exception {
+ loadSubmissions();
+
+ List<MSubmission> submissions =
+ handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(2, submissions.size());
+
+ MSubmission submission = submissions.get(0);
+ submission.setStatus(SubmissionStatus.SUCCEEDED);
+
+ handler.updateSubmission(submission, getDerbyConnection());
+
+ submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(1, submissions.size());
+ }
+
+ public void testPurgeSubmissions() throws Exception {
+ loadSubmissions();
+ List<MSubmission> submissions;
+
+ submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(2, submissions.size());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 5);
+
+ Calendar calendar = Calendar.getInstance();
+ // 2012-01-03 05:05:05
+ calendar.set(2012, Calendar.JANUARY, 3, 5, 5, 5);
+ handler.purgeSubmissions(calendar.getTime(), getDerbyConnection());
+
+ submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(1, submissions.size());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
+
+ handler.purgeSubmissions(new Date(), getDerbyConnection());
+
+ submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(0, submissions.size());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
+
+ handler.purgeSubmissions(new Date(), getDerbyConnection());
+
+ submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ assertNotNull(submissions);
+ assertEquals(0, submissions.size());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
+ }
+
+ /**
+ * Test that by directly removing jobs we will also remove associated
+ * submissions and counters.
+ *
+ * @throws Exception
+ */
+ public void testDeleteJobs() throws Exception {
+ loadSubmissions();
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 5);
+
+ handler.deleteJob(1, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 3);
+
+ handler.deleteJob(2, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
+
+ handler.deleteJob(3, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 1);
+
+ handler.deleteJob(4, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java b/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java
index d840a78..328f9b0 100644
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java
@@ -18,9 +18,7 @@
*/
package org.apache.sqoop.connector.spi;
-import org.apache.sqoop.model.MConnection;
import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MJobForms;
public abstract class MetadataUpgrader {
@@ -45,3 +43,4 @@ public abstract class MetadataUpgrader {
*/
public abstract void upgrade(MJobForms original, MJobForms upgradeTarget);
}
+
[30/52] [abbrv] SQOOP-1478: Sqoop2: Migrate remaining tests from
junit 3 to 4
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
index 7378eda..8dd1ce2 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
@@ -22,11 +22,15 @@ import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
+import org.junit.Before;
+import org.junit.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.junit.Assert.*;
+
/**
* Test link methods on Derby repository.
*/
@@ -34,7 +38,7 @@ public class TestLinkHandling extends DerbyTestCase {
DerbyRepositoryHandler handler;
- @Override
+ @Before
public void setUp() throws Exception {
super.setUp();
@@ -47,6 +51,7 @@ public class TestLinkHandling extends DerbyTestCase {
loadConnectorAndDriverConfig();
}
+ @Test
public void testFindLink() throws Exception {
// Let's try to find non existing link
try {
@@ -81,6 +86,7 @@ public class TestLinkHandling extends DerbyTestCase {
assertNull(forms.get(1).getInputs().get(1).getValue());
}
+ @Test
public void testFindLinks() throws Exception {
List<MLink> list;
@@ -98,6 +104,7 @@ public class TestLinkHandling extends DerbyTestCase {
assertEquals("CB", list.get(1).getName());
}
+ @Test
public void testExistsLink() throws Exception {
// There shouldn't be anything on empty repository
assertFalse(handler.existsLink(1, getDerbyDatabaseConnection()));
@@ -111,6 +118,7 @@ public class TestLinkHandling extends DerbyTestCase {
assertFalse(handler.existsLink(3, getDerbyDatabaseConnection()));
}
+ @Test
public void testCreateLink() throws Exception {
MLink link = getLink();
@@ -150,6 +158,7 @@ public class TestLinkHandling extends DerbyTestCase {
assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 8);
}
+ @Test
public void testInUseLink() throws Exception {
loadLinks();
@@ -160,6 +169,7 @@ public class TestLinkHandling extends DerbyTestCase {
assertTrue(handler.inUseLink(1, getDerbyDatabaseConnection()));
}
+ @Test
public void testUpdateLink() throws Exception {
loadLinks();
@@ -205,6 +215,7 @@ public class TestLinkHandling extends DerbyTestCase {
assertEquals(((Map)forms.get(1).getInputs().get(1).getValue()).size(), 0);
}
+ @Test
public void testEnableAndDisableLink() throws Exception {
loadLinks();
@@ -223,6 +234,7 @@ public class TestLinkHandling extends DerbyTestCase {
assertEquals(true, retrieved.getEnabled());
}
+ @Test
public void testDeleteLink() throws Exception {
loadLinks();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
index 38a995f..cf0944d 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
@@ -22,11 +22,15 @@ import org.apache.sqoop.submission.SubmissionStatus;
import org.apache.sqoop.submission.counter.Counter;
import org.apache.sqoop.submission.counter.CounterGroup;
import org.apache.sqoop.submission.counter.Counters;
+import org.junit.Before;
+import org.junit.Test;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
+import static org.junit.Assert.*;
+
/**
*
*/
@@ -34,7 +38,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
DerbyRepositoryHandler handler;
- @Override
+ @Before
public void setUp() throws Exception {
super.setUp();
@@ -53,6 +57,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
loadJobs();
}
+ @Test
public void testFindSubmissionsUnfinished() throws Exception {
List<MSubmission> submissions;
@@ -67,6 +72,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
assertEquals(2, submissions.size());
}
+ @Test
public void testExistsSubmission() throws Exception {
// There shouldn't be anything on empty repository
assertFalse(handler.existsSubmission(1, getDerbyDatabaseConnection()));
@@ -86,6 +92,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
assertFalse(handler.existsSubmission(6, getDerbyDatabaseConnection()));
}
+ @Test
public void testCreateSubmission() throws Exception {
Date creationDate = new Date();
Date updateDate = new Date();
@@ -168,6 +175,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
}
+ @Test
public void testUpdateConnection() throws Exception {
loadSubmissions();
@@ -186,6 +194,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
assertEquals(1, submissions.size());
}
+ @Test
public void testPurgeSubmissions() throws Exception {
loadSubmissions();
List<MSubmission> submissions;
@@ -226,6 +235,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
*
* @throws Exception
*/
+ @Test
public void testDeleteJobs() throws Exception {
loadSubmissions();
assertCountForTable("SQOOP.SQ_SUBMISSION", 5);
[18/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/json/TestConnectionBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestConnectionBean.java b/common/src/test/java/org/apache/sqoop/json/TestConnectionBean.java
deleted file mode 100644
index 19f81a8..0000000
--- a/common/src/test/java/org/apache/sqoop/json/TestConnectionBean.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json;
-
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MStringInput;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONValue;
-import org.junit.Test;
-
-import java.util.Date;
-
-import static junit.framework.Assert.*;
-import static org.apache.sqoop.json.TestUtil.*;
-
-/**
- *
- */
-public class TestConnectionBean {
- @Test
- public void testSerialization() {
- Date created = new Date();
- Date updated = new Date();
- MConnection connection = getConnection("ahoj");
- connection.setName("Connection");
- connection.setPersistenceId(666);
- connection.setCreationUser("admin");
- connection.setCreationDate(created);
- connection.setLastUpdateUser("user");
- connection.setLastUpdateDate(updated);
- connection.setEnabled(false);
-
- // Fill some data at the beginning
- MStringInput input = (MStringInput) connection.getConnectorPart().getForms()
- .get(0).getInputs().get(0);
- input.setValue("Hi there!");
-
- // Serialize it to JSON object
- ConnectionBean bean = new ConnectionBean(connection);
- JSONObject json = bean.extract(false);
-
- // Check for sensitivity
- JSONArray all = (JSONArray)json.get("all");
- JSONObject allItem = (JSONObject)all.get(0);
- JSONArray connectors = (JSONArray)allItem.get("connector");
- JSONObject connector = (JSONObject)connectors.get(0);
- JSONArray inputs = (JSONArray)connector.get("inputs");
- for (Object input1 : inputs) {
- assertTrue(((JSONObject)input1).containsKey("sensitive"));
- }
-
- // "Move" it across network in text form
- String string = json.toJSONString();
-
- // Retrieved transferred object
- JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
- ConnectionBean retrievedBean = new ConnectionBean();
- retrievedBean.restore(retrievedJson);
- MConnection target = retrievedBean.getConnections().get(0);
-
- // Check id and name
- assertEquals(666, target.getPersistenceId());
- assertEquals("Connection", target.getName());
- assertEquals("admin", target.getCreationUser());
- assertEquals(created, target.getCreationDate());
- assertEquals("user", target.getLastUpdateUser());
- assertEquals(updated, target.getLastUpdateDate());
- assertEquals(false, target.getEnabled());
-
- // Test that value was correctly moved
- MStringInput targetInput = (MStringInput) target.getConnectorPart()
- .getForms().get(0).getInputs().get(0);
- assertEquals("Hi there!", targetInput.getValue());
- }
-
- @Test
- public void testSensitivityFilter() {
- Date created = new Date();
- Date updated = new Date();
- MConnection connection = getConnection("ahoj");
- connection.setName("Connection");
- connection.setPersistenceId(666);
- connection.setCreationUser("admin");
- connection.setCreationDate(created);
- connection.setLastUpdateUser("user");
- connection.setLastUpdateDate(updated);
- connection.setEnabled(true);
-
- // Fill some data at the beginning
- MStringInput input = (MStringInput) connection.getConnectorPart().getForms()
- .get(0).getInputs().get(0);
- input.setValue("Hi there!");
-
- // Serialize it to JSON object
- ConnectionBean bean = new ConnectionBean(connection);
- JSONObject json = bean.extract(false);
- JSONObject jsonFiltered = bean.extract(true);
-
- // Sensitive values should exist
- JSONArray all = (JSONArray)json.get("all");
- JSONObject allItem = (JSONObject)all.get(0);
- JSONArray connectors = (JSONArray)allItem.get("connector");
- JSONObject connector = (JSONObject)connectors.get(0);
- JSONArray inputs = (JSONArray)connector.get("inputs");
- assertEquals(3, inputs.size());
- // Inputs are ordered when creating connection
- JSONObject password = (JSONObject)inputs.get(2);
- assertTrue(password.containsKey("value"));
-
- // Sensitive values should not exist
- all = (JSONArray)jsonFiltered.get("all");
- allItem = (JSONObject)all.get(0);
- connectors = (JSONArray)allItem.get("connector");
- connector = (JSONObject)connectors.get(0);
- inputs = (JSONArray)connector.get("inputs");
- assertEquals(3, inputs.size());
- // Inputs are ordered when creating connection
- password = (JSONObject)inputs.get(2);
- assertFalse(password.containsKey("value"));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java b/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
new file mode 100644
index 0000000..4309f10
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json;
+
+import org.apache.sqoop.model.MDriverConfig;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+import org.junit.Test;
+
+import java.util.ResourceBundle;
+
+import static org.apache.sqoop.json.TestUtil.*;
+
+
+import static org.junit.Assert.*;
+
+/**
+ *
+ */
+public class TestDriverConfigBean {
+
+ /**
+ * Test that by JSON serialization followed by deserialization we will get
+ * equal framework object.
+ */
+ @Test
+ public void testSerialization() {
+ MDriverConfig driverConfig = getDriverConfig();
+
+ // Serialize it to JSON object
+ DriverConfigBean bean = new DriverConfigBean(driverConfig, getResourceBundle());
+ JSONObject json = bean.extract(false);
+
+ // "Move" it across network in text form
+ String string = json.toJSONString();
+
+ // Retrieved transferred object
+ JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
+ DriverConfigBean retrievedBean = new DriverConfigBean();
+ retrievedBean.restore(retrievedJson);
+
+ assertEquals(driverConfig, retrievedBean.getDriverConfig());
+
+ ResourceBundle retrievedBundle = retrievedBean.getResourceBundle();
+ assertEquals("a", retrievedBundle.getString("a"));
+ assertEquals("b", retrievedBundle.getString("b"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/json/TestFrameworkBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestFrameworkBean.java b/common/src/test/java/org/apache/sqoop/json/TestFrameworkBean.java
deleted file mode 100644
index 5cc110a..0000000
--- a/common/src/test/java/org/apache/sqoop/json/TestFrameworkBean.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json;
-
-import org.apache.sqoop.model.MFramework;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-import org.junit.Test;
-
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.json.TestUtil.*;
-
-
-import static org.junit.Assert.*;
-
-/**
- *
- */
-public class TestFrameworkBean {
-
- /**
- * Test that by JSON serialization followed by deserialization we will get
- * equal framework object.
- */
- @Test
- public void testSerialization() {
- MFramework framework = getFramework();
-
- // Serialize it to JSON object
- FrameworkBean bean = new FrameworkBean(framework, getResourceBundle());
- JSONObject json = bean.extract(false);
-
- // "Move" it across network in text form
- String string = json.toJSONString();
-
- // Retrieved transferred object
- JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
- FrameworkBean retrievedBean = new FrameworkBean();
- retrievedBean.restore(retrievedJson);
-
- assertEquals(framework, retrievedBean.getFramework());
-
- ResourceBundle retrievedBundle = retrievedBean.getResourceBundle();
- assertEquals("a", retrievedBundle.getString("a"));
- assertEquals("b", retrievedBundle.getString("b"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestJobBean.java b/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
index 3106d39..c6461bf 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
@@ -68,8 +68,8 @@ public class TestJobBean {
// Check id and name
assertEquals(666, target.getPersistenceId());
- assertEquals(target.getConnectionId(Direction.FROM), 1);
- assertEquals(target.getConnectionId(Direction.TO), 2);
+ assertEquals(target.getLinkId(Direction.FROM), 1);
+ assertEquals(target.getLinkId(Direction.TO), 2);
assertEquals(target.getConnectorId(Direction.FROM), 1);
assertEquals(target.getConnectorId(Direction.TO), 2);
assertEquals("The big job", target.getName());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java b/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
new file mode 100644
index 0000000..8235275
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json;
+
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MStringInput;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONValue;
+import org.junit.Test;
+
+import java.util.Date;
+
+import static junit.framework.Assert.*;
+import static org.apache.sqoop.json.TestUtil.*;
+
+/**
+ *
+ */
+public class TestLinkBean {
+ @Test
+ public void testSerialization() {
+ Date created = new Date();
+ Date updated = new Date();
+ MLink link = getLink("ahoj");
+ link.setName("Connection");
+ link.setPersistenceId(666);
+ link.setCreationUser("admin");
+ link.setCreationDate(created);
+ link.setLastUpdateUser("user");
+ link.setLastUpdateDate(updated);
+ link.setEnabled(false);
+
+ // Fill some data at the beginning
+ MStringInput input = (MStringInput) link.getConnectorPart().getForms()
+ .get(0).getInputs().get(0);
+ input.setValue("Hi there!");
+
+ // Serialize it to JSON object
+ LinkBean bean = new LinkBean(link);
+ JSONObject json = bean.extract(false);
+
+ // Check for sensitivity
+ JSONArray all = (JSONArray)json.get("all");
+ JSONObject allItem = (JSONObject)all.get(0);
+ JSONArray connectors = (JSONArray)allItem.get("connector");
+ JSONObject connector = (JSONObject)connectors.get(0);
+ JSONArray inputs = (JSONArray)connector.get("inputs");
+ for (Object input1 : inputs) {
+ assertTrue(((JSONObject)input1).containsKey("sensitive"));
+ }
+
+ // "Move" it across network in text form
+ String string = json.toJSONString();
+
+ // Retrieved transferred object
+ JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
+ LinkBean retrievedBean = new LinkBean();
+ retrievedBean.restore(retrievedJson);
+ MLink target = retrievedBean.getLinks().get(0);
+
+ // Check id and name
+ assertEquals(666, target.getPersistenceId());
+ assertEquals("Connection", target.getName());
+ assertEquals("admin", target.getCreationUser());
+ assertEquals(created, target.getCreationDate());
+ assertEquals("user", target.getLastUpdateUser());
+ assertEquals(updated, target.getLastUpdateDate());
+ assertEquals(false, target.getEnabled());
+
+ // Test that value was correctly moved
+ MStringInput targetInput = (MStringInput) target.getConnectorPart()
+ .getForms().get(0).getInputs().get(0);
+ assertEquals("Hi there!", targetInput.getValue());
+ }
+
+ @Test
+ public void testSensitivityFilter() {
+ Date created = new Date();
+ Date updated = new Date();
+ MLink link = getLink("ahoj");
+ link.setName("Connection");
+ link.setPersistenceId(666);
+ link.setCreationUser("admin");
+ link.setCreationDate(created);
+ link.setLastUpdateUser("user");
+ link.setLastUpdateDate(updated);
+ link.setEnabled(true);
+
+ // Fill some data at the beginning
+ MStringInput input = (MStringInput) link.getConnectorPart().getForms()
+ .get(0).getInputs().get(0);
+ input.setValue("Hi there!");
+
+ // Serialize it to JSON object
+ LinkBean bean = new LinkBean(link);
+ JSONObject json = bean.extract(false);
+ JSONObject jsonFiltered = bean.extract(true);
+
+ // Sensitive values should exist
+ JSONArray all = (JSONArray)json.get("all");
+ JSONObject allItem = (JSONObject)all.get(0);
+ JSONArray connectors = (JSONArray)allItem.get("connector");
+ JSONObject connector = (JSONObject)connectors.get(0);
+ JSONArray inputs = (JSONArray)connector.get("inputs");
+ assertEquals(3, inputs.size());
+ // Inputs are ordered when creating link
+ JSONObject password = (JSONObject)inputs.get(2);
+ assertTrue(password.containsKey("value"));
+
+ // Sensitive values should not exist
+ all = (JSONArray)jsonFiltered.get("all");
+ allItem = (JSONObject)all.get(0);
+ connectors = (JSONArray)allItem.get("connector");
+ connector = (JSONObject)connectors.get(0);
+ inputs = (JSONArray)connector.get("inputs");
+ assertEquals(3, inputs.size());
+ // Inputs are ordered when creating link
+ password = (JSONObject)inputs.get(2);
+ assertFalse(password.containsKey("value"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/json/TestUtil.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestUtil.java b/common/src/test/java/org/apache/sqoop/json/TestUtil.java
index a9aa233..eaf9122 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestUtil.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestUtil.java
@@ -18,11 +18,11 @@
package org.apache.sqoop.json;
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MJobForms;
@@ -44,23 +44,18 @@ public class TestUtil {
getConnectionForms(), getJobForms(), getJobForms());
}
- public static MFramework getFramework() {
- return new MFramework(getConnectionForms(), getJobForms(), "1");
+ public static MDriverConfig getDriverConfig() {
+ return new MDriverConfig(getConnectionForms(), getJobForms(), "1");
}
- public static MConnection getConnection(String name) {
- return new MConnection(1,
- getConnector(name).getConnectionForms(),
- getFramework().getConnectionForms()
- );
+ public static MLink getLink(String name) {
+ return new MLink(1, getConnector(name).getConnectionForms(), getDriverConfig()
+ .getConnectionForms());
}
public static MJob getJob(String name) {
- return new MJob(1, 2, 1, 2,
- getConnector(name).getJobForms(Direction.FROM),
- getConnector(name).getJobForms(Direction.TO),
- getFramework().getJobForms()
- );
+ return new MJob(1, 2, 1, 2, getConnector(name).getJobForms(Direction.FROM), getConnector(name)
+ .getJobForms(Direction.TO), getDriverConfig().getJobForms());
}
public static MConnectionForms getConnectionForms() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java b/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
index c6086dd..f5f3389 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
@@ -102,9 +102,9 @@ public class TestValidationBean {
}
@Test
- public void testConnectionValidationBeanSerialization() {
+ public void testLinkValidationBeanSerialization() {
// Serialize it to JSON object
- ConnectionValidationBean bean = new ConnectionValidationBean(
+ LinkValidationBean bean = new LinkValidationBean(
getValidation(Status.FINE),
getValidation(Status.UNACCEPTABLE)
);
@@ -115,7 +115,7 @@ public class TestValidationBean {
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
- ConnectionValidationBean retrievedBean = new ConnectionValidationBean();
+ LinkValidationBean retrievedBean = new LinkValidationBean();
retrievedBean.restore(retrievedJson);
assertNull(retrievedBean.getId());
@@ -139,9 +139,9 @@ public class TestValidationBean {
}
@Test
- public void testConnectionValidationBeanId() {
+ public void testLinkValidationBeanId() {
// Serialize it to JSON object
- ConnectionValidationBean bean = new ConnectionValidationBean(
+ LinkValidationBean bean = new LinkValidationBean(
getValidation(Status.FINE),
getValidation(Status.FINE)
);
@@ -153,7 +153,7 @@ public class TestValidationBean {
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
- ConnectionValidationBean retrievedBean = new ConnectionValidationBean();
+ LinkValidationBean retrievedBean = new LinkValidationBean();
retrievedBean.restore(retrievedJson);
assertEquals((Long)(long) 10, retrievedBean.getId());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java b/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
index f3d4166..d56e165 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
@@ -41,21 +41,21 @@ public class TestMAccountableEntity {
list.add(input);
MForm form = new MForm("FORMNAME", list);
forms.add(form);
- MAccountableEntity connection = new MConnection(123l, new MConnectionForms(
+ MAccountableEntity link = new MLink(123l, new MConnectionForms(
forms), new MConnectionForms(forms));
// Initially creation date and last update date is same
- assertEquals(connection.getCreationDate(), connection.getLastUpdateDate());
+ assertEquals(link.getCreationDate(), link.getLastUpdateDate());
Date testCreationDate = new Date();
Date testLastUpdateDate = new Date();
- connection.setCreationUser("admin");
- connection.setCreationDate(testCreationDate);
- connection.setLastUpdateUser("user");
- connection.setLastUpdateDate(testLastUpdateDate);
- connection.setEnabled(false);
- assertEquals(testCreationDate, connection.getCreationDate());
- assertEquals("admin", connection.getCreationUser());
- assertEquals(testLastUpdateDate, connection.getLastUpdateDate());
- assertEquals(false, connection.getEnabled());
- assertEquals("user", connection.getLastUpdateUser());
+ link.setCreationUser("admin");
+ link.setCreationDate(testCreationDate);
+ link.setLastUpdateUser("user");
+ link.setLastUpdateDate(testLastUpdateDate);
+ link.setEnabled(false);
+ assertEquals(testCreationDate, link.getCreationDate());
+ assertEquals("admin", link.getCreationUser());
+ assertEquals(testLastUpdateDate, link.getLastUpdateDate());
+ assertEquals(false, link.getEnabled());
+ assertEquals("user", link.getLastUpdateUser());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/model/TestMConnection.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConnection.java b/common/src/test/java/org/apache/sqoop/model/TestMConnection.java
deleted file mode 100644
index 27959fb..0000000
--- a/common/src/test/java/org/apache/sqoop/model/TestMConnection.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Test;
-
-/**
- * Test class for org.apache.sqoop.model.MConnection
- */
-public class TestMConnection {
-
- /**
- * Test for initialization
- */
- @Test
- public void testInitialization() {
- // Test default constructor
- MConnection connection = connection();
- assertEquals(123l, connection.getConnectorId());
- assertEquals("Vampire", connection.getName());
- assertEquals("Buffy", connection.getCreationUser());
- assertEquals(forms1(), connection.getConnectorPart());
- assertEquals(forms2(), connection.getFrameworkPart());
-
- // Test copy constructor
- MConnection copy = new MConnection(connection);
- assertEquals(123l, copy.getConnectorId());
- assertEquals("Vampire", copy.getName());
- assertEquals("Buffy", copy.getCreationUser());
- assertEquals(connection.getCreationDate(), copy.getCreationDate());
- assertEquals(forms1(), copy.getConnectorPart());
- assertEquals(forms2(), copy.getFrameworkPart());
-
- // Test constructor for metadata upgrade (the order of forms is different)
- MConnection upgradeCopy = new MConnection(connection, forms2(), forms1());
- assertEquals(123l, upgradeCopy.getConnectorId());
- assertEquals("Vampire", upgradeCopy.getName());
- assertEquals("Buffy", upgradeCopy.getCreationUser());
- assertEquals(connection.getCreationDate(), upgradeCopy.getCreationDate());
- assertEquals(forms2(), upgradeCopy.getConnectorPart());
- assertEquals(forms1(), upgradeCopy.getFrameworkPart());
- }
-
- @Test
- public void testClone() {
- MConnection connection = connection();
-
- // Clone without value
- MConnection withoutValue = connection.clone(false);
- assertEquals(connection, withoutValue);
- assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutValue.getPersistenceId());
- assertNull(withoutValue.getName());
- assertNull(withoutValue.getCreationUser());
- assertEquals(forms1(), withoutValue.getConnectorPart());
- assertEquals(forms2(), withoutValue.getFrameworkPart());
- assertNull(withoutValue.getConnectorPart().getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
- assertNull(withoutValue.getConnectorPart().getForm("FORMNAME").getInput("STRING-INPUT").getValue());
-
- // Clone with value
- MConnection withValue = connection.clone(true);
- assertEquals(connection, withValue);
- assertEquals(connection.getPersistenceId(), withValue.getPersistenceId());
- assertEquals(connection.getName(), withValue.getName());
- assertEquals(connection.getCreationUser(), withValue.getCreationUser());
- assertEquals(forms1(), withValue.getConnectorPart());
- assertEquals(forms2(), withValue.getFrameworkPart());
- assertEquals(100, withValue.getConnectorPart().getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
- assertEquals("TEST-VALUE", withValue.getConnectorPart().getForm("FORMNAME").getInput("STRING-INPUT").getValue());
- }
-
- private MConnection connection() {
- MConnection connection = new MConnection(123l, forms1(), forms2());
- connection.setName("Vampire");
- connection.setCreationUser("Buffy");
- return connection;
- }
-
- private MConnectionForms forms1() {
- List<MForm> forms = new ArrayList<MForm>();
- MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
- input.setValue(100);
- MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
- strInput.setValue("TEST-VALUE");
- List<MInput<?>> list = new ArrayList<MInput<?>>();
- list.add(input);
- list.add(strInput);
- MForm form = new MForm("FORMNAME", list);
- forms.add(form);
- return new MConnectionForms(forms);
- }
-
- private MConnectionForms forms2() {
- List<MForm> forms = new ArrayList<MForm>();
- MMapInput input = new MMapInput("MAP-INPUT", false);
- List<MInput<?>> list = new ArrayList<MInput<?>>();
- list.add(input);
- MForm form = new MForm("form", list);
- forms.add(form);
- return new MConnectionForms(forms);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java b/common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java
new file mode 100644
index 0000000..9c23cc3
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ *
+ */
+public class TestMDriverConfig {
+
+ @Test
+ public void testForms() {
+ List<MForm> connectionFormList = new ArrayList<MForm>();
+ List<MForm> jobFormList = new ArrayList<MForm>();
+ connectionFormList.add(new MForm("connection-test", new ArrayList<MInput<?>>()));
+ jobFormList.add(new MForm("job-test", new ArrayList<MInput<?>>()));
+ MConnectionForms connectionForms = new MConnectionForms(connectionFormList);
+ MJobForms jobForms = new MJobForms(jobFormList);
+
+ MDriverConfig driver = new MDriverConfig(connectionForms, jobForms, "1");
+ assertEquals(1, driver.getJobForms().getForms().size());
+ assertEquals("job-test", driver.getJobForms().getForms().get(0).getName());
+ assertEquals(1, driver.getConnectionForms().getForms().size());
+ assertEquals("connection-test", driver.getConnectionForms().getForms().get(0).getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/model/TestMFramework.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMFramework.java b/common/src/test/java/org/apache/sqoop/model/TestMFramework.java
deleted file mode 100644
index 5273a90..0000000
--- a/common/src/test/java/org/apache/sqoop/model/TestMFramework.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-/**
- *
- */
-public class TestMFramework {
-
- @Test
- public void testForms() {
- List<MForm> connectionFormList = new ArrayList<MForm>();
- List<MForm> jobFormList = new ArrayList<MForm>();
- connectionFormList.add(new MForm("connection-test", new ArrayList<MInput<?>>()));
- jobFormList.add(new MForm("job-test", new ArrayList<MInput<?>>()));
- MConnectionForms connectionForms = new MConnectionForms(connectionFormList);
- MJobForms jobForms = new MJobForms(jobFormList);
-
- MFramework framework = new MFramework(connectionForms, jobForms, "1");
- assertEquals(1, framework.getJobForms().getForms().size());
- assertEquals("job-test", framework.getJobForms().getForms().get(0).getName());
- assertEquals(1, framework.getConnectionForms().getForms().size());
- assertEquals("connection-test", framework.getConnectionForms().getForms().get(0).getName());
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/test/java/org/apache/sqoop/model/TestMLink.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMLink.java b/common/src/test/java/org/apache/sqoop/model/TestMLink.java
new file mode 100644
index 0000000..1be37a1
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/model/TestMLink.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+/**
+ * Test class for org.apache.sqoop.model.MConnection
+ */
+public class TestMLink {
+
+ /**
+ * Test for initialization
+ */
+ @Test
+ public void testInitialization() {
+ // Test default constructor
+ MLink link = link();
+ assertEquals(123l, link.getConnectorId());
+ assertEquals("Vampire", link.getName());
+ assertEquals("Buffy", link.getCreationUser());
+ assertEquals(forms1(), link.getConnectorPart());
+ assertEquals(forms2(), link.getFrameworkPart());
+
+ // Test copy constructor
+ MLink copy = new MLink(link);
+ assertEquals(123l, copy.getConnectorId());
+ assertEquals("Vampire", copy.getName());
+ assertEquals("Buffy", copy.getCreationUser());
+ assertEquals(link.getCreationDate(), copy.getCreationDate());
+ assertEquals(forms1(), copy.getConnectorPart());
+ assertEquals(forms2(), copy.getFrameworkPart());
+
+ // Test constructor for metadata upgrade (the order of forms is different)
+ MLink upgradeCopy = new MLink(link, forms2(), forms1());
+ assertEquals(123l, upgradeCopy.getConnectorId());
+ assertEquals("Vampire", upgradeCopy.getName());
+ assertEquals("Buffy", upgradeCopy.getCreationUser());
+ assertEquals(link.getCreationDate(), upgradeCopy.getCreationDate());
+ assertEquals(forms2(), upgradeCopy.getConnectorPart());
+ assertEquals(forms1(), upgradeCopy.getFrameworkPart());
+ }
+
+ @Test
+ public void testClone() {
+ MLink link = link();
+
+ // Clone without value
+ MLink withoutValue = link.clone(false);
+ assertEquals(link, withoutValue);
+ assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutValue.getPersistenceId());
+ assertNull(withoutValue.getName());
+ assertNull(withoutValue.getCreationUser());
+ assertEquals(forms1(), withoutValue.getConnectorPart());
+ assertEquals(forms2(), withoutValue.getFrameworkPart());
+ assertNull(withoutValue.getConnectorPart().getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
+ assertNull(withoutValue.getConnectorPart().getForm("FORMNAME").getInput("STRING-INPUT").getValue());
+
+ // Clone with value
+ MLink withValue = link.clone(true);
+ assertEquals(link, withValue);
+ assertEquals(link.getPersistenceId(), withValue.getPersistenceId());
+ assertEquals(link.getName(), withValue.getName());
+ assertEquals(link.getCreationUser(), withValue.getCreationUser());
+ assertEquals(forms1(), withValue.getConnectorPart());
+ assertEquals(forms2(), withValue.getFrameworkPart());
+ assertEquals(100, withValue.getConnectorPart().getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
+ assertEquals("TEST-VALUE", withValue.getConnectorPart().getForm("FORMNAME").getInput("STRING-INPUT").getValue());
+ }
+
+ private MLink link() {
+ MLink link = new MLink(123l, forms1(), forms2());
+ link.setName("Vampire");
+ link.setCreationUser("Buffy");
+ return link;
+ }
+
+ private MConnectionForms forms1() {
+ List<MForm> forms = new ArrayList<MForm>();
+ MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
+ input.setValue(100);
+ MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
+ strInput.setValue("TEST-VALUE");
+ List<MInput<?>> list = new ArrayList<MInput<?>>();
+ list.add(input);
+ list.add(strInput);
+ MForm form = new MForm("FORMNAME", list);
+ forms.add(form);
+ return new MConnectionForms(forms);
+ }
+
+ private MConnectionForms forms2() {
+ List<MForm> forms = new ArrayList<MForm>();
+ MMapInput input = new MMapInput("MAP-INPUT", false);
+ List<MInput<?>> list = new ArrayList<MInput<?>>();
+ list.add(input);
+ MForm form = new MForm("form", list);
+ forms.add(form);
+ return new MConnectionForms(forms);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
index 3790daa..b4b6966 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
@@ -22,10 +22,10 @@ import java.util.ResourceBundle;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.VersionInfo;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.job.etl.From;
import org.apache.sqoop.job.etl.To;
import org.apache.sqoop.connector.spi.SqoopConnector;
@@ -65,8 +65,8 @@ public class GenericJdbcConnector extends SqoopConnector {
}
@Override
- public Class getConnectionConfigurationClass() {
- return ConnectionConfiguration.class;
+ public Class getLinkConfigurationClass() {
+ return LinkConfiguration.class;
}
@Override
@@ -97,8 +97,8 @@ public class GenericJdbcConnector extends SqoopConnector {
}
@Override
- public MetadataUpgrader getMetadataUpgrader() {
- return new GenericJdbcConnectorMetadataUpgrader();
+ public RepositoryUpgrader getRepositoryUpgrader() {
+ return new GenericJdbcConnectorUpgrader();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
index 62da2db..f63f013 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
@@ -24,7 +24,7 @@ public final class GenericJdbcConnectorConstants {
// Resource bundle name
public static final String RESOURCE_BUNDLE_NAME =
- "generic-jdbc-connector-resources";
+ "generic-jdbc-connector-config";
/*
* All jdbc connector related configuration is prefixed with this:
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java
deleted file mode 100644
index cbe72f6..0000000
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sqoop.connector.jdbc;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJobForms;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class GenericJdbcConnectorMetadataUpgrader extends MetadataUpgrader {
- private static final Logger LOG =
- Logger.getLogger(GenericJdbcConnectorMetadataUpgrader.class);
-
- /*
- * For now, there is no real upgrade. So copy all data over,
- * set the validation messages and error messages to be the same as for the
- * inputs in the original one.
- */
-
- @Override
- public void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
- }
-
- @Override
- public void upgrade(MJobForms original, MJobForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
- }
-
- @SuppressWarnings("unchecked")
- private void doUpgrade(List<MForm> original, List<MForm> target) {
- // Easier to find the form in the original forms list if we use a map.
- // Since the constructor of MJobForms takes a list,
- // index is not guaranteed to be the same, so we need to look for
- // equivalence
- Map<String, MForm> formMap = new HashMap<String, MForm>();
- for (MForm form : original) {
- formMap.put(form.getName(), form);
- }
- for (MForm form : target) {
- List<MInput<?>> inputs = form.getInputs();
- MForm originalForm = formMap.get(form.getName());
- if (originalForm == null) {
- LOG.warn("Form: '" + form.getName() + "' not present in old " +
- "connector. So it and its inputs will not be transferred by the upgrader.");
- continue;
- }
- for (MInput input : inputs) {
- try {
- MInput originalInput = originalForm.getInput(input.getName());
- input.setValue(originalInput.getValue());
- } catch (SqoopException ex) {
- LOG.warn("Input: '" + input.getName() + "' not present in old " +
- "connector. So it will not be transferred by the upgrader.");
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
new file mode 100644
index 0000000..8deddb0
--- /dev/null
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.connector.jdbc;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MJobForms;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class GenericJdbcConnectorUpgrader extends RepositoryUpgrader {
+ private static final Logger LOG =
+ Logger.getLogger(GenericJdbcConnectorUpgrader.class);
+
+ /*
+ * For now, there is no real upgrade. So copy all data over,
+ * set the validation messages and error messages to be the same as for the
+ * inputs in the original one.
+ */
+
+ @Override
+ public void upgrade(MConnectionForms original,
+ MConnectionForms upgradeTarget) {
+ doUpgrade(original.getForms(), upgradeTarget.getForms());
+ }
+
+ @Override
+ public void upgrade(MJobForms original, MJobForms upgradeTarget) {
+ doUpgrade(original.getForms(), upgradeTarget.getForms());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void doUpgrade(List<MForm> original, List<MForm> target) {
+ // Easier to find the form in the original forms list if we use a map.
+ // Since the constructor of MJobForms takes a list,
+ // index is not guaranteed to be the same, so we need to look for
+ // equivalence
+ Map<String, MForm> formMap = new HashMap<String, MForm>();
+ for (MForm form : original) {
+ formMap.put(form.getName(), form);
+ }
+ for (MForm form : target) {
+ List<MInput<?>> inputs = form.getInputs();
+ MForm originalForm = formMap.get(form.getName());
+ if (originalForm == null) {
+ LOG.warn("Form: '" + form.getName() + "' not present in old " +
+ "connector. So it and its inputs will not be transferred by the upgrader.");
+ continue;
+ }
+ for (MInput input : inputs) {
+ try {
+ MInput originalInput = originalForm.getInput(input.getName());
+ input.setValue(originalInput.getValue());
+ } catch (SqoopException ex) {
+ LOG.warn("Input: '" + input.getName() + "' not present in old " +
+ "connector. So it will not be transferred by the upgrader.");
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
index 9915603..e52610a 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
@@ -23,22 +23,23 @@ import java.sql.SQLException;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.job.etl.ExtractorContext;
import org.apache.sqoop.job.etl.Extractor;
-public class GenericJdbcExtractor extends Extractor<ConnectionConfiguration, FromJobConfiguration, GenericJdbcPartition> {
+public class GenericJdbcExtractor extends Extractor<LinkConfiguration, FromJobConfiguration, GenericJdbcPartition> {
public static final Logger LOG = Logger.getLogger(GenericJdbcExtractor.class);
private long rowsRead = 0;
@Override
- public void extract(ExtractorContext context, ConnectionConfiguration connection, FromJobConfiguration job, GenericJdbcPartition partition) {
- String driver = connection.connection.jdbcDriver;
- String url = connection.connection.connectionString;
- String username = connection.connection.username;
- String password = connection.connection.password;
+ public void extract(ExtractorContext context, LinkConfiguration linkConf,
+ FromJobConfiguration fromJobConf, GenericJdbcPartition partition) {
+ String driver = linkConf.link.jdbcDriver;
+ String url = linkConf.link.connectionString;
+ String username = linkConf.link.username;
+ String password = linkConf.link.password;
GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password);
String query = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
index 2df193c..d3a893f 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
@@ -18,18 +18,18 @@
package org.apache.sqoop.connector.jdbc;
import org.apache.log4j.Logger;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.job.etl.Destroyer;
import org.apache.sqoop.job.etl.DestroyerContext;
-public class GenericJdbcFromDestroyer extends Destroyer<ConnectionConfiguration, FromJobConfiguration> {
+public class GenericJdbcFromDestroyer extends Destroyer<LinkConfiguration, FromJobConfiguration> {
private static final Logger LOG =
Logger.getLogger(GenericJdbcFromDestroyer.class);
@Override
- public void destroy(DestroyerContext context, ConnectionConfiguration connection, FromJobConfiguration job) {
+ public void destroy(DestroyerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
LOG.info("Running generic JDBC connector destroyer");
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
index 87e0862..9d0c178 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
@@ -27,7 +27,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.connector.jdbc.util.SqlTypesUtils;
import org.apache.sqoop.job.Constants;
@@ -37,7 +37,7 @@ import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Column;
import org.apache.sqoop.utils.ClassUtils;
-public class GenericJdbcFromInitializer extends Initializer<ConnectionConfiguration, FromJobConfiguration> {
+public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, FromJobConfiguration> {
private static final Logger LOG =
Logger.getLogger(GenericJdbcFromInitializer.class);
@@ -45,34 +45,34 @@ public class GenericJdbcFromInitializer extends Initializer<ConnectionConfigurat
private GenericJdbcExecutor executor;
@Override
- public void initialize(InitializerContext context, ConnectionConfiguration connection, FromJobConfiguration job) {
- configureJdbcProperties(context.getContext(), connection, job);
+ public void initialize(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
+ configureJdbcProperties(context.getContext(), linkConf, fromJobConf);
try {
- configurePartitionProperties(context.getContext(), connection, job);
- configureTableProperties(context.getContext(), connection, job);
+ configurePartitionProperties(context.getContext(), linkConf, fromJobConf);
+ configureTableProperties(context.getContext(), linkConf, fromJobConf);
} finally {
executor.close();
}
}
@Override
- public List<String> getJars(InitializerContext context, ConnectionConfiguration connection, FromJobConfiguration job) {
+ public List<String> getJars(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
List<String> jars = new LinkedList<String>();
- jars.add(ClassUtils.jarForClass(connection.connection.jdbcDriver));
+ jars.add(ClassUtils.jarForClass(linkConf.link.jdbcDriver));
return jars;
}
@Override
- public Schema getSchema(InitializerContext context, ConnectionConfiguration connectionConfiguration, FromJobConfiguration fromJobConfiguration) {
- configureJdbcProperties(context.getContext(), connectionConfiguration, fromJobConfiguration);
+ public Schema getSchema(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
+ configureJdbcProperties(context.getContext(), linkConf, fromJobConf);
- String schemaName = fromJobConfiguration.fromTable.tableName;
+ String schemaName = fromJobConf.fromJobConfig.tableName;
if(schemaName == null) {
schemaName = "Query";
- } else if(fromJobConfiguration.fromTable.schemaName != null) {
- schemaName = fromJobConfiguration.fromTable.schemaName + "." + schemaName;
+ } else if(fromJobConf.fromJobConfig.schemaName != null) {
+ schemaName = fromJobConf.fromJobConfig.schemaName + "." + schemaName;
}
Schema schema = new Schema(schemaName);
@@ -117,11 +117,11 @@ public class GenericJdbcFromInitializer extends Initializer<ConnectionConfigurat
}
}
- private void configureJdbcProperties(MutableContext context, ConnectionConfiguration connectionConfig, FromJobConfiguration jobConfig) {
- String driver = connectionConfig.connection.jdbcDriver;
- String url = connectionConfig.connection.connectionString;
- String username = connectionConfig.connection.username;
- String password = connectionConfig.connection.password;
+ private void configureJdbcProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
+ String driver = connectionConfig.link.jdbcDriver;
+ String url = connectionConfig.link.connectionString;
+ String username = connectionConfig.link.username;
+ String password = connectionConfig.link.password;
assert driver != null;
assert url != null;
@@ -129,15 +129,15 @@ public class GenericJdbcFromInitializer extends Initializer<ConnectionConfigurat
executor = new GenericJdbcExecutor(driver, url, username, password);
}
- private void configurePartitionProperties(MutableContext context, ConnectionConfiguration connectionConfig, FromJobConfiguration jobConfig) {
+ private void configurePartitionProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
// ----- configure column name -----
- String partitionColumnName = jobConfig.fromTable.partitionColumn;
+ String partitionColumnName = fromJobConfig.fromJobConfig.partitionColumn;
if (partitionColumnName == null) {
// if column is not specified by the user,
// find the primary key of the fromTable (when there is a fromTable).
- String tableName = jobConfig.fromTable.tableName;
+ String tableName = fromJobConfig.fromJobConfig.tableName;
if (tableName != null) {
partitionColumnName = executor.getPrimaryKey(tableName);
}
@@ -155,14 +155,14 @@ public class GenericJdbcFromInitializer extends Initializer<ConnectionConfigurat
// ----- configure column type, min value, and max value -----
- String minMaxQuery = jobConfig.fromTable.boundaryQuery;
+ String minMaxQuery = fromJobConfig.fromJobConfig.boundaryQuery;
if (minMaxQuery == null) {
StringBuilder builder = new StringBuilder();
- String schemaName = jobConfig.fromTable.schemaName;
- String tableName = jobConfig.fromTable.tableName;
- String tableSql = jobConfig.fromTable.sql;
+ String schemaName = fromJobConfig.fromJobConfig.schemaName;
+ String tableName = fromJobConfig.fromJobConfig.tableName;
+ String tableSql = fromJobConfig.fromJobConfig.sql;
if (tableName != null && tableSql != null) {
// when both fromTable name and fromTable sql are specified:
@@ -234,14 +234,14 @@ public class GenericJdbcFromInitializer extends Initializer<ConnectionConfigurat
}
}
- private void configureTableProperties(MutableContext context, ConnectionConfiguration connectionConfig, FromJobConfiguration jobConfig) {
+ private void configureTableProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
String dataSql;
String fieldNames;
- String schemaName = jobConfig.fromTable.schemaName;
- String tableName = jobConfig.fromTable.tableName;
- String tableSql = jobConfig.fromTable.sql;
- String tableColumns = jobConfig.fromTable.columns;
+ String schemaName = fromJobConfig.fromJobConfig.schemaName;
+ String tableName = fromJobConfig.fromJobConfig.tableName;
+ String tableSql = fromJobConfig.fromJobConfig.sql;
+ String tableColumns = fromJobConfig.fromJobConfig.columns;
if (tableName != null && tableSql != null) {
// when both fromTable name and fromTable sql are specified:
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
index 07ae988..991e686 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
@@ -17,12 +17,12 @@
*/
package org.apache.sqoop.connector.jdbc;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.job.etl.Loader;
import org.apache.sqoop.job.etl.LoaderContext;
-public class GenericJdbcLoader extends Loader<ConnectionConfiguration, ToJobConfiguration> {
+public class GenericJdbcLoader extends Loader<LinkConfiguration, ToJobConfiguration> {
public static final int DEFAULT_ROWS_PER_BATCH = 100;
public static final int DEFAULT_BATCHES_PER_TRANSACTION = 100;
@@ -30,11 +30,11 @@ public class GenericJdbcLoader extends Loader<ConnectionConfiguration, ToJobConf
private int batchesPerTransaction = DEFAULT_BATCHES_PER_TRANSACTION;
@Override
- public void load(LoaderContext context, ConnectionConfiguration connection, ToJobConfiguration job) throws Exception{
- String driver = connection.connection.jdbcDriver;
- String url = connection.connection.connectionString;
- String username = connection.connection.username;
- String password = connection.connection.password;
+ public void load(LoaderContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) throws Exception{
+ String driver = linkConf.link.jdbcDriver;
+ String url = linkConf.link.connectionString;
+ String username = linkConf.link.username;
+ String password = linkConf.link.password;
GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password);
executor.setAutoCommit(false);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
index a25f344..6b11228 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
@@ -28,13 +28,13 @@ import java.util.List;
import java.util.TimeZone;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.job.etl.Partition;
import org.apache.sqoop.job.etl.Partitioner;
import org.apache.sqoop.job.etl.PartitionerContext;
-public class GenericJdbcPartitioner extends Partitioner<ConnectionConfiguration, FromJobConfiguration> {
+public class GenericJdbcPartitioner extends Partitioner<LinkConfiguration, FromJobConfiguration> {
private static final BigDecimal NUMERIC_MIN_INCREMENT = new BigDecimal(10000 * Double.MIN_VALUE);
@@ -47,7 +47,7 @@ public class GenericJdbcPartitioner extends Partitioner<ConnectionConfiguration,
private Boolean partitionColumnNull;
@Override
- public List<Partition> getPartitions(PartitionerContext context,ConnectionConfiguration connection, FromJobConfiguration job) {
+ public List<Partition> getPartitions(PartitionerContext context,LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
List<Partition> partitions = new LinkedList<Partition>();
numberPartitions = context.getMaxPartitions();
@@ -56,7 +56,7 @@ public class GenericJdbcPartitioner extends Partitioner<ConnectionConfiguration,
partitionMinValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE);
partitionMaxValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE);
- partitionColumnNull = job.fromTable.partitionColumnNull;
+ partitionColumnNull = fromJobConf.fromJobConfig.partitionColumnNull;
if (partitionColumnNull == null) {
partitionColumnNull = false;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
index 4ba278e..7bed1d9 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
@@ -18,36 +18,36 @@
package org.apache.sqoop.connector.jdbc;
import org.apache.log4j.Logger;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.job.etl.Destroyer;
import org.apache.sqoop.job.etl.DestroyerContext;
-public class GenericJdbcToDestroyer extends Destroyer<ConnectionConfiguration, ToJobConfiguration> {
+public class GenericJdbcToDestroyer extends Destroyer<LinkConfiguration, ToJobConfiguration> {
private static final Logger LOG = Logger.getLogger(GenericJdbcToDestroyer.class);
@Override
- public void destroy(DestroyerContext context, ConnectionConfiguration connection, ToJobConfiguration job) {
+ public void destroy(DestroyerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
LOG.info("Running generic JDBC connector destroyer");
- final String tableName = job.toTable.tableName;
- final String stageTableName = job.toTable.stageTableName;
+ final String tableName = toJobConf.toJobConfig.tableName;
+ final String stageTableName = toJobConf.toJobConfig.stageTableName;
final boolean stageEnabled = stageTableName != null &&
stageTableName.length() > 0;
if(stageEnabled) {
- moveDataToDestinationTable(connection,
+ moveDataToDestinationTable(linkConf,
context.isSuccess(), stageTableName, tableName);
}
}
- private void moveDataToDestinationTable(ConnectionConfiguration connectorConf,
+ private void moveDataToDestinationTable(LinkConfiguration linkConf,
boolean success, String stageTableName, String tableName) {
GenericJdbcExecutor executor =
- new GenericJdbcExecutor(connectorConf.connection.jdbcDriver,
- connectorConf.connection.connectionString,
- connectorConf.connection.username,
- connectorConf.connection.password);
+ new GenericJdbcExecutor(linkConf.link.jdbcDriver,
+ linkConf.link.connectionString,
+ linkConf.link.username,
+ linkConf.link.password);
try {
if(success) {
LOG.info("Job completed, transferring data from stage fromTable to " +
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
index 73a49b1..5d0ec93 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
@@ -27,7 +27,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.connector.jdbc.util.SqlTypesUtils;
import org.apache.sqoop.job.etl.Initializer;
@@ -36,44 +36,42 @@ import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Column;
import org.apache.sqoop.utils.ClassUtils;
-public class GenericJdbcToInitializer extends Initializer<ConnectionConfiguration, ToJobConfiguration> {
+public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJobConfiguration> {
private GenericJdbcExecutor executor;
private static final Logger LOG =
Logger.getLogger(GenericJdbcToInitializer.class);
@Override
- public void initialize(InitializerContext context, ConnectionConfiguration connection, ToJobConfiguration job) {
- configureJdbcProperties(context.getContext(), connection, job);
+ public void initialize(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
+ configureJdbcProperties(context.getContext(), linkConf, toJobConf);
try {
- configureTableProperties(context.getContext(), connection, job);
+ configureTableProperties(context.getContext(), linkConf, toJobConf);
} finally {
executor.close();
}
}
@Override
- public List<String> getJars(InitializerContext context, ConnectionConfiguration connection, ToJobConfiguration job) {
+ public List<String> getJars(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
List<String> jars = new LinkedList<String>();
-
- jars.add(ClassUtils.jarForClass(connection.connection.jdbcDriver));
-
+ jars.add(ClassUtils.jarForClass(linkConf.link.jdbcDriver));
return jars;
}
@Override
- public Schema getSchema(InitializerContext context, ConnectionConfiguration connectionConfiguration, ToJobConfiguration toJobConfiguration) {
- configureJdbcProperties(context.getContext(), connectionConfiguration, toJobConfiguration);
+ public Schema getSchema(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
+ configureJdbcProperties(context.getContext(), linkConf, toJobConf);
- String schemaName = toJobConfiguration.toTable.tableName;
+ String schemaName = toJobConf.toJobConfig.tableName;
if (schemaName == null) {
throw new SqoopException(GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0019,
"Table name extraction not supported yet.");
}
- if(toJobConfiguration.toTable.schemaName != null) {
- schemaName = toJobConfiguration.toTable.schemaName + "." + schemaName;
+ if(toJobConf.toJobConfig.schemaName != null) {
+ schemaName = toJobConf.toJobConfig.schemaName + "." + schemaName;
}
Schema schema = new Schema(schemaName);
@@ -112,11 +110,11 @@ public class GenericJdbcToInitializer extends Initializer<ConnectionConfiguratio
}
}
- private void configureJdbcProperties(MutableContext context, ConnectionConfiguration connectionConfig, ToJobConfiguration jobConfig) {
- String driver = connectionConfig.connection.jdbcDriver;
- String url = connectionConfig.connection.connectionString;
- String username = connectionConfig.connection.username;
- String password = connectionConfig.connection.password;
+ private void configureJdbcProperties(MutableContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
+ String driver = linkConf.link.jdbcDriver;
+ String url = linkConf.link.connectionString;
+ String username = linkConf.link.username;
+ String password = linkConf.link.password;
assert driver != null;
assert url != null;
@@ -124,18 +122,18 @@ public class GenericJdbcToInitializer extends Initializer<ConnectionConfiguratio
executor = new GenericJdbcExecutor(driver, url, username, password);
}
- private void configureTableProperties(MutableContext context, ConnectionConfiguration connectionConfig, ToJobConfiguration jobConfig) {
+ private void configureTableProperties(MutableContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConfig) {
String dataSql;
- String schemaName = jobConfig.toTable.schemaName;
- String tableName = jobConfig.toTable.tableName;
- String stageTableName = jobConfig.toTable.stageTableName;
- boolean clearStageTable = jobConfig.toTable.clearStageTable == null ?
- false : jobConfig.toTable.clearStageTable;
+ String schemaName = toJobConfig.toJobConfig.schemaName;
+ String tableName = toJobConfig.toJobConfig.tableName;
+ String stageTableName = toJobConfig.toJobConfig.stageTableName;
+ boolean clearStageTable = toJobConfig.toJobConfig.clearStageTable == null ?
+ false : toJobConfig.toJobConfig.clearStageTable;
final boolean stageEnabled =
stageTableName != null && stageTableName.length() > 0;
- String tableSql = jobConfig.toTable.sql;
- String tableColumns = jobConfig.toTable.columns;
+ String tableSql = toJobConfig.toJobConfig.sql;
+ String tableColumns = toJobConfig.toJobConfig.columns;
if (tableName != null && tableSql != null) {
// when both fromTable name and fromTable sql are specified:
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
index 0a60e90..ad1ee5c 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.connector.jdbc;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.validation.Status;
@@ -34,32 +34,32 @@ import java.sql.SQLException;
public class GenericJdbcValidator extends Validator {
@Override
- public Validation validateConnection(Object configuration) {
- Validation validation = new Validation(ConnectionConfiguration.class);
- ConnectionConfiguration config = (ConnectionConfiguration)configuration;
+ public Validation validateLink(Object configuration) {
+ Validation validation = new Validation(LinkConfiguration.class);
+ LinkConfiguration linkConf = (LinkConfiguration)configuration;
- if(config.connection.jdbcDriver == null) {
- validation.addMessage(Status.UNACCEPTABLE, "connection", "jdbcDriver", "Driver can't be empty");
+ if(linkConf.link.jdbcDriver == null) {
+ validation.addMessage(Status.UNACCEPTABLE, "link", "jdbcDriver", "Driver can't be empty");
} else {
try {
- Class.forName(config.connection.jdbcDriver);
+ Class.forName(linkConf.link.jdbcDriver);
} catch (ClassNotFoundException e) {
- validation.addMessage(Status.UNACCEPTABLE, "connection", "jdbcDriver", "Can't load specified driver");
+ validation.addMessage(Status.UNACCEPTABLE, "link", "jdbcDriver", "Can't load specified driver");
}
}
- if(config.connection.connectionString == null) {
- validation.addMessage(Status.UNACCEPTABLE, "connection", "connectionString", "JDBC URL can't be empty");
- } else if(!config.connection.connectionString.startsWith("jdbc:")) {
- validation.addMessage(Status.UNACCEPTABLE, "connection", "connectionString", "This do not seem as a valid JDBC URL");
+ if(linkConf.link.connectionString == null) {
+ validation.addMessage(Status.UNACCEPTABLE, "link", "connectionString", "JDBC URL can't be empty");
+ } else if(!linkConf.link.connectionString.startsWith("jdbc:")) {
+ validation.addMessage(Status.UNACCEPTABLE, "link", "connectionString", "This do not seem as a valid JDBC URL");
}
// See if we can connect to the database
try {
- DriverManager.getConnection(config.connection.connectionString,
- config.connection.username, config.connection.password);
+ DriverManager.getConnection(linkConf.link.connectionString,
+ linkConf.link.username, linkConf.link.password);
} catch (SQLException e) {
- validation.addMessage(Status.ACCEPTABLE, "connection", "Can't connect to the database with given credentials: " + e.getMessage());
+ validation.addMessage(Status.ACCEPTABLE, "link", "Can't connect to the database with given credentials: " + e.getMessage());
}
// Return final validation object
@@ -81,20 +81,20 @@ public class GenericJdbcValidator extends Validator {
private Validation validateToJobConfiguration(ToJobConfiguration configuration) {
Validation validation = new Validation(FromJobConfiguration.class);
- if(configuration.toTable.tableName == null && configuration.toTable.sql == null) {
- validation.addMessage(Status.UNACCEPTABLE, "toTable", "Either table name or SQL must be specified");
+ if(configuration.toJobConfig.tableName == null && configuration.toJobConfig.sql == null) {
+ validation.addMessage(Status.UNACCEPTABLE, "toJobConfig", "Either table name or SQL must be specified");
}
- if(configuration.toTable.tableName != null && configuration.toTable.sql != null) {
- validation.addMessage(Status.UNACCEPTABLE, "toTable", "Both table name and SQL cannot be specified");
+ if(configuration.toJobConfig.tableName != null && configuration.toJobConfig.sql != null) {
+ validation.addMessage(Status.UNACCEPTABLE, "toJobConfig", "Both table name and SQL cannot be specified");
}
- if(configuration.toTable.tableName == null &&
- configuration.toTable.stageTableName != null) {
- validation.addMessage(Status.UNACCEPTABLE, "toTable",
+ if(configuration.toJobConfig.tableName == null &&
+ configuration.toJobConfig.stageTableName != null) {
+ validation.addMessage(Status.UNACCEPTABLE, "toJobConfig",
"Stage table name cannot be specified without specifying table name");
}
- if(configuration.toTable.stageTableName == null &&
- configuration.toTable.clearStageTable != null) {
- validation.addMessage(Status.UNACCEPTABLE, "toTable",
+ if(configuration.toJobConfig.stageTableName == null &&
+ configuration.toJobConfig.clearStageTable != null) {
+ validation.addMessage(Status.UNACCEPTABLE, "toJobConfig",
"Clear stage table cannot be specified without specifying name of " +
"the stage table.");
}
@@ -105,18 +105,18 @@ public class GenericJdbcValidator extends Validator {
private Validation validateFromJobConfiguration(FromJobConfiguration configuration) {
Validation validation = new Validation(FromJobConfiguration.class);
- if(configuration.fromTable.tableName == null && configuration.fromTable.sql == null) {
- validation.addMessage(Status.UNACCEPTABLE, "fromTable", "Either table name or SQL must be specified");
+ if(configuration.fromJobConfig.tableName == null && configuration.fromJobConfig.sql == null) {
+ validation.addMessage(Status.UNACCEPTABLE, "fromJobConfig", "Either table name or SQL must be specified");
}
- if(configuration.fromTable.tableName != null && configuration.fromTable.sql != null) {
- validation.addMessage(Status.UNACCEPTABLE, "fromTable", "Both table name and SQL cannot be specified");
+ if(configuration.fromJobConfig.tableName != null && configuration.fromJobConfig.sql != null) {
+ validation.addMessage(Status.UNACCEPTABLE, "fromJobConfig", "Both table name and SQL cannot be specified");
}
- if(configuration.fromTable.schemaName != null && configuration.fromTable.sql != null) {
- validation.addMessage(Status.UNACCEPTABLE, "fromTable", "Both schema name and SQL cannot be specified");
+ if(configuration.fromJobConfig.schemaName != null && configuration.fromJobConfig.sql != null) {
+ validation.addMessage(Status.UNACCEPTABLE, "fromJobConfig", "Both schema name and SQL cannot be specified");
}
- if(configuration.fromTable.sql != null && !configuration.fromTable.sql.contains(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN)) {
- validation.addMessage(Status.UNACCEPTABLE, "fromTable", "sql", "SQL statement must contain placeholder for auto generated "
+ if(configuration.fromJobConfig.sql != null && !configuration.fromJobConfig.sql.contains(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN)) {
+ validation.addMessage(Status.UNACCEPTABLE, "fromJobConfig", "sql", "SQL statement must contain placeholder for auto generated "
+ "conditions - " + GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionConfiguration.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionConfiguration.java
deleted file mode 100644
index 6061600..0000000
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionConfiguration.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.jdbc.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-
-/**
- *
- */
-@ConfigurationClass
-public class ConnectionConfiguration {
-
- @Form public ConnectionForm connection;
-
- public ConnectionConfiguration() {
- connection = new ConnectionForm();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java
deleted file mode 100644
index 3b5d70a..0000000
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.jdbc.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-import org.apache.sqoop.model.Validator;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.validators.AbstractValidator;
-import org.apache.sqoop.validation.validators.NotEmpty;
-import org.apache.sqoop.validation.validators.ClassAvailable;
-import org.apache.sqoop.validation.validators.StartsWith;
-
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.Map;
-
-/**
- *
- */
-@FormClass(validators = {@Validator(ConnectionForm.FormValidator.class)})
-public class ConnectionForm {
- @Input(size = 128, validators = {@Validator(NotEmpty.class), @Validator(ClassAvailable.class)} )
- public String jdbcDriver;
-
- @Input(size = 128, validators = {@Validator(value = StartsWith.class, strArg = "jdbc:")} )
- public String connectionString;
-
- @Input(size = 40)
- public String username;
-
- @Input(size = 40, sensitive = true)
- public String password;
-
- @Input
- public Map<String, String> jdbcProperties;
-
- public static class FormValidator extends AbstractValidator<ConnectionForm> {
- @Override
- public void validate(ConnectionForm form) {
- // See if we can connect to the database
- try {
- DriverManager.getConnection(form.connectionString, form.username, form.password);
- } catch (SQLException e) {
- addMessage(Status.ACCEPTABLE, "Can't connect to the database with given credentials: " + e.getMessage());
- }
- }
- }
-}
[17/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java
new file mode 100644
index 0000000..8b23144
--- /dev/null
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.jdbc.configuration;
+
+import org.apache.sqoop.connector.jdbc.GenericJdbcConnectorConstants;
+import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.validators.AbstractValidator;
+import org.apache.sqoop.validation.validators.NullOrContains;
+
+/**
+ *
+ */
+@FormClass( validators = {@Validator(FromJobConfig.FormValidator.class)})
+public class FromJobConfig {
+ @Input(size = 50)
+ public String schemaName;
+
+ @Input(size = 50)
+ public String tableName;
+
+ @Input(size = 2000, validators = {@Validator(value = NullOrContains.class, strArg = GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN)})
+ public String sql;
+
+ @Input(size = 50)
+ public String columns;
+
+ @Input(size = 50)
+ public String partitionColumn;
+
+ @Input
+ public Boolean partitionColumnNull;
+
+ @Input(size = 50)
+ public String boundaryQuery;
+
+ public static class FormValidator extends AbstractValidator<FromJobConfig> {
+ @Override
+ public void validate(FromJobConfig form) {
+ if(form.tableName == null && form.sql == null) {
+ addMessage(Status.UNACCEPTABLE, "Either table name or SQL must be specified");
+ }
+ if(form.tableName != null && form.sql != null) {
+ addMessage(Status.UNACCEPTABLE, "Both table name and SQL cannot be specified");
+ }
+ if(form.schemaName != null && form.sql != null) {
+ addMessage(Status.UNACCEPTABLE, "Both schema name and SQL cannot be specified");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
index 64e9a9a..b036421 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
@@ -25,9 +25,9 @@ import org.apache.sqoop.model.Form;
*/
@ConfigurationClass
public class FromJobConfiguration {
- @Form public FromTableForm fromTable;
+ @Form public FromJobConfig fromJobConfig;
public FromJobConfiguration() {
- fromTable = new FromTableForm();
+ fromJobConfig = new FromJobConfig();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java
deleted file mode 100644
index 72902a2..0000000
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.jdbc.configuration;
-
-import org.apache.sqoop.connector.jdbc.GenericJdbcConnectorConstants;
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-import org.apache.sqoop.model.Validator;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.validators.AbstractValidator;
-import org.apache.sqoop.validation.validators.NullOrContains;
-
-/**
- *
- */
-@FormClass( validators = {@Validator(FromTableForm.FormValidator.class)})
-public class FromTableForm {
- @Input(size = 50)
- public String schemaName;
-
- @Input(size = 50)
- public String tableName;
-
- @Input(size = 2000, validators = {@Validator(value = NullOrContains.class, strArg = GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN)})
- public String sql;
-
- @Input(size = 50)
- public String columns;
-
- @Input(size = 50)
- public String partitionColumn;
-
- @Input
- public Boolean partitionColumnNull;
-
- @Input(size = 50)
- public String boundaryQuery;
-
- public static class FormValidator extends AbstractValidator<FromTableForm> {
- @Override
- public void validate(FromTableForm form) {
- if(form.tableName == null && form.sql == null) {
- addMessage(Status.UNACCEPTABLE, "Either fromTable name or SQL must be specified");
- }
- if(form.tableName != null && form.sql != null) {
- addMessage(Status.UNACCEPTABLE, "Both fromTable name and SQL cannot be specified");
- }
- if(form.schemaName != null && form.sql != null) {
- addMessage(Status.UNACCEPTABLE, "Both schema name and SQL cannot be specified");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java
new file mode 100644
index 0000000..7b76eeb
--- /dev/null
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.jdbc.configuration;
+
+import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.validators.AbstractValidator;
+import org.apache.sqoop.validation.validators.NotEmpty;
+import org.apache.sqoop.validation.validators.ClassAvailable;
+import org.apache.sqoop.validation.validators.StartsWith;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Map;
+
+/**
+ *
+ */
+@FormClass(validators = {@Validator(LinkConfig.FormValidator.class)})
+public class LinkConfig {
+ @Input(size = 128, validators = {@Validator(NotEmpty.class), @Validator(ClassAvailable.class)} )
+ public String jdbcDriver;
+
+ @Input(size = 128, validators = {@Validator(value = StartsWith.class, strArg = "jdbc:")} )
+ public String connectionString;
+
+ @Input(size = 40)
+ public String username;
+
+ @Input(size = 40, sensitive = true)
+ public String password;
+
+ @Input
+ public Map<String, String> jdbcProperties;
+
+ public static class FormValidator extends AbstractValidator<LinkConfig> {
+ @Override
+ public void validate(LinkConfig linkConfig) {
+ // See if we can connect to the database
+ try {
+ DriverManager.getConnection(linkConfig.connectionString, linkConfig.username, linkConfig.password);
+ } catch (SQLException e) {
+ addMessage(Status.ACCEPTABLE, "Can't connect to the database with given credentials: " + e.getMessage());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java
new file mode 100644
index 0000000..7d614f7
--- /dev/null
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.jdbc.configuration;
+
+import org.apache.sqoop.model.ConfigurationClass;
+import org.apache.sqoop.model.Form;
+
+/**
+ *
+ */
+@ConfigurationClass
+public class LinkConfiguration {
+
+ @Form public LinkConfig link;
+
+ public LinkConfiguration() {
+ link = new LinkConfig();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
new file mode 100644
index 0000000..a42a6ec
--- /dev/null
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.jdbc.configuration;
+
+import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.validators.AbstractValidator;
+
+/**
+ *
+ */
+@FormClass(validators = {@Validator(ToJobConfig.FormValidator.class)})
+public class ToJobConfig {
+ @Input(size = 50) public String schemaName;
+ @Input(size = 2000) public String tableName;
+ @Input(size = 50) public String sql;
+ @Input(size = 50) public String columns;
+ @Input(size = 2000) public String stageTableName;
+ @Input public Boolean clearStageTable;
+
+ public static class FormValidator extends AbstractValidator<ToJobConfig> {
+ @Override
+ public void validate(ToJobConfig form) {
+ if(form.tableName == null && form.sql == null) {
+ addMessage(Status.UNACCEPTABLE, "Either table name or SQL must be specified");
+ }
+ if(form.tableName != null && form.sql != null) {
+ addMessage(Status.UNACCEPTABLE, "Both table name and SQL cannot be specified");
+ }
+ if(form.tableName == null && form.stageTableName != null) {
+ addMessage(Status.UNACCEPTABLE, "Stage table name cannot be specified without specifying table name");
+ }
+ if(form.stageTableName == null && form.clearStageTable != null) {
+ addMessage(Status.UNACCEPTABLE, "Clear stage table cannot be specified without specifying name of the stage table.");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
index 9215e6c..ad68681 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
@@ -25,9 +25,9 @@ import org.apache.sqoop.model.Form;
*/
@ConfigurationClass
public class ToJobConfiguration {
- @Form public ToTableForm toTable;
+ @Form public ToJobConfig toJobConfig;
public ToJobConfiguration() {
- toTable = new ToTableForm();
+ toJobConfig = new ToJobConfig();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java
deleted file mode 100644
index 642da11..0000000
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.jdbc.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-import org.apache.sqoop.model.Validator;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.validators.AbstractValidator;
-
-/**
- *
- */
-@FormClass(validators = {@Validator(ToTableForm.FormValidator.class)})
-public class ToTableForm {
- @Input(size = 50) public String schemaName;
- @Input(size = 2000) public String tableName;
- @Input(size = 50) public String sql;
- @Input(size = 50) public String columns;
- @Input(size = 2000) public String stageTableName;
- @Input public Boolean clearStageTable;
-
- public static class FormValidator extends AbstractValidator<ToTableForm> {
- @Override
- public void validate(ToTableForm form) {
- if(form.tableName == null && form.sql == null) {
- addMessage(Status.UNACCEPTABLE, "Either fromTable name or SQL must be specified");
- }
- if(form.tableName != null && form.sql != null) {
- addMessage(Status.UNACCEPTABLE, "Both fromTable name and SQL cannot be specified");
- }
- if(form.tableName == null && form.stageTableName != null) {
- addMessage(Status.UNACCEPTABLE, "Stage fromTable name cannot be specified without specifying fromTable name");
- }
- if(form.stageTableName == null && form.clearStageTable != null) {
- addMessage(Status.UNACCEPTABLE, "Clear stage fromTable cannot be specified without specifying name of the stage fromTable.");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties b/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties
new file mode 100644
index 0000000..921ac28
--- /dev/null
+++ b/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generic JDBC Connector Resources
+
+############################
+# Link Config
+#
+link.label = Link configuration
+link.help = You must supply the information requested in order to \
+ create a connection object.
+
+# jdbc driver
+link.jdbcDriver.label = JDBC Driver Class
+link.jdbcDriver.help = Enter the fully qualified class name of the JDBC \
+ driver that will be used for establishing this connection.
+
+# connect string
+link.connectionString.label = JDBC Connection String
+link.connectionString.help = Enter the value of JDBC connection string to be \
+ used by this connector for creating connections.
+
+# username string
+link.username.label = Username
+link.username.help = Enter the username to be used for connecting to the \
+ database.
+
+# password string
+link.password.label = Password
+link.password.help = Enter the password to be used for connecting to the \
+ database.
+
+# jdbc properties
+link.jdbcProperties.label = JDBC Connection Properties
+link.jdbcProperties.help = Enter any JDBC properties that should be \
+ supplied during the creation of connection.
+
+# From Job Config
+#
+fromJobConfig.label = From database configuration
+fromJobConfig.help = You must supply the information requested in order to create \
+ a job object.
+
+# From schema name
+fromJobConfig.schemaName.label = Schema name
+fromJobConfig.schemaName.help = Schema name to process data in the remote database
+
+# From table name
+fromJobConfig.tableName.label = Table name
+fromJobConfig.tableName.help = Table name to process data in the remote database
+
+# From table SQL
+fromJobConfig.sql.label = Table SQL statement
+fromJobConfig.sql.help = SQL statement to process data in the remote database
+
+# From table columns
+fromJobConfig.columns.label = Table column names
+fromJobConfig.columns.help = Specific columns of a table name or a table SQL
+
+# From table warehouse
+fromJobConfig.warehouse.label = Data warehouse
+fromJobConfig.warehouse.help = The root directory for data
+
+# From table datadir
+fromJobConfig.dataDirectory.label = Data directory
+fromJobConfig.dataDirectory.help = The sub-directory under warehouse for data
+
+# From table pcol
+fromJobConfig.partitionColumn.label = Partition column name
+fromJobConfig.partitionColumn.help = A specific column for data partition
+
+# From table pcol is null
+fromJobConfig.partitionColumnNull.label = Nulls in partition column
+fromJobConfig.partitionColumnNull.help = Whether there are null values in partition column
+
+# From table boundary
+fromJobConfig.boundaryQuery.label = Boundary query
+fromJobConfig.boundaryQuery.help = The boundary query for data partition
+
+# ToJob Config
+#
+toJobConfig.label = To database configuration
+toJobConfig.help = You must supply the information requested in order to create \
+ a job object.
+
+# To schema name
+toJobConfig.schemaName.label = Schema name
+toJobConfig.schemaName.help = Schema name to process data in the remote database
+
+# To table name
+toJobConfig.tableName.label = Table name
+toJobConfig.tableName.help = Table name to process data in the remote database
+
+# To table SQL
+toJobConfig.sql.label = Table SQL statement
+toJobConfig.sql.help = SQL statement to process data in the remote database
+
+# To table columns
+toJobConfig.columns.label = Table column names
+toJobConfig.columns.help = Specific columns of a table name or a table SQL
+
+# To stage table name
+toJobConfig.stageTableName.label = Stage table name
+toJobConfig.stageTableName.help = Name of the stage table to use
+
+# To clear stage table
+toJobConfig.clearStageTable.label = Clear stage table
+toJobConfig.clearStageTable.help = Indicate if the stage table should be cleared
+
+# Placeholders to have some entities created
+ignored.label = Ignored
+ignored.help = This is completely ignored
+
+ignored.ignored.label = Ignored
+ignored.ignored.help = This is completely ignored
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-resources.properties
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-resources.properties b/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-resources.properties
deleted file mode 100644
index a1302c0..0000000
--- a/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-resources.properties
+++ /dev/null
@@ -1,127 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Generic JDBC Connector Resources
-
-############################
-# Connection Form
-#
-connection.label = Connection configuration
-connection.help = You must supply the information requested in order to \
- create a connection object.
-
-# jdbc driver
-connection.jdbcDriver.label = JDBC Driver Class
-connection.jdbcDriver.help = Enter the fully qualified class name of the JDBC \
- driver that will be used for establishing this connection.
-
-# connect string
-connection.connectionString.label = JDBC Connection String
-connection.connectionString.help = Enter the value of JDBC connection string to be \
- used by this connector for creating connections.
-
-# username string
-connection.username.label = Username
-connection.username.help = Enter the username to be used for connecting to the \
- database.
-
-# password string
-connection.password.label = Password
-connection.password.help = Enter the password to be used for connecting to the \
- database.
-
-# jdbc properties
-connection.jdbcProperties.label = JDBC Connection Properties
-connection.jdbcProperties.help = Enter any JDBC properties that should be \
- supplied during the creation of connection.
-
-# From Table Form
-#
-fromTable.label = From database configuration
-fromTable.help = You must supply the information requested in order to create \
- a job object.
-
-# From schema name
-fromTable.schemaName.label = Schema name
-fromTable.schemaName.help = Schema name to process data in the remote database
-
-# From table name
-fromTable.tableName.label = Table name
-fromTable.tableName.help = Table name to process data in the remote database
-
-# From table SQL
-fromTable.sql.label = Table SQL statement
-fromTable.sql.help = SQL statement to process data in the remote database
-
-# From table columns
-fromTable.columns.label = Table column names
-fromTable.columns.help = Specific columns of a table name or a table SQL
-
-# From table warehouse
-fromTable.warehouse.label = Data warehouse
-fromTable.warehouse.help = The root directory for data
-
-# From table datadir
-fromTable.dataDirectory.label = Data directory
-fromTable.dataDirectory.help = The sub-directory under warehouse for data
-
-# From table pcol
-fromTable.partitionColumn.label = Partition column name
-fromTable.partitionColumn.help = A specific column for data partition
-
-# From table pcol is null
-fromTable.partitionColumnNull.label = Nulls in partition column
-fromTable.partitionColumnNull.help = Whether there are null values in partition column
-
-# From table boundary
-fromTable.boundaryQuery.label = Boundary query
-fromTable.boundaryQuery.help = The boundary query for data partition
-
-# To table form
-#
-toTable.label = To database configuration
-toTable.help = You must supply the information requested in order to create \
- a job object.
-
-# From schema name
-toTable.schemaName.label = Schema name
-toTable.schemaName.help = Schema name to process data in the remote database
-
-# From table name
-toTable.tableName.label = Table name
-toTable.tableName.help = Table name to process data in the remote database
-
-# From table SQL
-toTable.sql.label = Table SQL statement
-toTable.sql.help = SQL statement to process data in the remote database
-
-# From table columns
-toTable.columns.label = Table column names
-toTable.columns.help = Specific columns of a table name or a table SQL
-
-# To stage table name
-toTable.stageTableName.label = Stage table name
-toTable.stageTableName.help = Name of the stage table to use
-
-# To clear stage table
-toTable.clearStageTable.label = Clear stage table
-toTable.clearStageTable.help = Indicate if the stage table should be cleared
-
-# Placeholders to have some entities created
-ignored.label = Ignored
-ignored.help = This is completely ignored
-
-ignored.ignored.label = Ignored
-ignored.ignored.help = This is completely ignored
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
index 2b1dec2..776359a 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
@@ -21,7 +21,7 @@ import junit.framework.TestCase;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.job.etl.Extractor;
import org.apache.sqoop.job.etl.ExtractorContext;
@@ -67,10 +67,10 @@ public class TestExtractor extends TestCase {
public void testQuery() throws Exception {
MutableContext context = new MutableMapContext();
- ConnectionConfiguration connectionConfig = new ConnectionConfiguration();
+ LinkConfiguration connectionConfig = new LinkConfiguration();
- connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL;
+ connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
@@ -99,10 +99,10 @@ public class TestExtractor extends TestCase {
public void testSubquery() throws Exception {
MutableContext context = new MutableMapContext();
- ConnectionConfiguration connectionConfig = new ConnectionConfiguration();
+ LinkConfiguration connectionConfig = new LinkConfiguration();
- connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL;
+ connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
index 2d07130..bf66bb1 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
@@ -23,7 +23,7 @@ import junit.framework.TestCase;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.job.Constants;
import org.apache.sqoop.job.etl.Initializer;
@@ -112,12 +112,12 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableName() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.tableName = schemalessTableName;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.tableName = schemalessTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -138,13 +138,13 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumns() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.tableName = schemalessTableName;
- jobConf.fromTable.columns = tableColumns;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.tableName = schemalessTableName;
+ jobConf.fromJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -165,13 +165,13 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableSql() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.sql = schemalessTableSql;
- jobConf.fromTable.partitionColumn = "DCOL";
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.sql = schemalessTableSql;
+ jobConf.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -192,14 +192,14 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableSqlWithTableColumns() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.sql = schemalessTableSql;
- jobConf.fromTable.columns = tableColumns;
- jobConf.fromTable.partitionColumn = "DCOL";
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.sql = schemalessTableSql;
+ jobConf.fromJobConfig.columns = tableColumns;
+ jobConf.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -221,15 +221,15 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableNameWithSchema() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.schemaName = schemaName;
- jobConf.fromTable.tableName = tableName;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.schemaName = schemaName;
+ jobConf.fromJobConfig.tableName = tableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -250,16 +250,16 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumnsWithSchema() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.schemaName = schemaName;
- jobConf.fromTable.tableName = tableName;
- jobConf.fromTable.columns = tableColumns;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.schemaName = schemaName;
+ jobConf.fromJobConfig.tableName = tableName;
+ jobConf.fromJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -280,16 +280,16 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableSqlWithSchema() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.schemaName = schemaName;
- jobConf.fromTable.sql = tableSql;
- jobConf.fromTable.partitionColumn = "DCOL";
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.schemaName = schemaName;
+ jobConf.fromJobConfig.sql = tableSql;
+ jobConf.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -311,14 +311,14 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testGetSchemaForTable() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.schemaName = schemaName;
- jobConf.fromTable.tableName = tableName;
- jobConf.fromTable.partitionColumn = "DCOL";
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.schemaName = schemaName;
+ jobConf.fromJobConfig.tableName = tableName;
+ jobConf.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -327,19 +327,19 @@ public class TestFromInitializer extends TestCase {
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
Schema schema = initializer.getSchema(initializerContext, connConf, jobConf);
- assertEquals(getSchema(jobConf.fromTable.schemaName + "." + tableName), schema);
+ assertEquals(getSchema(jobConf.fromJobConfig.schemaName + "." + tableName), schema);
}
@SuppressWarnings("unchecked")
public void testGetSchemaForSql() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.schemaName = schemaName;
- jobConf.fromTable.sql = tableSql;
- jobConf.fromTable.partitionColumn = "DCOL";
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.schemaName = schemaName;
+ jobConf.fromJobConfig.sql = tableSql;
+ jobConf.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -353,17 +353,17 @@ public class TestFromInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableSqlWithTableColumnsWithSchema() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromTable.schemaName = schemaName;
- jobConf.fromTable.sql = tableSql;
- jobConf.fromTable.columns = tableColumns;
- jobConf.fromTable.partitionColumn = "DCOL";
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.fromJobConfig.schemaName = schemaName;
+ jobConf.fromJobConfig.sql = tableSql;
+ jobConf.fromJobConfig.columns = tableColumns;
+ jobConf.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
index d7e8c6c..c349676 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
@@ -26,7 +26,7 @@ import java.util.Collection;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.etl.io.DataReader;
import org.apache.sqoop.job.etl.Loader;
@@ -82,10 +82,10 @@ public class TestLoader {
public void testInsert() throws Exception {
MutableContext context = new MutableMapContext();
- ConnectionConfiguration connectionConfig = new ConnectionConfiguration();
+ LinkConfiguration connectionConfig = new LinkConfiguration();
- connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL;
+ connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
ToJobConfiguration jobConfig = new ToJobConfiguration();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
index f1023c8..949877b 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
@@ -29,7 +29,7 @@ import junit.framework.TestCase;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.job.etl.Partition;
import org.apache.sqoop.job.etl.Partitioner;
@@ -55,7 +55,7 @@ public class TestPartitioner extends TestCase {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -86,7 +86,7 @@ public class TestPartitioner extends TestCase {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -115,7 +115,7 @@ public class TestPartitioner extends TestCase {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -151,7 +151,7 @@ public class TestPartitioner extends TestCase {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf((double)(START + NUMBER_OF_ROWS - 1)));
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -182,7 +182,7 @@ public class TestPartitioner extends TestCase {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf((double)(START + NUMBER_OF_ROWS - 1)));
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -203,7 +203,7 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(START));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(START + NUMBER_OF_ROWS - 1));
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -226,7 +226,7 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(new BigDecimal(START)));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(new BigDecimal(START + NUMBER_OF_ROWS - 1)));
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -247,7 +247,7 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(new BigDecimal(START)));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(new BigDecimal(START)));
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -272,7 +272,7 @@ public class TestPartitioner extends TestCase {
.toString());
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -300,7 +300,7 @@ public class TestPartitioner extends TestCase {
Time.valueOf("10:40:50").toString());
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -325,7 +325,7 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
Timestamp.valueOf("2013-12-31 10:40:50.654").toString());
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -349,7 +349,7 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "1");
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -372,7 +372,7 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "Z");
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -419,7 +419,7 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "Warty Warthog");
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
@@ -442,7 +442,7 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "AAF");
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
@@ -471,9 +471,9 @@ public class TestPartitioner extends TestCase {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "AAE");
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
- jobConf.fromTable.partitionColumnNull = true;
+ jobConf.fromJobConfig.partitionColumnNull = true;
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
index 4767215..00c2aab 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
@@ -21,7 +21,7 @@ import junit.framework.TestCase;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.job.etl.Initializer;
import org.apache.sqoop.job.etl.InitializerContext;
@@ -75,14 +75,14 @@ public class TestToInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableName() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemalessTableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.tableName = schemalessTableName;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.tableName = schemalessTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -96,15 +96,15 @@ public class TestToInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumns() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemalessTableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.tableName = schemalessTableName;
- jobConf.toTable.columns = tableColumns;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.tableName = schemalessTableName;
+ jobConf.toJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -118,12 +118,12 @@ public class TestToInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableSql() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.sql = schemalessTableSql;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.sql = schemalessTableSql;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -137,15 +137,15 @@ public class TestToInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableNameWithSchema() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.schemaName = schemaName;
- jobConf.toTable.tableName = tableName;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.schemaName = schemaName;
+ jobConf.toJobConfig.tableName = tableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -159,16 +159,16 @@ public class TestToInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumnsWithSchema() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.schemaName = schemaName;
- jobConf.toTable.tableName = tableName;
- jobConf.toTable.columns = tableColumns;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.schemaName = schemaName;
+ jobConf.toJobConfig.tableName = tableName;
+ jobConf.toJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -182,13 +182,13 @@ public class TestToInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testTableSqlWithSchema() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.schemaName = schemaName;
- jobConf.toTable.sql = tableSql;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.schemaName = schemaName;
+ jobConf.toJobConfig.sql = tableSql;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -216,13 +216,13 @@ public class TestToInitializer extends TestCase {
}
public void testNonExistingStageTable() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.tableName = schemalessTableName;
- jobConf.toTable.stageTableName = stageTableName;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.tableName = schemalessTableName;
+ jobConf.toJobConfig.stageTableName = stageTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -239,15 +239,15 @@ public class TestToInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testNonEmptyStageTable() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.tableName = schemalessTableName;
- jobConf.toTable.stageTableName = stageTableName;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.tableName = schemalessTableName;
+ jobConf.toJobConfig.stageTableName = stageTableName;
createTable(fullStageTableName);
executor.executeUpdate("INSERT INTO " + fullStageTableName +
" VALUES(1, 1.1, 'one')");
@@ -264,17 +264,16 @@ public class TestToInitializer extends TestCase {
}
}
- @SuppressWarnings("unchecked")
public void testClearStageTableValidation() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
//specifying clear stage table flag without specifying name of
// the stage table
- jobConf.toTable.tableName = schemalessTableName;
- jobConf.toTable.clearStageTable = false;
+ jobConf.toJobConfig.tableName = schemalessTableName;
+ jobConf.toJobConfig.clearStageTable = false;
ValidationRunner validationRunner = new ValidationRunner();
ValidationResult result = validationRunner.validate(jobConf);
assertEquals("User should not specify clear stage table flag without " +
@@ -282,49 +281,48 @@ public class TestToInitializer extends TestCase {
Status.UNACCEPTABLE,
result.getStatus());
assertTrue(result.getMessages().containsKey(
- "toTable"));
+ "toJobConfig"));
- jobConf.toTable.clearStageTable = true;
+ jobConf.toJobConfig.clearStageTable = true;
result = validationRunner.validate(jobConf);
assertEquals("User should not specify clear stage table flag without " +
"specifying name of the stage table",
Status.UNACCEPTABLE,
result.getStatus());
assertTrue(result.getMessages().containsKey(
- "toTable"));
+ "toJobConfig"));
}
- @SuppressWarnings("unchecked")
public void testStageTableWithoutTable() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
//specifying stage table without specifying table name
- jobConf.toTable.stageTableName = stageTableName;
- jobConf.toTable.sql = "";
+ jobConf.toJobConfig.stageTableName = stageTableName;
+ jobConf.toJobConfig.sql = "";
ValidationRunner validationRunner = new ValidationRunner();
ValidationResult result = validationRunner.validate(jobConf);
assertEquals("Stage table name cannot be specified without specifying " +
"table name", Status.UNACCEPTABLE, result.getStatus());
assertTrue(result.getMessages().containsKey(
- "toTable"));
+ "toJobConfig"));
}
@SuppressWarnings("unchecked")
public void testClearStageTable() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.tableName = schemalessTableName;
- jobConf.toTable.stageTableName = stageTableName;
- jobConf.toTable.clearStageTable = true;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.tableName = schemalessTableName;
+ jobConf.toJobConfig.stageTableName = stageTableName;
+ jobConf.toJobConfig.clearStageTable = true;
createTable(fullStageTableName);
executor.executeUpdate("INSERT INTO " + fullStageTableName +
" VALUES(1, 1.1, 'one')");
@@ -340,15 +338,15 @@ public class TestToInitializer extends TestCase {
@SuppressWarnings("unchecked")
public void testStageTable() throws Exception {
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
- connConf.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.connection.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toTable.tableName = schemalessTableName;
- jobConf.toTable.stageTableName = stageTableName;
+ connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ jobConf.toJobConfig.tableName = schemalessTableName;
+ jobConf.toJobConfig.stageTableName = stageTableName;
createTable(fullStageTableName);
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java
new file mode 100644
index 0000000..47b186c
--- /dev/null
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.connector.hdfs;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MJobForms;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class HdfsConfigUpgrader extends RepositoryUpgrader {
+ private static final Logger LOG =
+ Logger.getLogger(HdfsConfigUpgrader.class);
+
+ /*
+ * For now, there is no real upgrade. So copy all data over,
+ * set the validation messages and error messages to be the same as for the
+ * inputs in the original one.
+ */
+
+ @Override
+ public void upgrade(MConnectionForms original,
+ MConnectionForms upgradeTarget) {
+ doUpgrade(original.getForms(), upgradeTarget.getForms());
+ }
+
+ @Override
+ public void upgrade(MJobForms original, MJobForms upgradeTarget) {
+ doUpgrade(original.getForms(), upgradeTarget.getForms());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void doUpgrade(List<MForm> original, List<MForm> target) {
+ // Easier to find the form in the original forms list if we use a map.
+ // Since the constructor of MJobForms takes a list,
+ // index is not guaranteed to be the same, so we need to look for
+ // equivalence
+ Map<String, MForm> formMap = new HashMap<String, MForm>();
+ for (MForm form : original) {
+ formMap.put(form.getName(), form);
+ }
+ for (MForm form : target) {
+ List<MInput<?>> inputs = form.getInputs();
+ MForm originalForm = formMap.get(form.getName());
+ if (originalForm == null) {
+ LOG.warn("Form: '" + form.getName() + "' not present in old " +
+ "connector. So it and its inputs will not be transferred by the upgrader.");
+ continue;
+ }
+ for (MInput input : inputs) {
+ try {
+ MInput originalInput = originalForm.getInput(input.getName());
+ input.setValue(originalInput.getValue());
+ } catch (SqoopException ex) {
+ LOG.warn("Input: '" + input.getName() + "' not present in old " +
+ "connector. So it will not be transferred by the upgrader.");
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
index 883636c..70833a0 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
@@ -20,10 +20,10 @@ package org.apache.sqoop.connector.hdfs;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.VersionInfo;
-import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
import org.apache.sqoop.connector.hdfs.configuration.ToJobConfiguration;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.connector.spi.SqoopConnector;
import org.apache.sqoop.job.etl.From;
import org.apache.sqoop.job.etl.To;
@@ -71,8 +71,8 @@ public class HdfsConnector extends SqoopConnector {
* @return Get connection configuration class
*/
@Override
- public Class getConnectionConfigurationClass() {
- return ConnectionConfiguration.class;
+ public Class getLinkConfigurationClass() {
+ return LinkConfiguration.class;
}
/**
@@ -108,7 +108,7 @@ public class HdfsConnector extends SqoopConnector {
}
/**
- * Returns validation object that Sqoop framework can use to validate user
+ * Returns validation object that Sqoop can use to validate user
* supplied forms before accepting them. This object will be used both for
* connection and job forms.
*
@@ -120,13 +120,13 @@ public class HdfsConnector extends SqoopConnector {
}
/**
- * Returns an {@linkplain org.apache.sqoop.connector.spi.MetadataUpgrader} object that can upgrade the
+ * Returns an {@linkplain org.apache.sqoop.connector.spi.RepositoryUpgrader} object that can upgrade the
* connection and job metadata.
*
* @return MetadataUpgrader object
*/
@Override
- public MetadataUpgrader getMetadataUpgrader() {
- return new HdfsMetadataUpgrader();
+ public RepositoryUpgrader getRepositoryUpgrader() {
+ return new HdfsConfigUpgrader();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConstants.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConstants.java
index a27aff1..6e369c6 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConstants.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConstants.java
@@ -24,7 +24,7 @@ public final class HdfsConstants extends Constants {
// Resource bundle name
public static final String RESOURCE_BUNDLE_NAME =
- "hdfs-connector-resources";
+ "hdfs-connector-config";
public static final char DEFAULT_RECORD_DELIMITER = '\n';
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
index 7447071..436d243 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.LineReader;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
import org.apache.sqoop.etl.io.DataWriter;
import org.apache.sqoop.job.etl.Extractor;
@@ -42,7 +42,7 @@ import java.io.IOException;
* Extract from HDFS.
* Default field delimiter of a record is comma.
*/
-public class HdfsExtractor extends Extractor<ConnectionConfiguration, FromJobConfiguration, HdfsPartition> {
+public class HdfsExtractor extends Extractor<LinkConfiguration, FromJobConfiguration, HdfsPartition> {
public static final Logger LOG = Logger.getLogger(HdfsExtractor.class);
@@ -52,7 +52,7 @@ public class HdfsExtractor extends Extractor<ConnectionConfiguration, FromJobCon
@Override
public void extract(ExtractorContext context,
- ConnectionConfiguration connectionConfiguration,
+ LinkConfiguration connectionConfiguration,
FromJobConfiguration jobConfiguration, HdfsPartition partition) {
conf = ((PrefixContext) context.getContext()).getConfiguration();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
index d2d12a8..923f904 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
@@ -29,17 +29,16 @@ public class HdfsInitializer extends Initializer {
* promoted to all other part of the workflow automatically.
*
* @param context Initializer context object
- * @param connection Connector's connection configuration object
- * @param job Connector's job configuration object
+ * @param linkConf Connector's link configuration object
+ * @param jobConf Connector's job configuration object
*/
@Override
- public void initialize(InitializerContext context, Object connection, Object job) {
+ public void initialize(InitializerContext context, Object linkConf, Object jobConf) {
}
-
@Override
- public Schema getSchema(InitializerContext context, Object connection, Object job) {
+ public Schema getSchema(InitializerContext context, Object linkConf, Object jobConf) {
return null;
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
index 61676fe..4c546ba 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.sqoop.common.PrefixContext;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
-import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
import org.apache.sqoop.connector.hdfs.configuration.ToJobConfiguration;
import org.apache.sqoop.connector.hdfs.hdfsWriter.GenericHdfsWriter;
import org.apache.sqoop.connector.hdfs.hdfsWriter.HdfsSequenceWriter;
@@ -37,24 +37,24 @@ import org.apache.sqoop.utils.ClassUtils;
import java.io.IOException;
import java.util.UUID;
-public class HdfsLoader extends Loader<ConnectionConfiguration, ToJobConfiguration> {
+public class HdfsLoader extends Loader<LinkConfiguration, ToJobConfiguration> {
/**
* Load data to target.
*
* @param context Loader context object
- * @param connection Connection configuration
- * @param job Job configuration
+ * @param linkConf Link configuration
+ * @param toJobConf Job configuration
* @throws Exception
*/
@Override
- public void load(LoaderContext context, ConnectionConfiguration connection, ToJobConfiguration job) throws Exception {
+ public void load(LoaderContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) throws Exception {
DataReader reader = context.getDataReader();
Configuration conf = ((PrefixContext)context.getContext()).getConfiguration();
- String directoryName = job.output.outputDirectory;
- String codecname = getCompressionCodecName(job);
+ String directoryName = toJobConf.toJobConfig.outputDirectory;
+ String codecname = getCompressionCodecName(toJobConf);
CompressionCodec codec = null;
if (codecname != null) {
@@ -73,12 +73,12 @@ public class HdfsLoader extends Loader<ConnectionConfiguration, ToJobConfigurati
}
}
- String filename = directoryName + "/" + UUID.randomUUID() + getExtension(job,codec);
+ String filename = directoryName + "/" + UUID.randomUUID() + getExtension(toJobConf,codec);
try {
Path filepath = new Path(filename);
- GenericHdfsWriter filewriter = getWriter(job);
+ GenericHdfsWriter filewriter = getWriter(toJobConf);
filewriter.initialize(filepath,conf,codec);
@@ -95,18 +95,15 @@ public class HdfsLoader extends Loader<ConnectionConfiguration, ToJobConfigurati
}
- private GenericHdfsWriter getWriter(ToJobConfiguration job) {
- if (job.output.outputFormat == OutputFormat.SEQUENCE_FILE)
- return new HdfsSequenceWriter();
- else
- return new HdfsTextWriter();
+ private GenericHdfsWriter getWriter(ToJobConfiguration toJobConf) {
+ return (toJobConf.toJobConfig.outputFormat == ToFormat.SEQUENCE_FILE) ? new HdfsSequenceWriter()
+ : new HdfsTextWriter();
}
-
- private String getCompressionCodecName(ToJobConfiguration jobConf) {
- if(jobConf.output.compression == null)
+ private String getCompressionCodecName(ToJobConfiguration toJobConf) {
+ if(toJobConf.toJobConfig.compression == null)
return null;
- switch(jobConf.output.compression) {
+ switch(toJobConf.toJobConfig.compression) {
case NONE:
return null;
case DEFAULT:
@@ -124,14 +121,14 @@ public class HdfsLoader extends Loader<ConnectionConfiguration, ToJobConfigurati
case SNAPPY:
return "org.apache.hadoop.io.compress.SnappyCodec";
case CUSTOM:
- return jobConf.output.customCompression.trim();
+ return toJobConf.toJobConfig.customCompression.trim();
}
return null;
}
//TODO: We should probably support configurable extensions at some point
- private static String getExtension(ToJobConfiguration job, CompressionCodec codec) {
- if (job.output.outputFormat == OutputFormat.SEQUENCE_FILE)
+ private static String getExtension(ToJobConfiguration toJobConf, CompressionCodec codec) {
+ if (toJobConf.toJobConfig.outputFormat == ToFormat.SEQUENCE_FILE)
return ".seq";
if (codec == null)
return ".txt";
[16/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsMetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsMetadataUpgrader.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsMetadataUpgrader.java
deleted file mode 100644
index 3e51e38..0000000
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsMetadataUpgrader.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sqoop.connector.hdfs;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJobForms;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class HdfsMetadataUpgrader extends MetadataUpgrader {
- private static final Logger LOG =
- Logger.getLogger(HdfsMetadataUpgrader.class);
-
- /*
- * For now, there is no real upgrade. So copy all data over,
- * set the validation messages and error messages to be the same as for the
- * inputs in the original one.
- */
-
- @Override
- public void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
- }
-
- @Override
- public void upgrade(MJobForms original, MJobForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
- }
-
- @SuppressWarnings("unchecked")
- private void doUpgrade(List<MForm> original, List<MForm> target) {
- // Easier to find the form in the original forms list if we use a map.
- // Since the constructor of MJobForms takes a list,
- // index is not guaranteed to be the same, so we need to look for
- // equivalence
- Map<String, MForm> formMap = new HashMap<String, MForm>();
- for (MForm form : original) {
- formMap.put(form.getName(), form);
- }
- for (MForm form : target) {
- List<MInput<?>> inputs = form.getInputs();
- MForm originalForm = formMap.get(form.getName());
- if (originalForm == null) {
- LOG.warn("Form: '" + form.getName() + "' not present in old " +
- "connector. So it and its inputs will not be transferred by the upgrader.");
- continue;
- }
- for (MInput input : inputs) {
- try {
- MInput originalInput = originalForm.getInput(input.getName());
- input.setValue(originalInput.getValue());
- } catch (SqoopException ex) {
- LOG.warn("Input: '" + input.getName() + "' not present in old " +
- "connector. So it will not be transferred by the upgrader.");
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
index df764d2..6828de8 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
import org.apache.sqoop.job.etl.Partition;
import org.apache.sqoop.job.etl.Partitioner;
@@ -50,7 +50,7 @@ import org.apache.sqoop.common.PrefixContext;
* This class derives mostly from CombineFileInputFormat of Hadoop, i.e.
* org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.
*/
-public class HdfsPartitioner extends Partitioner<ConnectionConfiguration, FromJobConfiguration> {
+public class HdfsPartitioner extends Partitioner<LinkConfiguration, FromJobConfiguration> {
public static final String SPLIT_MINSIZE_PERNODE =
"mapreduce.input.fileinputformat.split.minsize.per.node";
@@ -68,12 +68,12 @@ public class HdfsPartitioner extends Partitioner<ConnectionConfiguration, FromJo
@Override
public List<Partition> getPartitions(PartitionerContext context,
- ConnectionConfiguration connectionConfiguration, FromJobConfiguration jobConfiguration) {
+ LinkConfiguration linkConfiguration, FromJobConfiguration jobConfiguration) {
Configuration conf = ((PrefixContext)context.getContext()).getConfiguration();
try {
- long numInputBytes = getInputSize(conf, jobConfiguration.input.inputDirectory);
+ long numInputBytes = getInputSize(conf, jobConfiguration.fromJobConfig.inputDirectory);
maxSplitSize = numInputBytes / context.getMaxPartitions();
if(numInputBytes % context.getMaxPartitions() != 0 ) {
@@ -118,7 +118,7 @@ public class HdfsPartitioner extends Partitioner<ConnectionConfiguration, FromJo
}
// all the files in input set
- String indir = jobConfiguration.input.inputDirectory;
+ String indir = jobConfiguration.fromJobConfig.inputDirectory;
FileSystem fs = FileSystem.get(conf);
List<Path> paths = new LinkedList<Path>();
@@ -147,7 +147,7 @@ public class HdfsPartitioner extends Partitioner<ConnectionConfiguration, FromJo
}
}
- //TODO: Perhaps get the FS from connection configuration so we can support remote HDFS
+ //TODO: Perhaps get the FS from link configuration so we can support remote HDFS
private long getInputSize(Configuration conf, String indir) throws IOException {
FileSystem fs = FileSystem.get(conf);
FileStatus[] files = fs.listStatus(new Path(indir));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
index 4efbd33..dfa3659 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
@@ -23,13 +23,13 @@ import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.Validator;
/**
- * Validate framework configuration objects
+ * Validate configuration objects
*/
public class HdfsValidator extends Validator {
@Override
- public Validation validateConnection(Object connectionConfiguration) {
- Validation validation = new Validation(ConnectionConfiguration.class);
+ public Validation validateLink(Object connectionConfiguration) {
+ Validation validation = new Validation(LinkConfiguration.class);
// No validation on connection object
return validation;
}
@@ -41,42 +41,39 @@ public class HdfsValidator extends Validator {
return super.validateJob(jobConfiguration);
}
- private Validation validateExportJob(Object jobConfiguration) {
+ @SuppressWarnings("unused")
+ private Validation validateFromJob(Object jobConfiguration) {
Validation validation = new Validation(FromJobConfiguration.class);
FromJobConfiguration configuration = (FromJobConfiguration)jobConfiguration;
-
- validateInputForm(validation, configuration.input);
-
-
+ validateInputForm(validation, configuration.fromJobConfig);
return validation;
}
- private Validation validateImportJob(Object jobConfiguration) {
+ @SuppressWarnings("unused")
+ private Validation validateToJob(Object jobConfiguration) {
Validation validation = new Validation(ToJobConfiguration.class);
ToJobConfiguration configuration = (ToJobConfiguration)jobConfiguration;
-
- validateOutputForm(validation, configuration.output);
-
+ validateOutputForm(validation, configuration.toJobConfig);
return validation;
}
- private void validateInputForm(Validation validation, InputForm input) {
+ private void validateInputForm(Validation validation, FromJobConfig input) {
if(input.inputDirectory == null || input.inputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "input", "inputDirectory", "Input directory is empty");
}
}
- private void validateOutputForm(Validation validation, OutputForm output) {
+ private void validateOutputForm(Validation validation, ToJobConfig output) {
if(output.outputDirectory == null || output.outputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "output", "outputDirectory", "Output directory is empty");
}
if(output.customCompression != null &&
output.customCompression.trim().length() > 0 &&
- output.compression != OutputCompression.CUSTOM) {
+ output.compression != ToCompression.CUSTOM) {
validation.addMessage(Status.UNACCEPTABLE, "output", "compression",
"custom compression should be blank as " + output.compression + " is being used.");
}
- if(output.compression == OutputCompression.CUSTOM &&
+ if(output.compression == ToCompression.CUSTOM &&
(output.customCompression == null ||
output.customCompression.trim().length() == 0)
) {
@@ -84,6 +81,4 @@ public class HdfsValidator extends Validator {
"custom compression is blank.");
}
}
-
-
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ConnectionConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ConnectionConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ConnectionConfiguration.java
deleted file mode 100644
index 6dd79d5..0000000
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ConnectionConfiguration.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.hdfs.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-
-@ConfigurationClass
-public class ConnectionConfiguration {
- @Form
- public ConnectionForm connection;
-
- public ConnectionConfiguration() {
- connection = new ConnectionForm();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ConnectionForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ConnectionForm.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ConnectionForm.java
deleted file mode 100644
index 7dad2a2..0000000
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ConnectionForm.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.hdfs.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-
-@FormClass
-public class ConnectionForm {
- //Todo: Didn't find anything that belongs here...
- // Since empty forms don't work (DERBYREPO_0008:The form contains no input metadata), I'm putting a dummy form here
-
- @Input(size = 255) public String dummy;
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java
new file mode 100644
index 0000000..2c98051
--- /dev/null
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs.configuration;
+
+import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Input;
+
+/**
+ *
+ */
+@FormClass
+public class FromJobConfig {
+
+ @Input(size = 255) public String inputDirectory;
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
index bccb99d..f861237 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
@@ -22,11 +22,10 @@ import org.apache.sqoop.model.Form;
@ConfigurationClass
public class FromJobConfiguration {
- @Form public InputForm input;
-
+ @Form public FromJobConfig fromJobConfig;
public FromJobConfiguration() {
- input = new InputForm();
+ fromJobConfig = new FromJobConfig();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/InputForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/InputForm.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/InputForm.java
deleted file mode 100644
index 413f04c..0000000
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/InputForm.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.hdfs.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-
-/**
- *
- */
-@FormClass
-public class InputForm {
-
- @Input(size = 255) public String inputDirectory;
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
new file mode 100644
index 0000000..b689854
--- /dev/null
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs.configuration;
+
+import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Input;
+
+@FormClass
+public class LinkConfig {
+ //Todo: Didn't find anything that belongs here...
+ // Since empty forms don't work (DERBYREPO_0008:The form contains no input metadata), I'm putting a dummy form here
+
+ @Input(size = 255) public String dummy;
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
new file mode 100644
index 0000000..4970821
--- /dev/null
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs.configuration;
+
+import org.apache.sqoop.model.ConfigurationClass;
+import org.apache.sqoop.model.Form;
+
+@ConfigurationClass
+public class LinkConfiguration {
+ @Form
+ public LinkConfig link;
+
+ public LinkConfiguration() {
+ link = new LinkConfig();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputCompression.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputCompression.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputCompression.java
deleted file mode 100644
index 55db1bc..0000000
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputCompression.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.hdfs.configuration;
-
-/**
- * Supported compressions
- */
-public enum OutputCompression {
- NONE,
- DEFAULT,
- DEFLATE,
- GZIP,
- BZIP2,
- LZO,
- LZ4,
- SNAPPY,
- CUSTOM,
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputForm.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputForm.java
deleted file mode 100644
index d57b4c2..0000000
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputForm.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.hdfs.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-
-/**
- *
- */
-@FormClass
-public class OutputForm {
-
- @Input public OutputFormat outputFormat;
-
- @Input public OutputCompression compression;
-
- @Input(size = 255) public String customCompression;
-
- @Input(size = 255) public String outputDirectory;
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputFormat.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputFormat.java
deleted file mode 100644
index 676c33c..0000000
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/OutputFormat.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.hdfs.configuration;
-
-/**
- * Various supported formats on disk
- */
-public enum OutputFormat {
- /**
- * Comma separated text file
- */
- TEXT_FILE,
-
- /**
- * Sequence file
- */
- SEQUENCE_FILE,
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToCompression.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToCompression.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToCompression.java
new file mode 100644
index 0000000..34e629a
--- /dev/null
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToCompression.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs.configuration;
+
+/**
+ * Supported compressions
+ */
+public enum ToCompression {
+ NONE,
+ DEFAULT,
+ DEFLATE,
+ GZIP,
+ BZIP2,
+ LZO,
+ LZ4,
+ SNAPPY,
+ CUSTOM,
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToFormat.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToFormat.java
new file mode 100644
index 0000000..27d121f
--- /dev/null
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToFormat.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs.configuration;
+
+/**
+ * Various supported formats on disk
+ */
+public enum ToFormat {
+ /**
+ * Comma separated text file
+ */
+ TEXT_FILE,
+
+ /**
+ * Sequence file
+ */
+ SEQUENCE_FILE,
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
new file mode 100644
index 0000000..b1308db
--- /dev/null
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs.configuration;
+
+import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Input;
+
+/**
+ *
+ */
+@FormClass
+public class ToJobConfig {
+
+ @Input public ToFormat outputFormat;
+
+ @Input public ToCompression compression;
+
+ @Input(size = 255) public String customCompression;
+
+ @Input(size = 255) public String outputDirectory;
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
index 65ee8a7..bba249c 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
@@ -23,9 +23,9 @@ import org.apache.sqoop.model.Form;
@ConfigurationClass
public class ToJobConfiguration {
@Form
- public OutputForm output;
+ public ToJobConfig toJobConfig;
public ToJobConfiguration() {
- output = new OutputForm();
+ toJobConfig = new ToJobConfig();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/hdfsWriter/HdfsSequenceWriter.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/hdfsWriter/HdfsSequenceWriter.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/hdfsWriter/HdfsSequenceWriter.java
index eb80121..75c2e7e 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/hdfsWriter/HdfsSequenceWriter.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/hdfsWriter/HdfsSequenceWriter.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
-
import java.io.IOException;
public class HdfsSequenceWriter extends GenericHdfsWriter {
@@ -32,6 +31,7 @@ public class HdfsSequenceWriter extends GenericHdfsWriter {
private SequenceFile.Writer filewriter;
private Text text;
+ @SuppressWarnings("deprecation")
public void initialize(Path filepath, Configuration conf, CompressionCodec codec) throws IOException {
if (codec != null) {
filewriter = SequenceFile.createWriter(filepath.getFileSystem(conf),
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
new file mode 100644
index 0000000..b603f2f
--- /dev/null
+++ b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generic HDFS Connector Resources
+
+############################
+# Link Config
+#
+link.label = Link configuration
+link.help = You must supply the information requested in order to \
+ create a connection object.
+
+link.dummy.label = Dummy parameter needed to get HDFS connector to register
+link.dummy.help = You can write anything here. Doesn't matter.
+
+# To Job Config
+#
+toJobConfig.label = ToJob configuration
+toJobConfig.help = You must supply the information requested in order to \
+ get information where you want to store your data.
+
+toJobConfig.storageType.label = Storage type
+toJobConfig.storageType.help = Target on Hadoop ecosystem where to store data
+
+toJobConfig.outputFormat.label = Output format
+toJobConfig.outputFormat.help = Format in which data should be serialized
+
+toJobConfig.compression.label = Compression format
+toJobConfig.compression.help = Compression that should be used for the data
+
+toJobConfig.customCompression.label = Custom compression format
+toJobConfig.customCompression.help = Full class name of the custom compression
+
+toJobConfig.outputDirectory.label = Output directory
+toJobConfig.outputDirectory.help = Output directory for final data
+
+toJobConfig.ignored.label = Ignored
+toJobConfig.ignored.help = This value is ignored
+
+# From Job Config
+#
+fromJobConfig.label = From Job configuration
+fromJobConfig.help = Specifies information required to get data from Hadoop ecosystem
+
+fromJobConfig.inputDirectory.label = Input directory
+fromJobConfig.inputDirectory.help = Directory that should be exported
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/main/resources/hdfs-connector-resources.properties
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/resources/hdfs-connector-resources.properties b/connector/connector-hdfs/src/main/resources/hdfs-connector-resources.properties
deleted file mode 100644
index 3125911..0000000
--- a/connector/connector-hdfs/src/main/resources/hdfs-connector-resources.properties
+++ /dev/null
@@ -1,58 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Generic HDFS Connector Resources
-
-############################
-# Connection Form
-#
-connection.label = Connection configuration
-connection.help = You must supply the information requested in order to \
- create a connection object.
-
-connection.dummy.label = Dummy parameter needed to get HDFS connector to register
-connection.dummy.help = You can write anything here. Doesn't matter.
-
-# Output From
-#
-output.label = Output configuration
-output.help = You must supply the information requested in order to \
- get information where you want to store your data.
-
-output.storageType.label = Storage type
-output.storageType.help = Target on Hadoop ecosystem where to store data
-
-output.outputFormat.label = Output format
-output.outputFormat.help = Format in which data should be serialized
-
-output.compression.label = Compression format
-output.compression.help = Compression that should be used for the data
-
-output.customCompression.label = Custom compression format
-output.customCompression.help = Full class name of the custom compression
-
-output.outputDirectory.label = Output directory
-output.outputDirectory.help = Output directory for final data
-
-output.ignored.label = Ignored
-output.ignored.help = This value is ignored
-
-# Input Form
-#
-input.label = Input configuration
-input.help = Specifies information required to get data from Hadoop ecosystem
-
-input.inputDirectory.label = Input directory
-input.inputDirectory.help = Directory that should be exported
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
index 5e21543..7942d59 100644
--- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
@@ -22,9 +22,9 @@ import org.apache.hadoop.io.compress.BZip2Codec;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.sqoop.common.PrefixContext;
-import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
-import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
+import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
import org.apache.sqoop.etl.io.DataWriter;
import org.apache.sqoop.job.etl.Extractor;
import org.apache.sqoop.job.etl.ExtractorContext;
@@ -40,8 +40,8 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.SEQUENCE_FILE;
-import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.TEXT_FILE;
+import static org.apache.sqoop.connector.hdfs.configuration.ToFormat.SEQUENCE_FILE;
+import static org.apache.sqoop.connector.hdfs.configuration.ToFormat.TEXT_FILE;
@RunWith(Parameterized.class)
public class TestExtractor extends TestHdfsBase {
@@ -49,12 +49,12 @@ public class TestExtractor extends TestHdfsBase {
private static final int NUMBER_OF_FILES = 5;
private static final int NUMBER_OF_ROWS_PER_FILE = 1000;
- private OutputFormat outputFileType;
+ private ToFormat outputFileType;
private Class<? extends CompressionCodec> compressionClass;
private final String inputDirectory;
private Extractor extractor;
- public TestExtractor(OutputFormat outputFileType,
+ public TestExtractor(ToFormat outputFileType,
Class<? extends CompressionCodec> compressionClass)
throws Exception {
this.inputDirectory = INPUT_ROOT + getClass().getSimpleName();
@@ -130,7 +130,7 @@ public class TestExtractor extends TestHdfsBase {
throw new AssertionError("Should not be writing object.");
}
}, null);
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
HdfsPartition partition = createPartition(FileUtils.listDir(inputDirectory));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
index 79cf1f1..552a751 100644
--- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
@@ -27,9 +27,9 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.sqoop.common.PrefixContext;
-import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
-import org.apache.sqoop.connector.hdfs.configuration.OutputCompression;
-import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.ToCompression;
+import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
import org.apache.sqoop.connector.hdfs.configuration.ToJobConfiguration;
import org.apache.sqoop.etl.io.DataReader;
import org.apache.sqoop.job.etl.Loader;
@@ -48,21 +48,21 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.SEQUENCE_FILE;
-import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.TEXT_FILE;
+import static org.apache.sqoop.connector.hdfs.configuration.ToFormat.SEQUENCE_FILE;
+import static org.apache.sqoop.connector.hdfs.configuration.ToFormat.TEXT_FILE;
@RunWith(Parameterized.class)
public class TestLoader extends TestHdfsBase {
private static final String INPUT_ROOT = System.getProperty("maven.build.directory", "/tmp") + "/sqoop/warehouse/";
private static final int NUMBER_OF_ROWS_PER_FILE = 1000;
- private OutputFormat outputFormat;
- private OutputCompression compression;
+ private ToFormat outputFormat;
+ private ToCompression compression;
private final String outputDirectory;
private Loader loader;
- public TestLoader(OutputFormat outputFormat,
- OutputCompression compression)
+ public TestLoader(ToFormat outputFormat,
+ ToCompression compression)
throws Exception {
this.outputDirectory = INPUT_ROOT + getClass().getSimpleName();
this.outputFormat = outputFormat;
@@ -73,10 +73,10 @@ public class TestLoader extends TestHdfsBase {
@Parameterized.Parameters
public static Collection<Object[]> data() {
List<Object[]> parameters = new ArrayList<Object[]>();
- for (OutputCompression compression : new OutputCompression[]{
- OutputCompression.DEFAULT,
- OutputCompression.BZIP2,
- OutputCompression.NONE
+ for (ToCompression compression : new ToCompression[]{
+ ToCompression.DEFAULT,
+ ToCompression.BZIP2,
+ ToCompression.NONE
}) {
for (Object outputFileType : new Object[]{TEXT_FILE, SEQUENCE_FILE}) {
parameters.add(new Object[]{outputFileType, compression});
@@ -121,11 +121,11 @@ public class TestLoader extends TestHdfsBase {
return null;
}
}, null);
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
- jobConf.output.outputDirectory = outputDirectory;
- jobConf.output.compression = compression;
- jobConf.output.outputFormat = outputFormat;
+ jobConf.toJobConfig.outputDirectory = outputDirectory;
+ jobConf.toJobConfig.compression = compression;
+ jobConf.toJobConfig.outputFormat = outputFormat;
Path outputPath = new Path(outputDirectory);
loader.load(context, connConf, jobConf);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java
index ae93b0a..9d177ec 100644
--- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java
@@ -22,9 +22,9 @@ import org.apache.hadoop.io.compress.BZip2Codec;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.sqoop.common.PrefixContext;
-import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
-import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
+import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
import org.apache.sqoop.job.etl.Partition;
import org.apache.sqoop.job.etl.Partitioner;
import org.apache.sqoop.job.etl.PartitionerContext;
@@ -39,7 +39,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.*;
+import static org.apache.sqoop.connector.hdfs.configuration.ToFormat.*;
import static org.junit.Assert.assertEquals;
@RunWith(Parameterized.class)
@@ -48,13 +48,13 @@ public class TestPartitioner extends TestHdfsBase {
private static final int NUMBER_OF_FILES = 5;
private static final int NUMBER_OF_ROWS_PER_FILE = 1000;
- private OutputFormat outputFileType;
+ private ToFormat outputFileType;
private Class<? extends CompressionCodec> compressionClass;
private Partitioner partitioner;
private final String inputDirectory;
- public TestPartitioner(OutputFormat outputFileType, Class<? extends CompressionCodec> compressionClass) {
+ public TestPartitioner(ToFormat outputFileType, Class<? extends CompressionCodec> compressionClass) {
this.inputDirectory = INPUT_ROOT + getClass().getSimpleName();
this.outputFileType = outputFileType;
this.compressionClass = compressionClass;
@@ -97,10 +97,10 @@ public class TestPartitioner extends TestHdfsBase {
Configuration conf = new Configuration();
PrefixContext prefixContext = new PrefixContext(conf, "org.apache.sqoop.job.connector.from.context.");
PartitionerContext context = new PartitionerContext(prefixContext, 5, null);
- ConnectionConfiguration connConf = new ConnectionConfiguration();
+ LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
- jobConf.input.inputDirectory = inputDirectory;
+ jobConf.fromJobConfig.inputDirectory = inputDirectory;
List<Partition> partitions = partitioner.getPartitions(context, connConf, jobConf);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
index 66d46a3..74b9518 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
@@ -26,7 +26,7 @@ import java.io.IOException;
/**
* Abstract class representing a pluggable intermediate data format the Sqoop
- * framework will use to move data to/from the connector. All intermediate
+ * driver will use to move data to/from the connector. All intermediate
* data formats are expected to have an internal/native implementation,
* but also should minimally be able to return a text (CSV) version of the
* data. The data format should also be able to return the data as an object
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
index 6c10b54..dbfdc03 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
@@ -106,7 +106,7 @@ public final class ConnectorHandler {
}
MConnectionForms connectionForms = new MConnectionForms(
- FormUtils.toForms(connector.getConnectionConfigurationClass()));
+ FormUtils.toForms(connector.getLinkConfigurationClass()));
String connectorVersion = connector.getVersion();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
index 898ec73..f341108 100644
--- a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
+++ b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
@@ -73,8 +73,8 @@ public final class ConfigurationConstants {
public static final String CONNECTOR_AUTO_UPGRADE =
"org.apache.sqoop.connector.autoupgrade";
- public static final String FRAMEWORK_AUTO_UPGRADE =
- "org.apache.sqoop.framework.autoupgrade";
+ public static final String DRIVER_AUTO_UPGRADE =
+ "org.apache.sqoop.driver.autoupgrade";
/**
* Enable Sqoop App to kill Tomcat in case that it will fail to load.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/core/SqoopServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopServer.java b/core/src/main/java/org/apache/sqoop/core/SqoopServer.java
index d4c3b39..ac836c7 100644
--- a/core/src/main/java/org/apache/sqoop/core/SqoopServer.java
+++ b/core/src/main/java/org/apache/sqoop/core/SqoopServer.java
@@ -20,8 +20,8 @@ package org.apache.sqoop.core;
import org.apache.log4j.Logger;
import org.apache.sqoop.audit.AuditLoggerManager;
import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.framework.FrameworkManager;
-import org.apache.sqoop.framework.JobManager;
+import org.apache.sqoop.driver.Driver;
+import org.apache.sqoop.driver.JobManager;
import org.apache.sqoop.repository.RepositoryManager;
/**
@@ -34,7 +34,7 @@ public class SqoopServer {
public static void destroy() {
LOG.info("Shutting down Sqoop server");
JobManager.getInstance().destroy();
- FrameworkManager.getInstance().destroy();
+ Driver.getInstance().destroy();
ConnectorManager.getInstance().destroy();
RepositoryManager.getInstance().destroy();
AuditLoggerManager.getInstance().destroy();
@@ -49,7 +49,7 @@ public class SqoopServer {
AuditLoggerManager.getInstance().initialize();
RepositoryManager.getInstance().initialize();
ConnectorManager.getInstance().initialize();
- FrameworkManager.getInstance().initialize();
+ Driver.getInstance().initialize();
JobManager.getInstance().initialize();
LOG.info("Sqoop server has successfully boot up");
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/Driver.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/Driver.java b/core/src/main/java/org/apache/sqoop/driver/Driver.java
new file mode 100644
index 0000000..5297bde
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/Driver.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import java.util.Locale;
+import java.util.ResourceBundle;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
+import org.apache.sqoop.core.ConfigurationConstants;
+import org.apache.sqoop.core.Reconfigurable;
+import org.apache.sqoop.core.SqoopConfiguration;
+import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
+import org.apache.sqoop.driver.configuration.JobConfiguration;
+import org.apache.sqoop.driver.configuration.LinkConfiguration;
+import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.repository.RepositoryManager;
+import org.apache.sqoop.validation.Validator;
+
+/**
+ * Sqoop driver that manages the job lifecyle
+ *
+ * All Sqoop internals are handled in this class:
+ * * Submission engine
+ * * Execution engine
+ * * Driver config
+ *
+ * Current implementation of entire submission engine is using repository
+ * for keeping track of running submissions. Thus, server might be restarted at
+ * any time without any affect on running jobs. This approach however might not
+ * be the fastest way and we might want to introduce internal structures for
+ * running jobs in case that this approach will be too slow.
+ */
+public class Driver implements Reconfigurable {
+
+ /**
+ * Logger object.
+ */
+ private static final Logger LOG = Logger.getLogger(Driver.class);
+
+ /**
+ * Private instance to singleton of this class.
+ */
+ private static Driver instance;
+
+ /**
+ * Create default object by default.
+ *
+ * Every Sqoop server application needs one so this should not be performance issue.
+ */
+ static {
+ instance = new Driver();
+ }
+
+ /**
+ * Return current instance.
+ *
+ * @return Current instance
+ */
+ public static Driver getInstance() {
+ return instance;
+ }
+
+ /**
+ * Allows to set instance in case that it's need.
+ *
+ * This method should not be normally used as the default instance should be sufficient. One target
+ * user use case for this method are unit tests.
+ *
+ * @param newInstance New instance
+ */
+ public static void setInstance(Driver newInstance) {
+ instance = newInstance;
+ }
+
+ /**
+ * Driver config structure
+ */
+ private MDriverConfig mDriverConfig;
+
+ /**
+ * Validator instance
+ */
+ private final Validator validator;
+
+ /**
+ * Driver config upgrader instance
+ */
+ private final RepositoryUpgrader driverConfigUpgrader;
+
+ /**
+ * Default driver config auto upgrade option value
+ */
+ private static final boolean DEFAULT_AUTO_UPGRADE = false;
+
+ public static final String CURRENT_DRIVER_VERSION = "1";
+
+ public Class getJobConfigurationClass() {
+ return JobConfiguration.class;
+ }
+
+ public Class getLinkConfigurationClass() {
+ return LinkConfiguration.class;
+ }
+
+ public Driver() {
+ MConnectionForms connectionForms = new MConnectionForms(
+ FormUtils.toForms(getLinkConfigurationClass())
+ );
+ mDriverConfig = new MDriverConfig(connectionForms, new MJobForms(FormUtils.toForms(getJobConfigurationClass())),
+ CURRENT_DRIVER_VERSION);
+
+ // Build validator
+ validator = new DriverValidator();
+ // Build upgrader
+ driverConfigUpgrader = new DriverConfigUpgrader();
+ }
+
+ public synchronized void initialize() {
+ initialize(SqoopConfiguration.getInstance().getContext().getBoolean(ConfigurationConstants.DRIVER_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE));
+ }
+
+ public synchronized void initialize(boolean autoUpgrade) {
+ LOG.trace("Begin Driver Config initialization");
+
+ // Register driver config in repository
+ mDriverConfig = RepositoryManager.getInstance().getRepository().registerDriverConfig(mDriverConfig, autoUpgrade);
+
+ SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this));
+
+ LOG.info("Driver Config initialized: OK");
+ }
+
+ public synchronized void destroy() {
+ LOG.trace("Begin Driver Config destroy");
+ }
+
+ public Validator getValidator() {
+ return validator;
+ }
+
+ public RepositoryUpgrader getDriverConfigRepositoryUpgrader() {
+ return driverConfigUpgrader;
+ }
+
+ public MDriverConfig getDriverConfig() {
+ return mDriverConfig;
+ }
+
+ public ResourceBundle getBundle(Locale locale) {
+ return ResourceBundle.getBundle(DriverConstants.DRIVER_CONFIG_BUNDLE, locale);
+ }
+
+ @Override
+ public void configurationChanged() {
+ LOG.info("Begin Driver reconfiguring");
+ // If there are configuration options for Driver,
+ // implement the reconfiguration procedure right here.
+ LOG.info("Driver reconfigured");
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
new file mode 100644
index 0000000..8d6eb78
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MJobForms;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class DriverConfigUpgrader extends RepositoryUpgrader{
+
+ private static final Logger LOG = Logger.getLogger(DriverConfigUpgrader.class);
+
+ @Override
+ public void upgrade(MConnectionForms original,
+ MConnectionForms upgradeTarget) {
+ doUpgrade(original.getForms(), upgradeTarget.getForms());
+ }
+
+ @Override
+ public void upgrade(MJobForms original, MJobForms upgradeTarget) {
+ doUpgrade(original.getForms(), upgradeTarget.getForms());
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private void doUpgrade(List<MForm> original, List<MForm> target) {
+ // Easier to find the form in the original forms list if we use a map.
+ // Since the constructor of MJobForms takes a list,
+ // index is not guaranteed to be the same, so we need to look for
+ // equivalence
+ Map<String, MForm> formMap = new HashMap<String, MForm>();
+ for (MForm form : original) {
+ formMap.put(form.getName(), form);
+ }
+ for (MForm form : target) {
+ List<MInput<?>> inputs = form.getInputs();
+ MForm originalForm = formMap.get(form.getName());
+ if(originalForm == null) {
+ LOG.warn("Form: " + form.getName() + " not present in old " +
+ "driver config. So it will not be transferred by the upgrader.");
+ continue;
+ }
+
+ for (MInput input : inputs) {
+ try {
+ MInput originalInput = originalForm.getInput(input.getName());
+ input.setValue(originalInput.getValue());
+ } catch (SqoopException ex) {
+ LOG.warn("Input: " + input.getName() + " not present in old " +
+ "driver config. So it will not be transferred by the upgrader.");
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/DriverConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConstants.java b/core/src/main/java/org/apache/sqoop/driver/DriverConstants.java
new file mode 100644
index 0000000..795944a
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverConstants.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.core.ConfigurationConstants;
+
+/**
+ * Constants that are used in driver module.
+ */
+public final class DriverConstants {
+
+ // Sqoop driver constants
+
+ public static final String PREFIX_SUBMISSION_CONFIG =
+ ConfigurationConstants.PREFIX_GLOBAL_CONFIG + "submission.";
+
+ public static final String PREFIX_EXECUTION_CONFIG =
+ ConfigurationConstants.PREFIX_GLOBAL_CONFIG + "execution.";
+
+ public static final String SYSCFG_SUBMISSION_ENGINE =
+ PREFIX_SUBMISSION_CONFIG + "engine";
+
+ public static final String PREFIX_SUBMISSION_ENGINE_CONFIG =
+ SYSCFG_SUBMISSION_ENGINE + ".";
+
+ public static final String PREFIX_SUBMISSION_PURGE_CONFIG =
+ PREFIX_SUBMISSION_CONFIG + "purge.";
+
+ public static final String SYSCFG_SUBMISSION_PURGE_THRESHOLD =
+ PREFIX_SUBMISSION_PURGE_CONFIG + "threshold";
+
+ public static final String SYSCFG_SUBMISSION_PURGE_SLEEP =
+ PREFIX_SUBMISSION_PURGE_CONFIG + "sleep";
+
+ public static final String PREFIX_SUBMISSION_UPDATE_CONFIG =
+ PREFIX_SUBMISSION_CONFIG + "update.";
+
+ public static final String SYSCFG_SUBMISSION_UPDATE_SLEEP =
+ PREFIX_SUBMISSION_UPDATE_CONFIG + "sleep";
+
+ public static final String SYSCFG_EXECUTION_ENGINE =
+ PREFIX_EXECUTION_CONFIG + "engine";
+
+ public static final String PREFIX_EXECUTION_ENGINE_CONFIG =
+ SYSCFG_EXECUTION_ENGINE + ".";
+
+ // Bundle name
+ public static final String DRIVER_CONFIG_BUNDLE = "driver-config";
+
+ private DriverConstants() {
+ // Instantiation of this class is prohibited
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/DriverError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverError.java b/core/src/main/java/org/apache/sqoop/driver/DriverError.java
new file mode 100644
index 0000000..56ef9bb
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverError.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.common.ErrorCode;
+
+/**
+ *
+ */
+public enum DriverError implements ErrorCode {
+
+ DRIVER_0000("Metadata are not registered in repository"),
+
+ DRIVER_0001("Invalid submission engine"),
+
+ DRIVER_0002("Given job is already running"),
+
+ DRIVER_0003("Given job is not running"),
+
+ DRIVER_0004("Unknown job id"),
+
+ DRIVER_0005("Unsupported job type"),
+
+ DRIVER_0006("Can't bootstrap job"),
+
+ DRIVER_0007("Invalid execution engine"),
+
+ DRIVER_0008("Invalid combination of submission and execution engines"),
+
+ DRIVER_0009("Job has been disabled. Cannot submit this job."),
+
+ DRIVER_0010("Connection for this job has been disabled. Cannot submit this job."),
+
+ DRIVER_0011("Connector does not support direction. Cannot submit this job."),
+
+ ;
+
+ private final String message;
+
+ private DriverError(String message) {
+ this.message = message;
+ }
+
+ public String getCode() {
+ return name();
+ }
+
+ public String getMessage() {
+ return message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
new file mode 100644
index 0000000..9cc51dd
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.driver.configuration.LinkConfiguration;
+import org.apache.sqoop.driver.configuration.JobConfiguration;
+import org.apache.sqoop.driver.configuration.ThrottlingForm;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.Validation;
+import org.apache.sqoop.validation.Validator;
+
+public class DriverValidator extends Validator {
+ @Override
+ public Validation validateLink(Object linkConfiguration) {
+ Validation validation = new Validation(LinkConfiguration.class);
+ // No validation on link object
+ return validation;
+ }
+
+ @Override
+ public Validation validateJob(Object jobConfiguration) {
+ Validation validation = new Validation(JobConfiguration.class);
+ JobConfiguration conf = (JobConfiguration)jobConfiguration;
+ validateThrottlingForm(validation,conf.throttling);
+
+ return validation;
+ };
+
+ private void validateThrottlingForm(Validation validation, ThrottlingForm throttling) {
+ if(throttling.extractors != null && throttling.extractors < 1) {
+ validation.addMessage(Status.UNACCEPTABLE, "throttling", "extractors", "You need to specify more than one extractor");
+ }
+
+ if(throttling.loaders != null && throttling.loaders < 1) {
+ validation.addMessage(Status.UNACCEPTABLE, "throttling", "loaders", "You need to specify more than one loader");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/ExecutionEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/ExecutionEngine.java b/core/src/main/java/org/apache/sqoop/driver/ExecutionEngine.java
new file mode 100644
index 0000000..ea38ed4
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/ExecutionEngine.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.common.ImmutableContext;
+
+/**
+ * Execution engine drives execution of sqoop job. It's responsible
+ * for executing all defined steps in the import/export workflow.
+ * A successful job execution will be recorded in the job submission entity
+ */
+public abstract class ExecutionEngine {
+
+ /**
+ * Initialize execution engine
+ *
+ * @param context Configuration context
+ * @parma prefix Execution engine prefix
+ */
+ public void initialize(ImmutableContext context, String prefix) {
+ }
+
+ /**
+ * Destroy execution engine when stopping server
+ */
+ public void destroy() {
+ }
+
+ /**
+ * Return new JobRequest class or any subclass if it's needed by
+ * execution and submission engine combination.
+ *
+ * @return new JobRequestobject
+ */
+ public JobRequest createJobRequest() {
+ return new JobRequest();
+ }
+
+ /**
+ * Prepare given job request.
+ *
+ * @param request JobRequest
+ */
+ public abstract void prepareJob(JobRequest request);
+}
[15/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/JobManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobManager.java b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
new file mode 100644
index 0000000..277c6be
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
@@ -0,0 +1,712 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import java.util.Date;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.MapContext;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.ConnectorManager;
+import org.apache.sqoop.connector.idf.IntermediateDataFormat;
+import org.apache.sqoop.connector.spi.SqoopConnector;
+import org.apache.sqoop.core.Reconfigurable;
+import org.apache.sqoop.core.SqoopConfiguration;
+import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
+import org.apache.sqoop.driver.configuration.JobConfiguration;
+import org.apache.sqoop.job.etl.Destroyer;
+import org.apache.sqoop.job.etl.DestroyerContext;
+import org.apache.sqoop.job.etl.Initializer;
+import org.apache.sqoop.job.etl.InitializerContext;
+import org.apache.sqoop.job.etl.Transferable;
+import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.repository.Repository;
+import org.apache.sqoop.repository.RepositoryManager;
+import org.apache.sqoop.request.HttpEventContext;
+import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.submission.SubmissionStatus;
+import org.apache.sqoop.submission.counter.Counters;
+import org.apache.sqoop.utils.ClassUtils;
+import org.json.simple.JSONValue;
+
+public class JobManager implements Reconfigurable {
+ /**
+ * Logger object.
+ */
+ private static final Logger LOG = Logger.getLogger(JobManager.class);
+
+ /**
+ * Private instance to singleton of this class.
+ */
+ private static JobManager instance;
+ /**
+ * Create default object by default.
+ *
+ * Every Sqoop server application needs one so this should not be performance
+ * issue.
+ */
+ static {
+ instance = new JobManager();
+ }
+
+ /**
+ * Return current instance.
+ *
+ * @return Current instance
+ */
+ public static JobManager getInstance() {
+ return instance;
+ }
+
+ /**
+ * Allows to set instance in case that it's need.
+ *
+ * This method should not be normally used as the default instance should be
+ * sufficient. One target user use case for this method are unit tests.
+ *
+ * @param newInstance
+ * New instance
+ */
+ public static void setInstance(JobManager newInstance) {
+ instance = newInstance;
+ }
+
+ /**
+ * Default interval for purging old submissions from repository.
+ */
+ private static final long DEFAULT_PURGE_THRESHOLD = 24 * 60 * 60 * 1000;
+
+ /**
+ * Default sleep interval for purge thread.
+ */
+ private static final long DEFAULT_PURGE_SLEEP = 24 * 60 * 60 * 1000;
+
+ /**
+ * Default interval for update thread.
+ */
+ private static final long DEFAULT_UPDATE_SLEEP = 60 * 5 * 1000;
+
+ /**
+ * Configured submission engine instance
+ */
+ private SubmissionEngine submissionEngine;
+
+ /**
+ * Configured execution engine instance
+ */
+ private ExecutionEngine executionEngine;
+
+ /**
+ * Purge thread that will periodically remove old submissions from repository.
+ */
+ private PurgeThread purgeThread = null;
+
+ /**
+ * Update thread that will periodically check status of running submissions.
+ */
+ private UpdateThread updateThread = null;
+
+ /**
+ * Synchronization variable between threads.
+ */
+ private boolean running = true;
+
+ /**
+ * Specifies how old submissions should be removed from repository.
+ */
+ private long purgeThreshold;
+
+ /**
+ * Number of milliseconds for purge thread to sleep.
+ */
+ private long purgeSleep;
+
+ /**
+ * Number of milliseconds for update thread to slepp.
+ */
+ private long updateSleep;
+
+ /**
+ * Base notification URL.
+ *
+ * Driver manager will always add job id.
+ */
+ private String notificationBaseUrl;
+
+ /**
+ * Set notification base URL.
+ *
+ * @param url
+ * Base URL
+ */
+ public void setNotificationBaseUrl(String url) {
+ LOG.debug("Setting notification base URL to " + url);
+ notificationBaseUrl = url;
+ }
+
+ /**
+ * Get base notification url.
+ *
+ * @return String representation of the URL
+ */
+ public String getNotificationBaseUrl() {
+ return notificationBaseUrl;
+ }
+
+ public synchronized void destroy() {
+ LOG.trace("Begin submission engine manager destroy");
+
+ running = false;
+
+ try {
+ purgeThread.interrupt();
+ purgeThread.join();
+ } catch (InterruptedException e) {
+ // TODO(jarcec): Do I want to wait until it actually finish here?
+ LOG.error("Interrupted joining purgeThread");
+ }
+
+ try {
+ updateThread.interrupt();
+ updateThread.join();
+ } catch (InterruptedException e) {
+ // TODO(jarcec): Do I want to wait until it actually finish here?
+ LOG.error("Interrupted joining updateThread");
+ }
+
+ if (submissionEngine != null) {
+ submissionEngine.destroy();
+ }
+
+ if (executionEngine != null) {
+ executionEngine.destroy();
+ }
+ }
+
+ public synchronized void initialize() {
+ LOG.trace("Begin submission engine manager initialization");
+ MapContext context = SqoopConfiguration.getInstance().getContext();
+
+ // Let's load configured submission engine
+ String submissionEngineClassName =
+ context.getString(DriverConstants.SYSCFG_SUBMISSION_ENGINE);
+
+ submissionEngine = (SubmissionEngine) ClassUtils
+ .instantiate(submissionEngineClassName);
+ if (submissionEngine == null) {
+ throw new SqoopException(DriverError.DRIVER_0001,
+ submissionEngineClassName);
+ }
+
+ submissionEngine.initialize(context,
+ DriverConstants.PREFIX_SUBMISSION_ENGINE_CONFIG);
+
+ // Execution engine
+ String executionEngineClassName =
+ context.getString(DriverConstants.SYSCFG_EXECUTION_ENGINE);
+
+ executionEngine = (ExecutionEngine) ClassUtils
+ .instantiate(executionEngineClassName);
+ if (executionEngine == null) {
+ throw new SqoopException(DriverError.DRIVER_0007,
+ executionEngineClassName);
+ }
+
+ // We need to make sure that user has configured compatible combination of
+ // submission engine and execution engine
+ if (!submissionEngine
+ .isExecutionEngineSupported(executionEngine.getClass())) {
+ throw new SqoopException(DriverError.DRIVER_0008);
+ }
+
+ executionEngine.initialize(context,
+ DriverConstants.PREFIX_EXECUTION_ENGINE_CONFIG);
+
+ // Set up worker threads
+ purgeThreshold = context.getLong(
+ DriverConstants.SYSCFG_SUBMISSION_PURGE_THRESHOLD,
+ DEFAULT_PURGE_THRESHOLD
+ );
+ purgeSleep = context.getLong(
+ DriverConstants.SYSCFG_SUBMISSION_PURGE_SLEEP,
+ DEFAULT_PURGE_SLEEP
+ );
+
+ purgeThread = new PurgeThread();
+ purgeThread.start();
+
+ updateSleep = context.getLong(
+ DriverConstants.SYSCFG_SUBMISSION_UPDATE_SLEEP,
+ DEFAULT_UPDATE_SLEEP
+ );
+
+ updateThread = new UpdateThread();
+ updateThread.start();
+
+ SqoopConfiguration.getInstance().getProvider()
+ .registerListener(new CoreConfigurationListener(this));
+
+ LOG.info("Submission manager initialized: OK");
+ }
+
+ public MSubmission submit(long jobId, HttpEventContext ctx) {
+
+ MSubmission mSubmission = createJobSubmission(ctx, jobId);
+ JobRequest jobRequest = createJobRequest(jobId, mSubmission);
+ // Bootstrap job to execute
+ prepareJob(jobRequest);
+ // Make sure that this job id is not currently running and submit the job
+ // only if it's not.
+ synchronized (getClass()) {
+ MSubmission lastSubmission = RepositoryManager.getInstance().getRepository()
+ .findSubmissionLastForJob(jobId);
+ if (lastSubmission != null && lastSubmission.getStatus().isRunning()) {
+ throw new SqoopException(DriverError.DRIVER_0002, "Job with id " + jobId);
+ }
+ // TODO(Abe): Call multiple destroyers.
+ // TODO(jarcec): We might need to catch all exceptions here to ensure
+ // that Destroyer will be executed in all cases.
+ // NOTE: the following is a blocking call
+ boolean success = submissionEngine.submit(jobRequest);
+ if (!success) {
+ destroySubmission(jobRequest);
+ mSubmission.setStatus(SubmissionStatus.FAILURE_ON_SUBMIT);
+ }
+ RepositoryManager.getInstance().getRepository().createSubmission(mSubmission);
+ }
+ return mSubmission;
+ }
+
+ private JobRequest createJobRequest(long jobId, MSubmission submission) {
+ // get job
+ MJob job = getJob(jobId);
+
+ // get from/to connections for the job
+ MLink fromConnection = getLink(job.getLinkId(Direction.FROM));
+ MLink toConnection = getLink(job.getLinkId(Direction.TO));
+
+ // get from/to connectors for the connection
+ SqoopConnector fromConnector = getConnector(fromConnection.getConnectorId());
+ validateSupportedDirection(fromConnector, Direction.FROM);
+ SqoopConnector toConnector = getConnector(toConnection.getConnectorId());
+ validateSupportedDirection(toConnector, Direction.TO);
+
+ // Transform config to fromConnector specific classes
+ Object fromConnectionConfig = ClassUtils.instantiate(fromConnector
+ .getLinkConfigurationClass());
+ FormUtils.fromForms(fromConnection.getConnectorPart().getForms(), fromConnectionConfig);
+
+ // Transform config to toConnector specific classes
+ Object toConnectorConfig = ClassUtils
+ .instantiate(toConnector.getLinkConfigurationClass());
+ FormUtils.fromForms(toConnection.getConnectorPart().getForms(), toConnectorConfig);
+
+ Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
+ FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromJob);
+
+ Object toJob = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
+ FormUtils.fromForms(job.getConnectorPart(Direction.TO).getForms(), toJob);
+
+ // Transform framework specific configs
+ // Q(VB) : Aren't the following 2 exactly the same?
+ Object fromDriverConnection = ClassUtils.instantiate(Driver.getInstance()
+ .getLinkConfigurationClass());
+ FormUtils.fromForms(fromConnection.getFrameworkPart().getForms(), fromDriverConnection);
+
+ Object toDriverConnection = ClassUtils.instantiate(Driver.getInstance()
+ .getLinkConfigurationClass());
+ FormUtils.fromForms(toConnection.getFrameworkPart().getForms(), toDriverConnection);
+
+ Object frameworkJob = ClassUtils.instantiate(Driver.getInstance()
+ .getJobConfigurationClass());
+ FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);
+
+ // Create a job request for submit/execution
+ JobRequest jobRequest = executionEngine.createJobRequest();
+ // Save important variables to the job request
+ jobRequest.setSummary(submission);
+ jobRequest.setConnector(Direction.FROM, fromConnector);
+ jobRequest.setConnector(Direction.TO, toConnector);
+ jobRequest.setConnectorLinkConfig(Direction.FROM, fromConnectionConfig);
+ jobRequest.setConnectorLinkConfig(Direction.TO, toConnectorConfig);
+ jobRequest.setConnectorJobConfig(Direction.FROM, fromJob);
+ jobRequest.setConnectorJobConfig(Direction.TO, toJob);
+ // TODO(Abe): Should we actually have 2 different Driver Connection config objects?
+ jobRequest.setFrameworkLinkConfig(Direction.FROM, fromDriverConnection);
+ jobRequest.setFrameworkLinkConfig(Direction.TO, toDriverConnection);
+ jobRequest.setFrameworkJobConfig(frameworkJob);
+ jobRequest.setJobName(job.getName());
+ jobRequest.setJobId(job.getPersistenceId());
+ jobRequest.setNotificationUrl(notificationBaseUrl + jobId);
+ Class<? extends IntermediateDataFormat<?>> dataFormatClass =
+ fromConnector.getIntermediateDataFormat();
+ jobRequest.setIntermediateDataFormat(fromConnector.getIntermediateDataFormat());
+
+
+ jobRequest.setFrom(fromConnector.getFrom());
+ jobRequest.setTo(toConnector.getTo());
+
+ addStandardJars(jobRequest);
+ addConnectorJars(jobRequest, fromConnector, toConnector, dataFormatClass);
+ addConnectorInitializerJars(jobRequest, Direction.FROM);
+ addConnectorInitializerJars(jobRequest, Direction.TO);
+
+ Schema fromSchema = getSchemaFromConnector(jobRequest, Direction.FROM);
+ Schema toSchema = getSchemaFromConnector(jobRequest, Direction.TO);
+
+ // TODO(Gwen): Need better logic here once the Schema refactor: SQOOP-1378
+ if (fromSchema != null) {
+ jobRequest.getSummary().setFromSchema(fromSchema);
+ }
+ else {
+ jobRequest.getSummary().setFromSchema(toSchema);
+ }
+ LOG.debug("Using entities: " + jobRequest.getFrom() + ", " + jobRequest.getTo());
+ return jobRequest;
+ }
+
+ private void addConnectorJars(JobRequest jobRequest, SqoopConnector fromConnector,
+ SqoopConnector toConnector, Class<? extends IntermediateDataFormat<?>> dataFormatClass) {
+ jobRequest.addJarForClass(fromConnector.getClass());
+ jobRequest.addJarForClass(toConnector.getClass());
+ jobRequest.addJarForClass(dataFormatClass);
+ }
+
+ private void addStandardJars(JobRequest jobRequest) {
+ // Let's register all important jars
+ // sqoop-common
+ jobRequest.addJarForClass(MapContext.class);
+ // sqoop-core
+ jobRequest.addJarForClass(Driver.class);
+ // sqoop-spi
+ jobRequest.addJarForClass(SqoopConnector.class);
+ // Execution engine jar
+ jobRequest.addJarForClass(executionEngine.getClass());
+ // Extra libraries that Sqoop code requires
+ jobRequest.addJarForClass(JSONValue.class);
+ }
+
+ MSubmission createJobSubmission(HttpEventContext ctx, long jobId) {
+ MSubmission summary = new MSubmission(jobId);
+ summary.setCreationUser(ctx.getUsername());
+ summary.setLastUpdateUser(ctx.getUsername());
+ return summary;
+ }
+
+ SqoopConnector getConnector(long connnectorId) {
+ return ConnectorManager.getInstance().getConnector(connnectorId);
+ }
+
+ void validateSupportedDirection(SqoopConnector connector, Direction direction) {
+ // Make sure that connector supports the given direction
+ if (!connector.getSupportedDirections().contains(direction)) {
+ throw new SqoopException(DriverError.DRIVER_0011, "Connector: "
+ + connector.getClass().getCanonicalName());
+ }
+ }
+
+ MLink getLink(long linkId) {
+ MLink link = RepositoryManager.getInstance().getRepository()
+ .findLink(linkId);
+ if (!link.getEnabled()) {
+ throw new SqoopException(DriverError.DRIVER_0010, "Connection id: "
+ + link.getPersistenceId());
+ }
+ return link;
+ }
+
+ MJob getJob(long jobId) {
+ MJob job = RepositoryManager.getInstance().getRepository().findJob(jobId);
+ if (job == null) {
+ throw new SqoopException(DriverError.DRIVER_0004, "Unknown job id: " + jobId);
+ }
+
+ if (!job.getEnabled()) {
+ throw new SqoopException(DriverError.DRIVER_0009, "Job id: " + job.getPersistenceId());
+ }
+ return job;
+ }
+
+ private Schema getSchemaFromConnector(JobRequest jobRequest, Direction direction) {
+
+ Initializer initializer = getConnectorInitializer(jobRequest, direction);
+
+ // Initializer context
+ InitializerContext initializerContext = getConnectorInitializerContext(jobRequest, direction);
+
+ // Initialize submission from the connector perspective
+ initializer.initialize(initializerContext, jobRequest.getConnectorLinkConfig(direction),
+ jobRequest.getConnectorJobConfig(direction));
+
+ // TODO(Abe): Alter behavior of Schema here.
+ return initializer.getSchema(initializerContext,
+ jobRequest.getConnectorLinkConfig(direction),
+ jobRequest.getConnectorJobConfig(direction));
+ }
+
+ private void addConnectorInitializerJars(JobRequest jobRequest, Direction direction) {
+
+ Initializer initializer = getConnectorInitializer(jobRequest, direction);
+ InitializerContext initializerContext = getConnectorInitializerContext(jobRequest, direction);
+ // Add job specific jars to
+ jobRequest.addJars(initializer.getJars(initializerContext,
+ jobRequest.getConnectorLinkConfig(direction),
+ jobRequest.getConnectorJobConfig(direction)));
+ }
+
+ private Initializer getConnectorInitializer(JobRequest jobRequest, Direction direction) {
+ Transferable transferable = direction.equals(Direction.FROM) ? jobRequest.getFrom() : jobRequest.getTo();
+ Class<? extends Initializer> initializerClass = transferable.getInitializer();
+ Initializer initializer = (Initializer) ClassUtils.instantiate(initializerClass);
+
+ if (initializer == null) {
+ throw new SqoopException(DriverError.DRIVER_0006,
+ "Can't create connector initializer instance: " + initializerClass.getName());
+ }
+ return initializer;
+ }
+
+ private InitializerContext getConnectorInitializerContext(JobRequest jobRequest, Direction direction) {
+ return new InitializerContext(jobRequest.getConnectorContext(direction));
+ }
+
+ void prepareJob(JobRequest request) {
+ JobConfiguration jobConfiguration = (JobConfiguration) request.getFrameworkJobConfig();
+ // We're directly moving configured number of extractors and loaders to
+ // underlying request object. In the future we might need to throttle this
+ // count based on other running jobs to meet our SLAs.
+ request.setExtractors(jobConfiguration.throttling.extractors);
+ request.setLoaders(jobConfiguration.throttling.loaders);
+
+ // Delegate rest of the job to execution engine
+ executionEngine.prepareJob(request);
+ }
+
+ /**
+ * Callback that will be called only if we failed to submit the job to the
+ * remote cluster.
+ */
+ void destroySubmission(JobRequest request) {
+ Transferable from = request.getFrom();
+ Transferable to = request.getTo();
+
+ Class<? extends Destroyer> fromDestroyerClass = from.getDestroyer();
+ Class<? extends Destroyer> toDestroyerClass = to.getDestroyer();
+ Destroyer fromDestroyer = (Destroyer) ClassUtils.instantiate(fromDestroyerClass);
+ Destroyer toDestroyer = (Destroyer) ClassUtils.instantiate(toDestroyerClass);
+
+ if (fromDestroyer == null) {
+ throw new SqoopException(DriverError.DRIVER_0006,
+ "Can't create toDestroyer instance: " + fromDestroyerClass.getName());
+ }
+
+ if (toDestroyer == null) {
+ throw new SqoopException(DriverError.DRIVER_0006,
+ "Can't create toDestroyer instance: " + toDestroyerClass.getName());
+ }
+
+ // TODO(Abe): Update context to manage multiple connectors. As well as summary.
+ DestroyerContext fromDestroyerContext = new DestroyerContext(
+ request.getConnectorContext(Direction.FROM), false, request.getSummary()
+ .getFromSchema());
+ DestroyerContext toDestroyerContext = new DestroyerContext(
+ request.getConnectorContext(Direction.TO), false, request.getSummary()
+ .getToSchema());
+
+ // destroy submission from connector perspective
+ fromDestroyer.destroy(fromDestroyerContext, request.getConnectorLinkConfig(Direction.FROM),
+ request.getConnectorJobConfig(Direction.FROM));
+ toDestroyer.destroy(toDestroyerContext, request.getConnectorLinkConfig(Direction.TO),
+ request.getConnectorJobConfig(Direction.TO));
+ }
+
+ public MSubmission stop(long jobId, HttpEventContext ctx) {
+
+ Repository repository = RepositoryManager.getInstance().getRepository();
+ MSubmission mSubmission = repository.findSubmissionLastForJob(jobId);
+
+ if (mSubmission == null || !mSubmission.getStatus().isRunning()) {
+ throw new SqoopException(DriverError.DRIVER_0003, "Job with id " + jobId
+ + " is not running");
+ }
+ submissionEngine.stop(mSubmission.getExternalId());
+
+ mSubmission.setLastUpdateUser(ctx.getUsername());
+
+ // Fetch new information to verify that the stop command has actually worked
+ update(mSubmission);
+
+ // Return updated structure
+ return mSubmission;
+ }
+
+ public MSubmission status(long jobId) {
+ Repository repository = RepositoryManager.getInstance().getRepository();
+ MSubmission mSubmission = repository.findSubmissionLastForJob(jobId);
+
+ if (mSubmission == null) {
+ return new MSubmission(jobId, new Date(), SubmissionStatus.NEVER_EXECUTED);
+ }
+
+ // If the submission is in running state, let's update it
+ if (mSubmission.getStatus().isRunning()) {
+ update(mSubmission);
+ }
+
+ return mSubmission;
+ }
+
+ private void update(MSubmission submission) {
+ double progress = -1;
+ Counters counters = null;
+ String externalId = submission.getExternalId();
+ SubmissionStatus newStatus = submissionEngine.status(externalId);
+ String externalLink = submissionEngine.externalLink(externalId);
+
+ if (newStatus.isRunning()) {
+ progress = submissionEngine.progress(externalId);
+ } else {
+ counters = submissionEngine.counters(externalId);
+ }
+
+ submission.setStatus(newStatus);
+ submission.setProgress(progress);
+ submission.setCounters(counters);
+ submission.setExternalLink(externalLink);
+ submission.setLastUpdateDate(new Date());
+
+ RepositoryManager.getInstance().getRepository()
+ .updateSubmission(submission);
+ }
+
+ @Override
+ public synchronized void configurationChanged() {
+ LOG.info("Begin submission engine manager reconfiguring");
+ MapContext newContext = SqoopConfiguration.getInstance().getContext();
+ MapContext oldContext = SqoopConfiguration.getInstance().getOldContext();
+
+ String newSubmissionEngineClassName = newContext
+ .getString(DriverConstants.SYSCFG_SUBMISSION_ENGINE);
+ if (newSubmissionEngineClassName == null
+ || newSubmissionEngineClassName.trim().length() == 0) {
+ throw new SqoopException(DriverError.DRIVER_0001,
+ newSubmissionEngineClassName);
+ }
+
+ String oldSubmissionEngineClassName = oldContext
+ .getString(DriverConstants.SYSCFG_SUBMISSION_ENGINE);
+ if (!newSubmissionEngineClassName.equals(oldSubmissionEngineClassName)) {
+ LOG.warn("Submission engine cannot be replaced at the runtime. " +
+ "You might need to restart the server.");
+ }
+
+ String newExecutionEngineClassName = newContext
+ .getString(DriverConstants.SYSCFG_EXECUTION_ENGINE);
+ if (newExecutionEngineClassName == null
+ || newExecutionEngineClassName.trim().length() == 0) {
+ throw new SqoopException(DriverError.DRIVER_0007,
+ newExecutionEngineClassName);
+ }
+
+ String oldExecutionEngineClassName = oldContext
+ .getString(DriverConstants.SYSCFG_EXECUTION_ENGINE);
+ if (!newExecutionEngineClassName.equals(oldExecutionEngineClassName)) {
+ LOG.warn("Execution engine cannot be replaced at the runtime. " +
+ "You might need to restart the server.");
+ }
+
+ // Set up worker threads
+ purgeThreshold = newContext.getLong(
+ DriverConstants.SYSCFG_SUBMISSION_PURGE_THRESHOLD,
+ DEFAULT_PURGE_THRESHOLD
+ );
+ purgeSleep = newContext.getLong(
+ DriverConstants.SYSCFG_SUBMISSION_PURGE_SLEEP,
+ DEFAULT_PURGE_SLEEP
+ );
+ purgeThread.interrupt();
+
+ updateSleep = newContext.getLong(
+ DriverConstants.SYSCFG_SUBMISSION_UPDATE_SLEEP,
+ DEFAULT_UPDATE_SLEEP
+ );
+ updateThread.interrupt();
+
+ LOG.info("Submission engine manager reconfigured.");
+ }
+
+ private class PurgeThread extends Thread {
+ public PurgeThread() {
+ super("PurgeThread");
+ }
+
+ public void run() {
+ LOG.info("Starting submission manager purge thread");
+
+ while (running) {
+ try {
+ LOG.info("Purging old submissions");
+ Date threshold = new Date((new Date()).getTime() - purgeThreshold);
+ RepositoryManager.getInstance().getRepository()
+ .purgeSubmissions(threshold);
+ Thread.sleep(purgeSleep);
+ } catch (InterruptedException e) {
+ LOG.debug("Purge thread interrupted", e);
+ }
+ }
+
+ LOG.info("Ending submission manager purge thread");
+ }
+ }
+
+ private class UpdateThread extends Thread {
+ public UpdateThread() {
+ super("UpdateThread");
+ }
+
+ public void run() {
+ LOG.info("Starting submission manager update thread");
+
+ while (running) {
+ try {
+ LOG.debug("Updating running submissions");
+
+ // Let's get all running submissions from repository to check them out
+ List<MSubmission> unfinishedSubmissions =
+ RepositoryManager.getInstance().getRepository()
+ .findSubmissionsUnfinished();
+
+ for (MSubmission submission : unfinishedSubmissions) {
+ update(submission);
+ }
+
+ Thread.sleep(updateSleep);
+ } catch (InterruptedException e) {
+ LOG.debug("Purge thread interrupted", e);
+ }
+ }
+
+ LOG.info("Ending submission manager update thread");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
new file mode 100644
index 0000000..63e1e49
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
@@ -0,0 +1,356 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.DirectionError;
+import org.apache.sqoop.common.MutableMapContext;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.idf.IntermediateDataFormat;
+import org.apache.sqoop.connector.spi.SqoopConnector;
+import org.apache.sqoop.job.etl.Transferable;
+import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.utils.ClassUtils;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Submission details class is used when creating new submission and contains
+ * all information that we need to create a new submission (including mappers,
+ * reducers, ...).
+ */
+public class JobRequest {
+
+ /**
+ * Submission summary
+ */
+ MSubmission summary;
+
+ /**
+ * Original job name
+ */
+ String jobName;
+
+ /**
+ * Associated job (from metadata perspective) id
+ */
+ long jobId;
+
+ /**
+ * Connector instances associated with this submission request
+ */
+ SqoopConnector fromConnector;
+ SqoopConnector toConnector;
+
+ /**
+ * List of required local jars for the job
+ */
+ List<String> jars;
+
+ /**
+ * From entity
+ */
+ Transferable from;
+
+ /**
+ * To entity
+ */
+ Transferable to;
+
+ /**
+ * All configuration objects
+ */
+ Object fromConnectorLinkConfig;
+ Object toConnectorLinkConfig;
+ Object fromConnectorJobConfig;
+ Object toConnectorJobConfig;
+ Object fromFrameworkLinkConfig;
+ Object toFrameworkLinkConfig;
+ Object frameworkJobConfig;
+
+ /**
+ * Connector context (submission specific configuration)
+ */
+ MutableMapContext fromConnectorContext;
+ MutableMapContext toConnectorContext;
+
+ /**
+ * Framework context (submission specific configuration)
+ */
+ MutableMapContext driverContext;
+
+ /**
+ * Optional notification URL for job progress
+ */
+ String notificationUrl;
+
+ /**
+ * Number of extractors
+ */
+ Integer extractors;
+
+ /**
+ * Number of loaders
+ */
+ Integer loaders;
+
+ /**
+ * The intermediate data format this submission should use.
+ */
+ Class<? extends IntermediateDataFormat> intermediateDataFormat;
+
+ public JobRequest() {
+ this.jars = new LinkedList<String>();
+ this.fromConnectorContext = new MutableMapContext();
+ this.toConnectorContext = new MutableMapContext();
+ this.driverContext = new MutableMapContext();
+ this.fromConnector = null;
+ this.toConnector = null;
+ this.fromConnectorLinkConfig = null;
+ this.toConnectorLinkConfig = null;
+ this.fromConnectorJobConfig = null;
+ this.toConnectorJobConfig = null;
+ this.fromFrameworkLinkConfig = null;
+ this.toFrameworkLinkConfig = null;
+ }
+
+ public MSubmission getSummary() {
+ return summary;
+ }
+
+ public void setSummary(MSubmission summary) {
+ this.summary = summary;
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ public void setJobName(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public long getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(long jobId) {
+ this.jobId = jobId;
+ }
+
+ public SqoopConnector getConnector(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromConnector;
+
+ case TO:
+ return toConnector;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public void setConnector(Direction type, SqoopConnector connector) {
+ switch(type) {
+ case FROM:
+ fromConnector = connector;
+ break;
+
+ case TO:
+ toConnector = connector;
+ break;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public List<String> getJars() {
+ return jars;
+ }
+
+ public void addJar(String jar) {
+ if(!jars.contains(jar)) {
+ jars.add(jar);
+ }
+ }
+
+ public void addJarForClass(Class klass) {
+ addJar(ClassUtils.jarForClass(klass));
+ }
+
+ public void addJars(List<String> jars) {
+ for(String j : jars) {
+ addJar(j);
+ }
+ }
+
+ public Transferable getFrom() {
+ return from;
+ }
+
+ public void setFrom(Transferable from) {
+ this.from = from;
+ }
+
+ public Transferable getTo() {
+ return to;
+ }
+
+ public void setTo(Transferable to) {
+ this.to = to;
+ }
+
+ public Object getConnectorLinkConfig(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromConnectorLinkConfig;
+
+ case TO:
+ return toConnectorLinkConfig;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public void setConnectorLinkConfig(Direction type, Object config) {
+ switch(type) {
+ case FROM:
+ fromConnectorLinkConfig = config;
+ break;
+ case TO:
+ toConnectorLinkConfig = config;
+ break;
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public Object getConnectorJobConfig(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromConnectorJobConfig;
+
+ case TO:
+ return toConnectorJobConfig;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public void setConnectorJobConfig(Direction type, Object config) {
+ switch(type) {
+ case FROM:
+ fromConnectorJobConfig = config;
+ break;
+ case TO:
+ toConnectorJobConfig = config;
+ break;
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public Object getFrameworkLinkConfig(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromFrameworkLinkConfig;
+
+ case TO:
+ return toFrameworkLinkConfig;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public void setFrameworkLinkConfig(Direction type, Object config) {
+ switch(type) {
+ case FROM:
+ fromFrameworkLinkConfig = config;
+ break;
+ case TO:
+ toFrameworkLinkConfig = config;
+ break;
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public Object getFrameworkJobConfig() {
+ return frameworkJobConfig;
+ }
+
+ public void setFrameworkJobConfig(Object config) {
+ frameworkJobConfig = config;
+ }
+
+ public MutableMapContext getConnectorContext(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromConnectorContext;
+
+ case TO:
+ return toConnectorContext;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public MutableMapContext getDriverContext() {
+ return driverContext;
+ }
+
+ public String getNotificationUrl() {
+ return notificationUrl;
+ }
+
+ public void setNotificationUrl(String url) {
+ this.notificationUrl = url;
+ }
+
+ public Integer getExtractors() {
+ return extractors;
+ }
+
+ public void setExtractors(Integer extractors) {
+ this.extractors = extractors;
+ }
+
+ public Integer getLoaders() {
+ return loaders;
+ }
+
+ public void setLoaders(Integer loaders) {
+ this.loaders = loaders;
+ }
+
+ public Class<? extends IntermediateDataFormat> getIntermediateDataFormat() {
+ return intermediateDataFormat;
+ }
+
+ public void setIntermediateDataFormat(Class<? extends IntermediateDataFormat> intermediateDataFormat) {
+ this.intermediateDataFormat = intermediateDataFormat;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/SubmissionEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/SubmissionEngine.java b/core/src/main/java/org/apache/sqoop/driver/SubmissionEngine.java
new file mode 100644
index 0000000..3a32e9f
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/SubmissionEngine.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.common.MapContext;
+import org.apache.sqoop.submission.counter.Counters;
+import org.apache.sqoop.submission.SubmissionStatus;
+
+/**
+ * Submission engine is responsible in conveying the information about the
+ * job instances (submissions) to remote (hadoop) cluster.
+ */
+public abstract class SubmissionEngine {
+
+ /**
+ * Initialize submission engine
+ *
+ * @param context Configuration context
+ * @param prefix Submission engine prefix
+ */
+ public void initialize(MapContext context, String prefix) {
+ }
+
+ /**
+ * Destroy submission engine when stopping server
+ */
+ public void destroy() {
+ }
+
+ /**
+ * Callback to verify that configured submission engine and execution engine
+ * are compatible.
+ *
+ * @param executionEngineClass Configured execution class.
+ * @return True if such execution engine is supported
+ */
+ public abstract boolean isExecutionEngineSupported(Class<?> executionEngineClass);
+
+ /**
+ * Submit new job to remote (hadoop) cluster. This method *must* fill
+ * submission.getSummary.setExternalId(), otherwise Sqoop won't
+ * be able to track progress on this job!
+ *
+ * @return Return true if we were able to submit job to remote cluster.
+ */
+ public abstract boolean submit(JobRequest submission);
+
+ /**
+ * Hard stop for given submission.
+ *
+ * @param submissionId Submission internal id.
+ */
+ public abstract void stop(String submissionId);
+
+ /**
+ * Return status of given submission.
+ *
+ * @param submissionId Submission internal id.
+ * @return Current submission status.
+ */
+ public abstract SubmissionStatus status(String submissionId);
+
+ /**
+ * Return submission progress.
+ *
+ * Expected is number from interval <0, 1> denoting how far the processing
+ * has gone or -1 in case that this submission engine do not supports
+ * progress reporting.
+ *
+ * @param submissionId Submission internal id.
+ * @return {-1} union <0, 1>
+ */
+ public double progress(String submissionId) {
+ return -1;
+ }
+
+ /**
+ * Return statistics for given submission id.
+ *
+ * Sqoop will call counters only for submission in state SUCCEEDED,
+ * it's consider exceptional state to call this method for other states.
+ *
+ * @param submissionId Submission internal id.
+ * @return Submission statistics
+ */
+ public Counters counters(String submissionId) {
+ return null;
+ }
+
+ /**
+ * Return link to external web page with given submission.
+ *
+ * @param submissionId Submission internal id.
+ * @return Null in case that external page is not supported or available or
+ * HTTP link to given submission.
+ */
+ public String externalLink(String submissionId) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
new file mode 100644
index 0000000..908a4eb
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.ConfigurationClass;
+import org.apache.sqoop.model.Form;
+
+/**
+ * Representing the core job configuration
+ */
+@ConfigurationClass
+public class JobConfiguration {
+ @Form
+ public ThrottlingForm throttling;
+
+ public JobConfiguration() {
+ throttling = new ThrottlingForm();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
new file mode 100644
index 0000000..3202844
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.ConfigurationClass;
+
+/**
+ * Representing the core link configuration
+ */
+@ConfigurationClass
+public class LinkConfiguration {
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
new file mode 100644
index 0000000..e73007e
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Input;
+
+/**
+ * Form to set up number of loaders and extractors
+ */
+@FormClass
+public class ThrottlingForm {
+
+ @Input public Integer extractors;
+
+ @Input public Integer loaders;
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java b/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java
deleted file mode 100644
index 75b570d..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.common.ImmutableContext;
-
-/**
- * Execution engine drives execution of sqoop job. It's responsible
- * for executing all defined steps in the import/export workflow.
- * A successful job execution will be recorded in the job submission entity
- */
-public abstract class ExecutionEngine {
-
- /**
- * Initialize execution engine
- *
- * @param context Configuration context
- * @parma prefix Execution engine prefix
- */
- public void initialize(ImmutableContext context, String prefix) {
- }
-
- /**
- * Destroy execution engine when stopping server
- */
- public void destroy() {
- }
-
- /**
- * Return new JobRequest class or any subclass if it's needed by
- * execution and submission engine combination.
- *
- * @return new JobRequestobject
- */
- public JobRequest createJobRequest() {
- return new JobRequest();
- }
-
- /**
- * Prepare given job request.
- *
- * @param request JobRequest
- */
- public abstract void prepareJob(JobRequest request);
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/FrameworkConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/FrameworkConstants.java b/core/src/main/java/org/apache/sqoop/framework/FrameworkConstants.java
deleted file mode 100644
index 4293dce..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/FrameworkConstants.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.core.ConfigurationConstants;
-
-/**
- * Constants that are used in framework module.
- */
-public final class FrameworkConstants {
-
- // Sqoop configuration constants
-
- public static final String PREFIX_SUBMISSION_CONFIG =
- ConfigurationConstants.PREFIX_GLOBAL_CONFIG + "submission.";
-
- public static final String PREFIX_EXECUTION_CONFIG =
- ConfigurationConstants.PREFIX_GLOBAL_CONFIG + "execution.";
-
- public static final String SYSCFG_SUBMISSION_ENGINE =
- PREFIX_SUBMISSION_CONFIG + "engine";
-
- public static final String PREFIX_SUBMISSION_ENGINE_CONFIG =
- SYSCFG_SUBMISSION_ENGINE + ".";
-
- public static final String PREFIX_SUBMISSION_PURGE_CONFIG =
- PREFIX_SUBMISSION_CONFIG + "purge.";
-
- public static final String SYSCFG_SUBMISSION_PURGE_THRESHOLD =
- PREFIX_SUBMISSION_PURGE_CONFIG + "threshold";
-
- public static final String SYSCFG_SUBMISSION_PURGE_SLEEP =
- PREFIX_SUBMISSION_PURGE_CONFIG + "sleep";
-
- public static final String PREFIX_SUBMISSION_UPDATE_CONFIG =
- PREFIX_SUBMISSION_CONFIG + "update.";
-
- public static final String SYSCFG_SUBMISSION_UPDATE_SLEEP =
- PREFIX_SUBMISSION_UPDATE_CONFIG + "sleep";
-
- public static final String SYSCFG_EXECUTION_ENGINE =
- PREFIX_EXECUTION_CONFIG + "engine";
-
- public static final String PREFIX_EXECUTION_ENGINE_CONFIG =
- SYSCFG_EXECUTION_ENGINE + ".";
-
- // Bundle names
-
- public static final String RESOURCE_BUNDLE_NAME = "framework-resources";
-
- private FrameworkConstants() {
- // Instantiation of this class is prohibited
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/FrameworkError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/FrameworkError.java b/core/src/main/java/org/apache/sqoop/framework/FrameworkError.java
deleted file mode 100644
index 8ecb197..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/FrameworkError.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.common.ErrorCode;
-
-/**
- *
- */
-public enum FrameworkError implements ErrorCode {
-
- FRAMEWORK_0000("Metadata are not registered in repository"),
-
- FRAMEWORK_0001("Invalid submission engine"),
-
- FRAMEWORK_0002("Given job is already running"),
-
- FRAMEWORK_0003("Given job is not running"),
-
- FRAMEWORK_0004("Unknown job id"),
-
- FRAMEWORK_0005("Unsupported job type"),
-
- FRAMEWORK_0006("Can't bootstrap job"),
-
- FRAMEWORK_0007("Invalid execution engine"),
-
- FRAMEWORK_0008("Invalid combination of submission and execution engines"),
-
- FRAMEWORK_0009("Job has been disabled. Cannot submit this job."),
-
- FRAMEWORK_0010("Connection for this job has been disabled. Cannot submit this job."),
-
- FRAMEWORK_0011("Connector does not support direction. Cannot submit this job."),
-
- ;
-
- private final String message;
-
- private FrameworkError(String message) {
- this.message = message;
- }
-
- public String getCode() {
- return name();
- }
-
- public String getMessage() {
- return message;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/FrameworkManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/FrameworkManager.java b/core/src/main/java/org/apache/sqoop/framework/FrameworkManager.java
deleted file mode 100644
index 81e1147..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/FrameworkManager.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
-import org.apache.sqoop.core.ConfigurationConstants;
-import org.apache.sqoop.core.Reconfigurable;
-import org.apache.sqoop.core.SqoopConfiguration;
-import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
-import org.apache.sqoop.framework.configuration.ConnectionConfiguration;
-import org.apache.sqoop.framework.configuration.JobConfiguration;
-import org.apache.sqoop.model.*;
-import org.apache.sqoop.repository.RepositoryManager;
-import org.apache.sqoop.validation.Validator;
-
-import java.util.Locale;
-import java.util.ResourceBundle;
-
-/**
- * Manager for Sqoop framework itself.
- *
- * All Sqoop internals are handled in this class:
- * * Submission engine
- * * Execution engine
- * * Framework metadata
- *
- * Current implementation of entire submission engine is using repository
- * for keeping track of running submissions. Thus, server might be restarted at
- * any time without any affect on running jobs. This approach however might not
- * be the fastest way and we might want to introduce internal structures for
- * running jobs in case that this approach will be too slow.
- */
-public class FrameworkManager implements Reconfigurable {
-
- /**
- * Logger object.
- */
- private static final Logger LOG = Logger.getLogger(FrameworkManager.class);
-
- /**
- * Private instance to singleton of this class.
- */
- private static FrameworkManager instance;
-
- /**
- * Create default object by default.
- *
- * Every Sqoop server application needs one so this should not be performance issue.
- */
- static {
- instance = new FrameworkManager();
- }
-
- /**
- * Return current instance.
- *
- * @return Current instance
- */
- public static FrameworkManager getInstance() {
- return instance;
- }
-
- /**
- * Allows to set instance in case that it's need.
- *
- * This method should not be normally used as the default instance should be sufficient. One target
- * user use case for this method are unit tests.
- *
- * @param newInstance New instance
- */
- public static void setInstance(FrameworkManager newInstance) {
- instance = newInstance;
- }
-
- /**
- * Framework metadata structures in MForm format
- */
- private MFramework mFramework;
-
- /**
- * Validator instance
- */
- private final Validator validator;
-
- /**
- * Upgrader instance
- */
- private final MetadataUpgrader upgrader;
-
- /**
- * Default framework auto upgrade option value
- */
- private static final boolean DEFAULT_AUTO_UPGRADE = false;
-
- public static final String CURRENT_FRAMEWORK_VERSION = "1";
-
- public Class getJobConfigurationClass() {
- return JobConfiguration.class;
- }
-
- public Class getConnectionConfigurationClass() {
- return ConnectionConfiguration.class;
- }
-
- public FrameworkManager() {
- MConnectionForms connectionForms = new MConnectionForms(
- FormUtils.toForms(getConnectionConfigurationClass())
- );
- mFramework = new MFramework(connectionForms, new MJobForms(FormUtils.toForms(getJobConfigurationClass())),
- CURRENT_FRAMEWORK_VERSION);
-
- // Build validator
- validator = new FrameworkValidator();
-
- // Build upgrader
- upgrader = new FrameworkMetadataUpgrader();
- }
-
- public synchronized void initialize() {
- initialize(SqoopConfiguration.getInstance().getContext().getBoolean(ConfigurationConstants.FRAMEWORK_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE));
- }
-
- public synchronized void initialize(boolean autoUpgrade) {
- LOG.trace("Begin submission engine manager initialization");
-
- // Register framework metadata in repository
- mFramework = RepositoryManager.getInstance().getRepository().registerFramework(mFramework, autoUpgrade);
-
- SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this));
-
- LOG.info("Submission manager initialized: OK");
- }
-
- public synchronized void destroy() {
- LOG.trace("Begin submission engine manager destroy");
- }
-
- public Validator getValidator() {
- return validator;
- }
-
- public MetadataUpgrader getMetadataUpgrader() {
- return upgrader;
- }
-
- public MFramework getFramework() {
- return mFramework;
- }
-
- public ResourceBundle getBundle(Locale locale) {
- return ResourceBundle.getBundle(
- FrameworkConstants.RESOURCE_BUNDLE_NAME, locale);
- }
-
- @Override
- public void configurationChanged() {
- LOG.info("Begin framework manager reconfiguring");
- // If there are configuration options for FrameworkManager,
- // implement the reconfiguration procedure right here.
- LOG.info("Framework manager reconfigured");
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java b/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java
deleted file mode 100644
index 2437fa6..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJobForms;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class FrameworkMetadataUpgrader extends MetadataUpgrader{
-
- private static final Logger LOG = Logger.getLogger(FrameworkMetadataUpgrader.class);
-
- @Override
- public void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
- }
-
- @Override
- public void upgrade(MJobForms original, MJobForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
-
- }
-
- @SuppressWarnings("unchecked")
- private void doUpgrade(List<MForm> original, List<MForm> target) {
- // Easier to find the form in the original forms list if we use a map.
- // Since the constructor of MJobForms takes a list,
- // index is not guaranteed to be the same, so we need to look for
- // equivalence
- Map<String, MForm> formMap = new HashMap<String, MForm>();
- for (MForm form : original) {
- formMap.put(form.getName(), form);
- }
- for (MForm form : target) {
- List<MInput<?>> inputs = form.getInputs();
- MForm originalForm = formMap.get(form.getName());
- if(originalForm == null) {
- LOG.warn("Form: " + form.getName() + " not present in old " +
- "framework metadata. So it will not be transferred by the upgrader.");
- continue;
- }
-
- for (MInput input : inputs) {
- try {
- MInput originalInput = originalForm.getInput(input.getName());
- input.setValue(originalInput.getValue());
- } catch (SqoopException ex) {
- LOG.warn("Input: " + input.getName() + " not present in old " +
- "framework metadata. So it will not be transferred by the upgrader.");
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/FrameworkValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/FrameworkValidator.java b/core/src/main/java/org/apache/sqoop/framework/FrameworkValidator.java
deleted file mode 100644
index 46257f2..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/FrameworkValidator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.framework.configuration.ConnectionConfiguration;
-import org.apache.sqoop.framework.configuration.JobConfiguration;
-import org.apache.sqoop.framework.configuration.ThrottlingForm;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
-import org.apache.sqoop.validation.Validator;
-
-public class FrameworkValidator extends Validator {
- @Override
- public Validation validateConnection(Object connectionConfiguration) {
- Validation validation = new Validation(ConnectionConfiguration.class);
- // No validation on connection object
- return validation;
- }
-
- @Override
- public Validation validateJob(Object jobConfiguration) {
- Validation validation = new Validation(JobConfiguration.class);
- JobConfiguration conf = (JobConfiguration)jobConfiguration;
- validateThrottlingForm(validation,conf.throttling);
-
- return validation;
- };
-
- private void validateThrottlingForm(Validation validation, ThrottlingForm throttling) {
- if(throttling.extractors != null && throttling.extractors < 1) {
- validation.addMessage(Status.UNACCEPTABLE, "throttling", "extractors", "You need to specify more than one extractor");
- }
-
- if(throttling.loaders != null && throttling.loaders < 1) {
- validation.addMessage(Status.UNACCEPTABLE, "throttling", "loaders", "You need to specify more than one loader");
- }
- }
-
-}
[09/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/DeleteConnectionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/DeleteConnectionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/DeleteConnectionFunction.java
deleted file mode 100644
index 1eb7e51..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/DeleteConnectionFunction.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.validation.Status;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class DeleteConnectionFunction extends SqoopFunction {
- @SuppressWarnings("static-access")
- public DeleteConnectionFunction() {
- this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_PROMPT_CONN_ID))
- .withLongOpt(Constants.OPT_XID)
- .isRequired()
- .hasArg()
- .create('x'));
- }
-
- @Override
- public Object executeFunction(CommandLine line, boolean isInteractive) {
- client.deleteConnection(getLong(line, Constants.OPT_XID));
- return Status.FINE;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/DeleteJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/DeleteJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/DeleteJobFunction.java
index 5d48c91..da26153 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/DeleteJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/DeleteJobFunction.java
@@ -37,7 +37,7 @@ public class DeleteJobFunction extends SqoopFunction {
.withLongOpt(Constants.OPT_JID)
.isRequired()
.hasArg()
- .create('j'));
+ .create(Constants.OPT_JID_CHAR));
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/DeleteLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/DeleteLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/DeleteLinkFunction.java
new file mode 100644
index 0000000..c81917e
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/DeleteLinkFunction.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.validation.Status;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class DeleteLinkFunction extends SqoopFunction {
+ @SuppressWarnings("static-access")
+ public DeleteLinkFunction() {
+ this.addOption(OptionBuilder
+ .withDescription(resourceString(Constants.RES_PROMPT_LINK_ID))
+ .withLongOpt(Constants.OPT_LID)
+ .isRequired()
+ .hasArg()
+ .create(Constants.OPT_LID_CHAR));
+ }
+
+ @Override
+ public Object executeFunction(CommandLine line, boolean isInteractive) {
+ client.deleteLink(getLong(line, Constants.OPT_LID));
+ return Status.FINE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/DisableCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/DisableCommand.java b/shell/src/main/java/org/apache/sqoop/shell/DisableCommand.java
index fa3263f..03aa922 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/DisableCommand.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/DisableCommand.java
@@ -31,7 +31,7 @@ public class DisableCommand extends SqoopCommand {
Constants.CMD_DISABLE,
Constants.CMD_DISABLE_SC,
ImmutableMap.of(
- Constants.FN_CONNECTION, DisableConnectionFunction.class,
+ Constants.FN_LINK, DisableLinkFunction.class,
Constants.FN_JOB, DisableJobFunction.class
)
);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/DisableConnectionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/DisableConnectionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/DisableConnectionFunction.java
deleted file mode 100644
index 816ff75..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/DisableConnectionFunction.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.validation.Status;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-
-/**
- * Handles enabling of a connection object
- */
-@SuppressWarnings("serial")
-public class DisableConnectionFunction extends SqoopFunction {
- @SuppressWarnings("static-access")
- public DisableConnectionFunction() {
- this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_PROMPT_CONN_ID))
- .withLongOpt(Constants.OPT_XID)
- .isRequired()
- .hasArg()
- .create('x'));
- }
-
- @Override
- public Object executeFunction(CommandLine line, boolean isInteractive) {
- client.enableConnection(getLong(line, Constants.OPT_XID), false);
- return Status.FINE;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/DisableLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/DisableLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/DisableLinkFunction.java
new file mode 100644
index 0000000..6b85292
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/DisableLinkFunction.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.validation.Status;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+
+/**
+ * Handles enabling of a connection object
+ */
+@SuppressWarnings("serial")
+public class DisableLinkFunction extends SqoopFunction {
+ @SuppressWarnings("static-access")
+ public DisableLinkFunction() {
+ this.addOption(OptionBuilder
+ .withDescription(resourceString(Constants.RES_PROMPT_LINK_ID))
+ .withLongOpt(Constants.OPT_LID)
+ .isRequired()
+ .hasArg()
+ .create(Constants.OPT_LID_CHAR));
+ }
+
+ @Override
+ public Object executeFunction(CommandLine line, boolean isInteractive) {
+ client.enableLink(getLong(line, Constants.OPT_LID), false);
+ return Status.FINE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/EnableCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/EnableCommand.java b/shell/src/main/java/org/apache/sqoop/shell/EnableCommand.java
index b48647b..42cc9fc 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/EnableCommand.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/EnableCommand.java
@@ -31,7 +31,7 @@ public class EnableCommand extends SqoopCommand {
Constants.CMD_ENABLE,
Constants.CMD_ENABLE_SC,
ImmutableMap.of(
- Constants.FN_CONNECTION, EnableConnectionFunction.class,
+ Constants.FN_LINK, EnableLinkFunction.class,
Constants.FN_JOB, EnableJobFunction.class
)
);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/EnableConnectionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/EnableConnectionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/EnableConnectionFunction.java
deleted file mode 100644
index 174c3df..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/EnableConnectionFunction.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.validation.Status;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-
-/**
- * Handles enabling of a connection object
- */
-@SuppressWarnings("serial")
-public class EnableConnectionFunction extends SqoopFunction {
- @SuppressWarnings("static-access")
- public EnableConnectionFunction() {
- this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_PROMPT_CONN_ID))
- .withLongOpt(Constants.OPT_XID)
- .isRequired()
- .hasArg()
- .create('x'));
- }
-
- @Override
- public Object executeFunction(CommandLine line, boolean isInteractive) {
- client.enableConnection(getLong(line, Constants.OPT_XID), true);
- return Status.FINE;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/EnableLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/EnableLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/EnableLinkFunction.java
new file mode 100644
index 0000000..1dd30e8
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/EnableLinkFunction.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.validation.Status;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+
+/**
+ * Handles enabling of a connection object
+ */
+@SuppressWarnings("serial")
+public class EnableLinkFunction extends SqoopFunction {
+ @SuppressWarnings("static-access")
+ public EnableLinkFunction() {
+ this.addOption(OptionBuilder
+ .withDescription(resourceString(Constants.RES_PROMPT_LINK_ID))
+ .withLongOpt(Constants.OPT_LID)
+ .isRequired()
+ .hasArg()
+ .create(Constants.OPT_LID_CHAR));
+ }
+
+ @Override
+ public Object executeFunction(CommandLine line, boolean isInteractive) {
+ client.enableLink(getLong(line, Constants.OPT_LID), true);
+ return Status.FINE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java b/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java
index ba1d384..28bc752 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java
@@ -31,8 +31,8 @@ public class ShowCommand extends SqoopCommand {
.put(Constants.FN_SERVER, ShowServerFunction.class)
.put(Constants.FN_VERSION, ShowVersionFunction.class)
.put(Constants.FN_CONNECTOR, ShowConnectorFunction.class)
- .put(Constants.FN_FRAMEWORK, ShowFrameworkFunction.class)
- .put(Constants.FN_CONNECTION, ShowConnectionFunction.class)
+ .put(Constants.FN_DRIVER_CONFIG, ShowDriverConfigFunction.class)
+ .put(Constants.FN_LINK, ShowLinkFunction.class)
.put(Constants.FN_JOB, ShowJobFunction.class)
.put(Constants.FN_SUBMISSION, ShowSubmissionFunction.class)
.put(Constants.FN_OPTION, ShowOptionFunction.class)
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/ShowConnectionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectionFunction.java
deleted file mode 100644
index b6cf403..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectionFunction.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.TableDisplayer;
-import org.apache.sqoop.validation.Status;
-
-import java.text.DateFormat;
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormDisplayer.*;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class ShowConnectionFunction extends SqoopFunction {
- @SuppressWarnings("static-access")
- public ShowConnectionFunction() {
- this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_SHOW_PROMPT_DISPLAY_ALL_CONNS))
- .withLongOpt(Constants.OPT_ALL)
- .create(Constants.OPT_ALL_CHAR));
- this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_FROM)
- .withDescription(resourceString(Constants.RES_SHOW_PROMPT_DISPLAY_CONN_XID))
- .withLongOpt(Constants.OPT_FROM)
- .create(Constants.OPT_XID_CHAR));
- }
-
- @Override
- public Object executeFunction(CommandLine line, boolean isInteractive) {
- if (line.hasOption(Constants.OPT_ALL)) {
- showConnections();
- } else if (line.hasOption(Constants.OPT_FROM)) {
- showConnection(getLong(line, Constants.OPT_FROM));
- } else {
- showSummary();
- }
-
- return Status.FINE;
- }
-
- private void showSummary() {
- List<MConnection> connections = client.getConnections();
-
- List<String> header = new LinkedList<String>();
- header.add(resourceString(Constants.RES_TABLE_HEADER_ID));
- header.add(resourceString(Constants.RES_TABLE_HEADER_NAME));
- header.add(resourceString(Constants.RES_TABLE_HEADER_CONNECTOR));
- header.add(resourceString(Constants.RES_TABLE_HEADER_ENABLED));
-
- List<String> ids = new LinkedList<String>();
- List<String> names = new LinkedList<String>();
- List<String> connectors = new LinkedList<String>();
- List<String> availabilities = new LinkedList<String>();
-
- for(MConnection connection : connections) {
- ids.add(String.valueOf(connection.getPersistenceId()));
- names.add(connection.getName());
- connectors.add(String.valueOf(connection.getConnectorId()));
- availabilities.add(String.valueOf(connection.getEnabled()));
- }
-
- TableDisplayer.display(header, ids, names, connectors, availabilities);
- }
-
- private void showConnections() {
- List<MConnection> connections = client.getConnections();
-
- printlnResource(Constants.RES_SHOW_PROMPT_CONNS_TO_SHOW, connections.size());
-
- for (MConnection connection : connections) {
- displayConnection(connection);
- }
- }
-
- private void showConnection(Long xid) {
- MConnection connection = client.getConnection(xid);
-
- printlnResource(Constants.RES_SHOW_PROMPT_CONNS_TO_SHOW, 1);
-
- displayConnection(connection);
- }
-
- private void displayConnection(MConnection connection) {
- DateFormat formatter = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
-
- printlnResource(Constants.RES_SHOW_PROMPT_CONN_INFO,
- connection.getPersistenceId(),
- connection.getName(),
- connection.getEnabled(),
- connection.getCreationUser(),
- formatter.format(connection.getCreationDate()),
- connection.getLastUpdateUser(),
- formatter.format(connection.getLastUpdateDate())
- );
-
- long connectorId = connection.getConnectorId();
- printlnResource(Constants.RES_SHOW_PROMPT_CONN_CID_INFO, connectorId);
-
- // Display connector part
- displayForms(connection.getConnectorPart().getForms(),
- client.getResourceBundle(connectorId));
- displayForms(connection.getFrameworkPart().getForms(),
- client.getFrameworkResourceBundle());
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
index 50978ff..2ba75b4 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
@@ -115,7 +115,7 @@ public class ShowConnectorFunction extends SqoopFunction {
connector.getVersion(),
getSupportedDirections(connector)
);
- displayFormMetadataDetails(connector, client.getResourceBundle(connector.getPersistenceId()));
+ displayFormMetadataDetails(connector, client.getConnectorConfigResourceBundle(connector.getPersistenceId()));
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/ShowDriverConfigFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowDriverConfigFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowDriverConfigFunction.java
new file mode 100644
index 0000000..e6e1004
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowDriverConfigFunction.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.validation.Status;
+
+import java.util.ResourceBundle;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+import static org.apache.sqoop.shell.utils.FormDisplayer.*;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class ShowDriverConfigFunction extends SqoopFunction {
+ public ShowDriverConfigFunction() {
+ }
+
+ @Override
+ public boolean validateArgs(CommandLine line) {
+ if (line.getArgs().length != 0) {
+ printlnResource(Constants.RES_SHOW_DRIVER_CONFIG_USAGE);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public Object executeFunction(CommandLine line, boolean isInteractive) {
+ showFramework();
+ return Status.FINE;
+ }
+
+ private void showFramework() {
+ MDriverConfig framework = client.getDriverConfig();
+ ResourceBundle bundle = client.getDriverConfigBundle();
+
+ printlnResource(Constants.RES_SHOW_PROMPT_DRIVER_CONFIG_OPTS, framework.getPersistenceId());
+ displayFormMetadataDetails(framework, bundle);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/ShowFrameworkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowFrameworkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowFrameworkFunction.java
deleted file mode 100644
index 0c587b2..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowFrameworkFunction.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.sqoop.model.MFramework;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.validation.Status;
-
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormDisplayer.*;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class ShowFrameworkFunction extends SqoopFunction {
- public ShowFrameworkFunction() {
- }
-
- @Override
- public boolean validateArgs(CommandLine line) {
- if (line.getArgs().length != 0) {
- printlnResource(Constants.RES_SHOW_FRAMEWORK_USAGE);
- return false;
- }
- return true;
- }
-
- @Override
- public Object executeFunction(CommandLine line, boolean isInteractive) {
- showFramework();
- return Status.FINE;
- }
-
- private void showFramework() {
- MFramework framework = client.getFramework();
- ResourceBundle bundle = client.getFrameworkResourceBundle();
-
- printlnResource(Constants.RES_SHOW_PROMPT_FRAMEWORK_OPTS, framework.getPersistenceId());
- displayFormMetadataDetails(framework, bundle);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
index 2fcd729..e3f1f47 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
@@ -120,16 +120,16 @@ public class ShowJobFunction extends SqoopFunction {
job.getLastUpdateUser(),
formatter.format(job.getLastUpdateDate())
);
- printlnResource(Constants.RES_SHOW_PROMPT_JOB_XID_CID_INFO,
- job.getConnectionId(Direction.FROM),
+ printlnResource(Constants.RES_SHOW_PROMPT_JOB_LID_CID_INFO,
+ job.getLinkId(Direction.FROM),
job.getConnectorId(Direction.FROM));
// Display connector part
displayForms(job.getConnectorPart(Direction.FROM).getForms(),
- client.getResourceBundle(job.getConnectorId(Direction.FROM)));
+ client.getConnectorConfigResourceBundle(job.getConnectorId(Direction.FROM)));
displayForms(job.getFrameworkPart().getForms(),
- client.getFrameworkResourceBundle());
+ client.getDriverConfigBundle());
displayForms(job.getConnectorPart(Direction.TO).getForms(),
- client.getResourceBundle(job.getConnectorId(Direction.TO)));
+ client.getConnectorConfigResourceBundle(job.getConnectorId(Direction.TO)));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
new file mode 100644
index 0000000..f500b9e
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.shell.utils.TableDisplayer;
+import org.apache.sqoop.validation.Status;
+
+import java.text.DateFormat;
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+import static org.apache.sqoop.shell.utils.FormDisplayer.*;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class ShowLinkFunction extends SqoopFunction {
+ @SuppressWarnings("static-access")
+ public ShowLinkFunction() {
+ this.addOption(OptionBuilder
+ .withDescription(resourceString(Constants.RES_SHOW_PROMPT_DISPLAY_ALL_LINKS))
+ .withLongOpt(Constants.OPT_ALL)
+ .create(Constants.OPT_ALL_CHAR));
+ this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_FROM)
+ .withDescription(resourceString(Constants.RES_SHOW_PROMPT_DISPLAY_LINK_LID))
+ .withLongOpt(Constants.OPT_FROM)
+ .create(Constants.OPT_LID_CHAR));
+ }
+
+ @Override
+ public Object executeFunction(CommandLine line, boolean isInteractive) {
+ if (line.hasOption(Constants.OPT_ALL)) {
+ showLinks();
+ } else if (line.hasOption(Constants.OPT_FROM)) {
+ showLink(getLong(line, Constants.OPT_FROM));
+ } else {
+ showSummary();
+ }
+
+ return Status.FINE;
+ }
+
+ private void showSummary() {
+ List<MLink> links = client.getLinks();
+
+ List<String> header = new LinkedList<String>();
+ header.add(resourceString(Constants.RES_TABLE_HEADER_ID));
+ header.add(resourceString(Constants.RES_TABLE_HEADER_NAME));
+ header.add(resourceString(Constants.RES_TABLE_HEADER_CONNECTOR));
+ header.add(resourceString(Constants.RES_TABLE_HEADER_ENABLED));
+
+ List<String> ids = new LinkedList<String>();
+ List<String> names = new LinkedList<String>();
+ List<String> connectors = new LinkedList<String>();
+ List<String> availabilities = new LinkedList<String>();
+
+ for(MLink link : links) {
+ ids.add(String.valueOf(link.getPersistenceId()));
+ names.add(link.getName());
+ connectors.add(String.valueOf(link.getConnectorId()));
+ availabilities.add(String.valueOf(link.getEnabled()));
+ }
+
+ TableDisplayer.display(header, ids, names, connectors, availabilities);
+ }
+
+ private void showLinks() {
+ List<MLink> links = client.getLinks();
+
+ printlnResource(Constants.RES_SHOW_PROMPT_LINKS_TO_SHOW, links.size());
+
+ for (MLink link : links) {
+ displayLink(link);
+ }
+ }
+
+ private void showLink(Long xid) {
+ MLink link = client.getLink(xid);
+
+ printlnResource(Constants.RES_SHOW_PROMPT_LINKS_TO_SHOW, 1);
+
+ displayLink(link);
+ }
+
+ private void displayLink(MLink connection) {
+ DateFormat formatter = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
+
+ printlnResource(Constants.RES_SHOW_PROMPT_LINK_INFO,
+ connection.getPersistenceId(),
+ connection.getName(),
+ connection.getEnabled(),
+ connection.getCreationUser(),
+ formatter.format(connection.getCreationDate()),
+ connection.getLastUpdateUser(),
+ formatter.format(connection.getLastUpdateDate())
+ );
+
+ long connectorId = connection.getConnectorId();
+ printlnResource(Constants.RES_SHOW_PROMPT_LINK_CID_INFO, connectorId);
+
+ // Display connector part
+ displayForms(connection.getConnectorPart().getForms(),
+ client.getConnectorConfigResourceBundle(connectorId));
+ displayForms(connection.getFrameworkPart().getForms(),
+ client.getDriverConfigBundle());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/ShowVersionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowVersionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowVersionFunction.java
index 6cb28e5..4e7ad38 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowVersionFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowVersionFunction.java
@@ -21,7 +21,7 @@ import java.util.Arrays;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.client.request.VersionRequest;
+import org.apache.sqoop.client.request.VersionResourceRequest;
import org.apache.sqoop.common.VersionInfo;
import org.apache.sqoop.json.VersionBean;
import org.apache.sqoop.shell.core.Constants;
@@ -31,7 +31,7 @@ import static org.apache.sqoop.shell.ShellEnvironment.*;
@SuppressWarnings("serial")
public class ShowVersionFunction extends SqoopFunction {
- private VersionRequest versionRequest;
+ private VersionResourceRequest versionRequest;
@SuppressWarnings("static-access")
@@ -106,9 +106,9 @@ public class ShowVersionFunction extends SqoopFunction {
}
if (versionRequest == null) {
- versionRequest = new VersionRequest();
+ versionRequest = new VersionResourceRequest();
}
- VersionBean versionBean = versionRequest.doGet(getServerUrl());
+ VersionBean versionBean = versionRequest.read(getServerUrl());
if (server) {
printlnResource(Constants.RES_SHOW_PROMPT_VERSION_CLIENT_SERVER,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/UpdateCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/UpdateCommand.java b/shell/src/main/java/org/apache/sqoop/shell/UpdateCommand.java
index d291c42..c75d5f5 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/UpdateCommand.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/UpdateCommand.java
@@ -31,7 +31,7 @@ public class UpdateCommand extends SqoopCommand {
Constants.CMD_UPDATE,
Constants.CMD_UPDATE_SC,
ImmutableMap.of(
- Constants.FN_CONNECTION, UpdateConnectionFunction.class,
+ Constants.FN_LINK, UpdateLinkFunction.class,
Constants.FN_JOB, UpdateJobFunction.class
)
);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/UpdateConnectionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/UpdateConnectionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/UpdateConnectionFunction.java
deleted file mode 100644
index 405b0ea..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/UpdateConnectionFunction.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import jline.ConsoleReader;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.ConnectionDynamicFormOptions;
-import org.apache.sqoop.shell.utils.FormDisplayer;
-import org.apache.sqoop.shell.utils.FormOptions;
-import org.apache.sqoop.validation.Status;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class UpdateConnectionFunction extends SqoopFunction {
- @SuppressWarnings("static-access")
- public UpdateConnectionFunction() {
- this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_PROMPT_CONN_ID))
- .withLongOpt(Constants.OPT_XID)
- .isRequired()
- .hasArg()
- .create(Constants.OPT_XID_CHAR));
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Object executeFunction(CommandLine line, boolean isInteractive) throws IOException {
- return updateConnection(getLong(line, Constants.OPT_XID), line.getArgList(), isInteractive);
- }
-
- private Status updateConnection(Long connectionId, List<String> args, boolean isInteractive) throws IOException {
- printlnResource(Constants.RES_UPDATE_UPDATING_CONN, connectionId);
-
- ConsoleReader reader = new ConsoleReader();
-
- MConnection connection = client.getConnection(connectionId);
-
- ResourceBundle connectorBundle = client.getResourceBundle(connection.getConnectorId());
- ResourceBundle frameworkBundle = client.getFrameworkResourceBundle();
-
- Status status = Status.FINE;
-
- if (isInteractive) {
- printlnResource(Constants.RES_PROMPT_UPDATE_CONN_METADATA);
-
- do {
- // Print error introduction if needed
- if( !status.canProceed() ) {
- errorIntroduction();
- }
-
- // Fill in data from user
- if(!fillConnection(reader, connection, connectorBundle, frameworkBundle)) {
- return null;
- }
-
- // Try to create
- status = client.updateConnection(connection);
- } while(!status.canProceed());
- } else {
- ConnectionDynamicFormOptions options = new ConnectionDynamicFormOptions();
- options.prepareOptions(connection);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
- if (fillConnection(line, connection)) {
- status = client.updateConnection(connection);
- if (!status.canProceed()) {
- printConnectionValidationMessages(connection);
- return null;
- }
- } else {
- printConnectionValidationMessages(connection);
- return null;
- }
- }
- FormDisplayer.displayFormWarning(connection);
- printlnResource(Constants.RES_UPDATE_CONN_SUCCESSFUL, status.name());
-
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
index 9fc9181..96d1fd8 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
@@ -63,16 +63,16 @@ public class UpdateJobFunction extends SqoopFunction {
MJob job = client.getJob(jobId);
- ResourceBundle fromConnectorBundle = client.getResourceBundle(
+ ResourceBundle fromConnectorBundle = client.getConnectorConfigResourceBundle(
job.getConnectorId(Direction.FROM));
- ResourceBundle frameworkBundle = client.getFrameworkResourceBundle();
- ResourceBundle toConnectorBundle = client.getResourceBundle(
+ ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+ ResourceBundle toConnectorBundle = client.getConnectorConfigResourceBundle(
job.getConnectorId(Direction.TO));
Status status = Status.FINE;
if (isInteractive) {
- printlnResource(Constants.RES_PROMPT_UPDATE_JOB_METADATA);
+ printlnResource(Constants.RES_PROMPT_UPDATE_JOB_CONFIG);
do {
// Print error introduction if needed
@@ -81,7 +81,7 @@ public class UpdateJobFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillJob(reader, job, fromConnectorBundle, frameworkBundle, toConnectorBundle)) {
+ if(!fillJob(reader, job, fromConnectorBundle, driverConfigBundle, toConnectorBundle)) {
return status;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
new file mode 100644
index 0000000..30bb63e
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import jline.ConsoleReader;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.shell.utils.LinkDynamicFormOptions;
+import org.apache.sqoop.shell.utils.FormDisplayer;
+import org.apache.sqoop.shell.utils.FormOptions;
+import org.apache.sqoop.validation.Status;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ResourceBundle;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+import static org.apache.sqoop.shell.utils.FormFiller.*;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class UpdateLinkFunction extends SqoopFunction {
+ @SuppressWarnings("static-access")
+ public UpdateLinkFunction() {
+ this.addOption(OptionBuilder
+ .withDescription(resourceString(Constants.RES_PROMPT_LINK_ID))
+ .withLongOpt(Constants.OPT_LID)
+ .isRequired()
+ .hasArg()
+ .create(Constants.OPT_LID_CHAR));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object executeFunction(CommandLine line, boolean isInteractive) throws IOException {
+ return updateLink(getLong(line, Constants.OPT_LID), line.getArgList(), isInteractive);
+ }
+
+ private Status updateLink(Long linkId, List<String> args, boolean isInteractive) throws IOException {
+ printlnResource(Constants.RES_UPDATE_UPDATING_LINK, linkId);
+
+ ConsoleReader reader = new ConsoleReader();
+
+ MLink link = client.getLink(linkId);
+
+ ResourceBundle connectorConfigBundle = client.getConnectorConfigResourceBundle(link.getConnectorId());
+ ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+
+ Status status = Status.FINE;
+
+ if (isInteractive) {
+ printlnResource(Constants.RES_PROMPT_UPDATE_LINK_CONFIG);
+
+ do {
+ // Print error introduction if needed
+ if( !status.canProceed() ) {
+ errorIntroduction();
+ }
+
+ // Fill in data from user
+ if(!fillLink(reader, link, connectorConfigBundle, driverConfigBundle)) {
+ return null;
+ }
+
+ // Try to create
+ status = client.updateLink(link);
+ } while(!status.canProceed());
+ } else {
+ LinkDynamicFormOptions options = new LinkDynamicFormOptions();
+ options.prepareOptions(link);
+ CommandLine line = FormOptions.parseOptions(options, 0, args, false);
+ if (fillConnection(line, link)) {
+ status = client.updateLink(link);
+ if (!status.canProceed()) {
+ printLinkValidationMessages(link);
+ return null;
+ }
+ } else {
+ printLinkValidationMessages(link);
+ return null;
+ }
+ }
+ FormDisplayer.displayFormWarning(link);
+ printlnResource(Constants.RES_UPDATE_LINK_SUCCESSFUL, status.name());
+
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
index f0dc3a6..4d3838c 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
@@ -34,7 +34,7 @@ public class Constants {
// Options
- public static final String OPT_XID = "xid";
+ public static final String OPT_LID = "lid";
public static final String OPT_FROM = "from";
public static final String OPT_TO = "to";
public static final String OPT_ALL = "all";
@@ -54,9 +54,9 @@ public class Constants {
public static final String OPT_POLL_TIMEOUT = "poll-timeout";
public static final String OPT_DETAIL = "detail";
- public static final char OPT_XID_CHAR = 'x';
- public static final char OPT_FXID_CHAR = 'f';
- public static final char OPT_TXID_CHAR = 't';
+ public static final char OPT_LID_CHAR = 'l';
+ public static final char OPT_FROM_CHAR = 'f';
+ public static final char OPT_TO_CHAR = 't';
public static final char OPT_ALL_CHAR = 'a';
public static final char OPT_JID_CHAR = 'j';
public static final char OPT_CID_CHAR = 'c';
@@ -111,14 +111,14 @@ public class Constants {
public static final String CMD_DISABLE = "disable";
public static final String CMD_DISABLE_SC = "\\di";
- public static final String FN_CONNECTION = "connection";
+ public static final String FN_LINK = "link";
public static final String FN_JOB = "job";
public static final String FN_SUBMISSION = "submission";
public static final String FN_SERVER = "server";
public static final String FN_OPTION = "option";
public static final String FN_CONNECTOR = "connector";
public static final String FN_VERSION = "version";
- public static final String FN_FRAMEWORK = "framework";
+ public static final String FN_DRIVER_CONFIG = "driverConfig";
public static final String PROP_HOMEDIR = "user.home";
public static final String PROP_CURDIR = "user.dir";
@@ -130,8 +130,8 @@ public class Constants {
// Resource Keys for various messages
- public static final String RES_ARGS_XID_MISSING =
- "args.xid_missing";
+ public static final String RES_ARGS_LID_MISSING =
+ "args.lid_missing";
public static final String RES_ARGS_FROM_MISSING =
"args.from_missing";
public static final String RES_ARGS_TO_MISSING =
@@ -145,46 +145,46 @@ public class Constants {
public static final String RES_ARGS_VALUE_MISSING =
"args.value_missing";
- public static final String RES_PROMPT_CONN_ID =
- "prompt.conn_id";
+ public static final String RES_PROMPT_LINK_ID =
+ "prompt.link_id";
public static final String RES_PROMPT_JOB_ID =
"prompt.job_id";
public static final String RES_CONNECTOR_ID =
"prompt.connector_id";
- public static final String RES_PROMPT_UPDATE_CONN_METADATA =
- "prompt.update_conn_metadata";
- public static final String RES_PROMPT_UPDATE_JOB_METADATA =
- "prompt.update_job_metadata";
- public static final String RES_PROMPT_FILL_CONN_METADATA =
- "prompt.fill_conn_metadata";
- public static final String RES_PROMPT_FILL_JOB_METADATA =
- "prompt.fill_job_metadata";
-
- public static final String RES_CLONE_CONN_SUCCESSFUL =
- "clone.conn.successful";
+ public static final String RES_PROMPT_UPDATE_LINK_CONFIG =
+ "prompt.update_link_config";
+ public static final String RES_PROMPT_UPDATE_JOB_CONFIG =
+ "prompt.update_job_config";
+ public static final String RES_PROMPT_FILL_LINK_CONFIG =
+ "prompt.fill_link_config";
+ public static final String RES_PROMPT_FILL_JOB_CONFIG =
+ "prompt.fill_job_config";
+
+ public static final String RES_CLONE_LINK_SUCCESSFUL =
+ "clone.link.successful";
public static final String RES_CLONE_JOB_SUCCESSFUL =
"clone.job.successful";
- public static final String RES_CLONE_CLONING_CONN =
- "clone.cloning_conn";
+ public static final String RES_CLONE_CLONING_LINK =
+ "clone.cloning_link";
public static final String RES_CLONE_CLONING_JOB =
"clone.cloning_job";
- public static final String RES_CREATE_CONN_SUCCESSFUL =
- "create.conn_successful";
+ public static final String RES_CREATE_LINK_SUCCESSFUL =
+ "create.link_successful";
public static final String RES_CREATE_JOB_SUCCESSFUL =
"create.job_successful";
- public static final String RES_CREATE_CREATING_CONN =
- "create.creating_conn";
+ public static final String RES_CREATE_CREATING_LINK =
+ "create.creating_link";
public static final String RES_CREATE_CREATING_JOB =
"create.creating_job";
- public static final String RES_DISABLE_CONNECTION_SUCCESSFUL =
- "disable.conn_successful";
+ public static final String RES_DISABLE_LINK_SUCCESSFUL =
+ "disable.link_successful";
public static final String RES_DISABLE_JOB_SUCCESSFUL =
"disable.job_successful";
- public static final String RES_ENABLE_CONNECTION_SUCCESSFUL =
- "enable.conn_successful";
+ public static final String RES_ENABLE_LINK_SUCCESSFUL =
+ "enable.link_successful";
public static final String RES_ENABLE_JOB_SUCCESSFUL =
"enable.job_successful";
@@ -233,16 +233,16 @@ public class Constants {
public static final String RES_SET_SERVER_IGNORED =
"set.server_ignored";
- public static final String RES_SHOW_PROMPT_DISPLAY_ALL_CONNS =
- "show.prompt_display_all_conns";
- public static final String RES_SHOW_PROMPT_DISPLAY_CONN_XID =
- "show.prompt_display_conn_xid";
- public static final String RES_SHOW_PROMPT_CONNS_TO_SHOW =
- "show.prompt_conns_to_show";
- public static final String RES_SHOW_PROMPT_CONN_INFO =
- "show.prompt_conn_info";
- public static final String RES_SHOW_PROMPT_CONN_CID_INFO =
- "show.prompt_conn_cid_info";
+ public static final String RES_SHOW_PROMPT_DISPLAY_ALL_LINKS =
+ "show.prompt_display_all_links";
+ public static final String RES_SHOW_PROMPT_DISPLAY_LINK_LID =
+ "show.prompt_display_link_lid";
+ public static final String RES_SHOW_PROMPT_LINKS_TO_SHOW =
+ "show.prompt_links_to_show";
+ public static final String RES_SHOW_PROMPT_LINK_INFO =
+ "show.prompt_link_info";
+ public static final String RES_SHOW_PROMPT_LINK_CID_INFO =
+ "show.prompt_link_cid_info";
public static final String RES_SHOW_PROMPT_DISPLAY_ALL_CONNECTORS =
"show.prompt_display_all_connectors";
@@ -253,10 +253,10 @@ public class Constants {
public static final String RES_SHOW_PROMPT_CONNECTOR_INFO =
"show.prompt_connector_info";
- public static final String RES_SHOW_FRAMEWORK_USAGE =
- "show.framework_usage";
- public static final String RES_SHOW_PROMPT_FRAMEWORK_OPTS =
- "show.prompt_framework_opts";
+ public static final String RES_SHOW_DRIVER_CONFIG_USAGE =
+ "show.driver_config_usage";
+ public static final String RES_SHOW_PROMPT_DRIVER_CONFIG_OPTS =
+ "show.prompt_driver_config_opts";
public static final String RES_SHOW_PROMPT_DISPLAY_ALL_JOBS =
"show.prompt_display_all_jobs";
@@ -266,8 +266,8 @@ public class Constants {
"show.prompt_jobs_to_show";
public static final String RES_SHOW_PROMPT_JOB_INFO =
"show.prompt_job_info";
- public static final String RES_SHOW_PROMPT_JOB_XID_CID_INFO =
- "show.prompt_job_xid_cid_info";
+ public static final String RES_SHOW_PROMPT_JOB_LID_CID_INFO =
+ "show.prompt_job_lid_cid_info";
public static final String RES_SHOW_PROMPT_DISPLAY_ALL_SUBMISSIONS =
"show.prompt_display_all_submissions";
@@ -316,10 +316,10 @@ public class Constants {
public static final String RES_SQOOP_PROMPT_SHELL_LOADEDRC =
"sqoop.prompt_shell_loadedrc";
- public static final String RES_UPDATE_UPDATING_CONN =
- "update.conn";
- public static final String RES_UPDATE_CONN_SUCCESSFUL =
- "update.conn_successful";
+ public static final String RES_UPDATE_UPDATING_LINK =
+ "update.link";
+ public static final String RES_UPDATE_LINK_SUCCESSFUL =
+ "update.link_successful";
public static final String RES_UPDATE_UPDATING_JOB =
"update.job";
public static final String RES_UPDATE_JOB_SUCCESSFUL =
@@ -353,7 +353,7 @@ public class Constants {
"table.header.enabled";
public static final String RES_FORMDISPLAYER_CONNECTION =
- "formdisplayer.connection";
+ "formdisplayer.link";
public static final String RES_FORMDISPLAYER_JOB =
"formdisplayer.job";
public static final String RES_FORMDISPLAYER_FORM =
@@ -402,10 +402,10 @@ public class Constants {
"submission.executed_success";
public static final String RES_SUBMISSION_SERVER_URL =
"submission.server_url";
- public static final String RES_CONNECTOR_SCHEMA =
- "submission.connector_schema";
- public static final String RES_HIO_SCHEMA =
- "submission.hio_schema";
+ public static final String RES_FROM_SCHEMA =
+ "submission.from_schema";
+ public static final String RES_TO_SCHEMA =
+ "submission.to_schema";
private Constants() {
// Instantiation is prohibited
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/utils/ConnectionDynamicFormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/ConnectionDynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/ConnectionDynamicFormOptions.java
deleted file mode 100644
index 6b6e858..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/ConnectionDynamicFormOptions.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.model.MConnection;
-
-/**
- * Automatically create dynamic options for connections.
- */
-@SuppressWarnings("serial")
-public class ConnectionDynamicFormOptions extends DynamicFormOptions<MConnection> {
-
- @SuppressWarnings("static-access")
- @Override
- public void prepareOptions(MConnection connection) {
- this.addOption(OptionBuilder
- .withLongOpt("name")
- .hasArg()
- .create());
- for (Option option : FormOptions.getFormsOptions("connector", connection.getConnectorPart().getForms())) {
- this.addOption(option);
- }
- for (Option option : FormOptions.getFormsOptions("framework", connection.getFrameworkPart().getForms())) {
- this.addOption(option);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
index bbad183..dcbccef 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
@@ -21,11 +21,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MAccountableEntity;
import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MEnumInput;
import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MInputType;
import org.apache.sqoop.model.MIntegerInput;
@@ -49,15 +49,15 @@ import static org.apache.sqoop.shell.ShellEnvironment.*;
*/
public final class FormDisplayer {
- public static void displayFormMetadataDetails(MFramework framework,
+ public static void displayFormMetadataDetails(MDriverConfig driverConfig,
ResourceBundle bundle) {
displayFormsMetadata(
- framework.getConnectionForms().getForms(),
+ driverConfig.getConnectionForms().getForms(),
resourceString(Constants.RES_FORMDISPLAYER_CONNECTION),
bundle);
displayFormsMetadata(
- framework.getJobForms().getForms(),
+ driverConfig.getJobForms().getForms(),
resourceString(Constants.RES_FORMDISPLAYER_JOB),
bundle);
}
@@ -143,15 +143,15 @@ public final class FormDisplayer {
/**
* Method prints the warning message of ACCEPTABLE status
- * @param entity - connection or job instance
+ * @param entity - link or job instance
*/
public static void displayFormWarning(MAccountableEntity entity) {
List<MForm> formList = new ArrayList<MForm>();
boolean showMessage = true;
- if (entity instanceof MConnection) {
- MConnection connection = (MConnection) entity;
- formList.addAll(connection.getConnectorPart().getForms());
- formList.addAll(connection.getFrameworkPart().getForms());
+ if (entity instanceof MLink) {
+ MLink link = (MLink) entity;
+ formList.addAll(link.getConnectorPart().getForms());
+ formList.addAll(link.getFrameworkPart().getForms());
} else if(entity instanceof MJob) {
MJob job = (MJob) entity;
formList.addAll(job.getConnectorPart(Direction.FROM).getForms());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
index bbf7af7..a0a5dc2 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
@@ -23,7 +23,7 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MEnumInput;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MInput;
@@ -50,7 +50,7 @@ import static org.apache.sqoop.shell.ShellEnvironment.*;
public final class FormFiller {
/**
- * Internal input that will be reused for loading names for connection and
+ * Internal input that will be reused for loading names for link and
* job objects.
*/
private static MStringInput nameInput = new MStringInput("object-name", false, (short)25);
@@ -81,14 +81,14 @@ public final class FormFiller {
* @param reader Associated console reader object
* @param job Job that user is suppose to fill in
* @param fromConnectorBundle Connector resource bundle
- * @param frameworkBundle Framework resource bundle
+ * @param driverConfigBundle Driver config resource bundle
* @return True if we filled all inputs, false if user has stopped processing
* @throws IOException
*/
public static boolean fillJob(ConsoleReader reader,
MJob job,
ResourceBundle fromConnectorBundle,
- ResourceBundle frameworkBundle,
+ ResourceBundle driverConfigBundle,
ResourceBundle toConnectorBundle)
throws IOException {
@@ -99,55 +99,55 @@ public final class FormFiller {
job.getConnectorPart(Direction.FROM).getForms(),
fromConnectorBundle,
job.getFrameworkPart().getForms(),
- frameworkBundle,
+ driverConfigBundle,
job.getConnectorPart(Direction.TO).getForms(),
toConnectorBundle);
}
/**
- * Fill connection object based on CLI options.
+ * Fill link object based on CLI options.
*
* @param line Associated command line options
- * @param connection Connection that user is suppose to fill in
+ * @param link Link that user is suppose to fill in
* @return True if we filled all inputs, false if user has stopped processing
* @throws IOException
*/
public static boolean fillConnection(CommandLine line,
- MConnection connection)
+ MLink link)
throws IOException {
- connection.setName(line.getOptionValue("name"));
+ link.setName(line.getOptionValue("name"));
// Fill in data from user
return fillForms(line,
- connection.getConnectorPart().getForms(),
- connection.getFrameworkPart().getForms());
+ link.getConnectorPart().getForms(),
+ link.getFrameworkPart().getForms());
}
/**
- * Fill connection object based on user input.
+ * Fill link object based on user input.
*
* @param reader Associated console reader object
- * @param connection Connection that user is suppose to fill in
- * @param connectorBundle Connector resource bundle
- * @param frameworkBundle Framework resouce bundle
+ * @param link Link that user is suppose to fill in
+ * @param connectorConfigBundle Connector resource bundle
+ * @param driverConfigBundle Driver config resource bundle
* @return True if we filled all inputs, false if user has stopped processing
* @throws IOException
*/
- public static boolean fillConnection(ConsoleReader reader,
- MConnection connection,
- ResourceBundle connectorBundle,
- ResourceBundle frameworkBundle)
+ public static boolean fillLink(ConsoleReader reader,
+ MLink link,
+ ResourceBundle connectorConfigBundle,
+ ResourceBundle driverConfigBundle)
throws IOException {
- connection.setName(getName(reader, connection.getName()));
+ link.setName(getName(reader, link.getName()));
// Fill in data from user
return fillForms(reader,
- connection.getConnectorPart().getForms(),
- connectorBundle,
- connection.getFrameworkPart().getForms(),
- frameworkBundle);
+ link.getConnectorPart().getForms(),
+ connectorConfigBundle,
+ link.getFrameworkPart().getForms(),
+ driverConfigBundle);
}
/**
@@ -391,18 +391,18 @@ public final class FormFiller {
public static boolean fillForms(ConsoleReader reader,
List<MForm> connectorForms,
- ResourceBundle connectorBundle,
+ ResourceBundle connectorConfigBundle,
List<MForm> frameworkForms,
- ResourceBundle frameworkBundle) throws IOException {
+ ResourceBundle driverConfigBundle) throws IOException {
// Query connector forms
- if(!fillForms(connectorForms, reader, connectorBundle)) {
+ if(!fillForms(connectorForms, reader, connectorConfigBundle)) {
return false;
}
// Query framework forms
- if(!fillForms(frameworkForms, reader, frameworkBundle)) {
+ if(!fillForms(frameworkForms, reader, driverConfigBundle)) {
return false;
}
return true;
@@ -412,7 +412,7 @@ public final class FormFiller {
List<MForm> fromConnectorForms,
ResourceBundle fromConnectorBundle,
List<MForm> frameworkForms,
- ResourceBundle frameworkBundle,
+ ResourceBundle driverConfigBundle,
List<MForm> toConnectorForms,
ResourceBundle toConnectorBundle) throws IOException {
@@ -423,7 +423,7 @@ public final class FormFiller {
}
// Query framework forms
- if(!fillForms(frameworkForms, reader, frameworkBundle)) {
+ if(!fillForms(frameworkForms, reader, driverConfigBundle)) {
return false;
}
@@ -902,13 +902,13 @@ public final class FormFiller {
println("@|red There are issues with entered data, please revise your input:|@");
}
- public static void printConnectionValidationMessages(MConnection connection) {
- for (MForm form : connection.getConnectorPart().getForms()) {
+ public static void printLinkValidationMessages(MLink link) {
+ for (MForm form : link.getConnectorPart().getForms()) {
for (MInput<?> input : form.getInputs()) {
printValidationMessage(input, true);
}
}
- for (MForm form : connection.getFrameworkPart().getForms()) {
+ for (MForm form : link.getFrameworkPart().getForms()) {
for (MInput<?> input : form.getInputs()) {
printValidationMessage(input, true);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
new file mode 100644
index 0000000..2952be2
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.model.MLink;
+
+/**
+ * Automatically create dynamic options for connections.
+ */
+@SuppressWarnings("serial")
+public class LinkDynamicFormOptions extends DynamicFormOptions<MLink> {
+
+ @SuppressWarnings("static-access")
+ @Override
+ public void prepareOptions(MLink link) {
+ this.addOption(OptionBuilder.withLongOpt("name").hasArg().create());
+ for (Option option : FormOptions.getFormsOptions("connector", link.getConnectorPart()
+ .getForms())) {
+ this.addOption(option);
+ }
+ for (Option option : FormOptions.getFormsOptions("framework", link.getFrameworkPart()
+ .getForms())) {
+ this.addOption(option);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
index 51f778b..60acfb6 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
@@ -69,12 +69,12 @@ public final class SubmissionDisplayer {
}
if(isVerbose() && submission.getFromSchema() != null) {
- print(resourceString(Constants.RES_CONNECTOR_SCHEMA)+": ");
+ print(resourceString(Constants.RES_FROM_SCHEMA)+": ");
println(submission.getFromSchema());
}
if(isVerbose() && submission.getToSchema() != null) {
- print(resourceString(Constants.RES_HIO_SCHEMA)+": ");
+ print(resourceString(Constants.RES_TO_SCHEMA)+": ");
println(submission.getToSchema());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/resources/shell-resource.properties
----------------------------------------------------------------------
diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties
index 247ceae..b59bd81 100644
--- a/shell/src/main/resources/shell-resource.properties
+++ b/shell/src/main/resources/shell-resource.properties
@@ -32,65 +32,65 @@ shared.usage = @|bold Usage:|@ {0} {1}
shared.unknown.function = The specified function "{0}" is not recognized.
# Argument related
-args.xid_missing = Required argument --xid is missing.
+args.lid_missing = Required argument --link Id is missing.
args.from_missing = Required argument --from is missing.
args.to_missing = Required argument --to is missing.
-args.jid_missing = Required argument --jid is missing.
-args.cid_missing = Required argument --cid is missing.
+args.jid_missing = Required argument --job Id is missing.
+args.cid_missing = Required argument --connector Id is missing.
args.name_missing = Required argument --name is missing.
args.value_missing = Required argument --value is missing.
## Generic description of various ids, types etc
-prompt.conn_id = Connection ID
-prompt.connector_id = Connector ID
-prompt.job_id = Job ID
+prompt.link_id = Link Id
+prompt.connector_id = Connector Id
+prompt.job_id = Job Id
prompt.job_type = Job type
-## Prompt messages for updating, filling metadata info
+## Prompt messages for updating, filling entity info
-prompt.update_conn_metadata = Please update connection metadata:
-prompt.update_job_metadata = Please update job metadata:
-prompt.fill_conn_metadata = Please fill following values to create new \
-connection object
-prompt.fill_job_metadata = Please fill following values to create new \
+prompt.update_link_config = Please update link:
+prompt.update_job_config = Please update job:
+prompt.fill_link_config = Please fill following values to create new \
+link object
+prompt.fill_job_config = Please fill following values to create new \
job object
# Update command
update.description = Update objects in Sqoop repository
-update.conn = Updating connection with id {0}
+update.link = Updating link with id {0}
update.job = Updating job with id {0}
-update.conn_successful = Connection was successfully updated with status {0}
+update.link_successful = link was successfully updated with status {0}
update.job_successful = Job was successfully updated with status {0}
# Clone command
clone.description = Create new object based on existing one
-clone.conn.successful = Connection was successfully created with validation \
+clone.link.successful = link was successfully created with validation \
status {0} and persistent id {1}
clone.job.successful = Job was successfully created with validation \
status {0} and persistent id {1}
-clone.cloning_conn = Cloning connection with id {0}
+clone.cloning_link = Cloning link with id {0}
clone.cloning_job = Cloning job with id {0}
# Create command
create.description = Create new object in Sqoop repository
-create.conn_successful = New connection was successfully created with \
+create.link_successful = New link was successfully created with \
validation status {0} and persistent id {1}
create.job_successful = New job was successfully created with validation \
status {0} and persistent id {1}
-create.creating_conn = Creating connection for connector with id {0}
-create.creating_job = Creating job for connections with id {0} and {1}
+create.creating_link = Creating link for connector with id {0}
+create.creating_job = Creating job for links with from id {0} and to id {1}
# Delete command
delete.description = Delete existing object in Sqoop repository
# Enable command
enable.description = Enable object in Sqoop repository
-enable.conn_successful = Connection {0} was successfully enabled
+enable.link_successful = link {0} was successfully enabled
enable.job_successful = Job {0} was successfully enabled
# Disable command
disable.description = Disable object in Sqoop repository
-disable.conn_successful = Connection {0} was successfully disabled
+disable.link_successful = link {0} was successfully disabled
disable.job_successful = Job {0} was successfully disabled
# Help command
@@ -126,12 +126,12 @@ set.server_ignored = --host, --port or --webapp option is ignored, because --url
# Show command
show.description = Display various objects and configuration options
-show.prompt_display_all_conns = Display all connections
-show.prompt_display_conn_xid = Display the connection with xid
-show.conn_usage = Usage: show connection
-show.prompt_conns_to_show = @|bold {0} connection(s) to show: |@
-show.prompt_conn_info = Connection with id {0} and name {1} (Enabled: {2}, Created by {3} at {4}, Updated by {5} at {6})
-show.prompt_conn_cid_info = Using Connector id {0}
+show.prompt_display_all_links = Display all links
+show.prompt_display_link_lid = Display the link with lid
+show.link_usage = Usage: show link
+show.prompt_links_to_show = @|bold {0} link(s) to show: |@
+show.prompt_link_info = link with id {0} and name {1} (Enabled: {2}, Created by {3} at {4}, Updated by {5} at {6})
+show.prompt_link_cid_info = Using Connector id {0}
show.prompt_display_all_connectors = Display all connectors
show.prompt_display_connector_cid = Display the connector with cid
@@ -140,15 +140,15 @@ show.prompt_connectors_to_show = @|bold {0} connector(s) to show: |@
show.prompt_connector_info = Connector with id {0}:\n Name: {1} \n \
Class: {2}\n Version: {3}\n Supported Directions {4}
-show.framework_usage = Usage: show framework
-show.prompt_framework_opts = @|bold Framework specific options: |@\nPersistent id: {0}
+show.driver_config_usage = Usage: show driver config
+show.prompt_driver_config_opts = @|bold Driver config specific options: |@\nPersistent id: {0}
show.prompt_display_all_jobs = Display all jobs
show.prompt_display_job_jid = Display jobwith given jid
show.job_usage = Usage: show job
show.prompt_jobs_to_show = @|bold {0} job(s) to show: |@
show.prompt_job_info = Job with id {0} and name {1} (Enabled: {2}, Created by {3} at {4}, Updated by {5} at {6})
-show.prompt_job_xid_cid_info = Using Connection id {0} and Connector id {1}
+show.prompt_job_lid_cid_info = Using link id {0} and Connector id {1}
show.prompt_display_all_submissions = Display all submissions
show.prompt_display_all_submissions_jid = Display all submissions given jid
@@ -201,7 +201,7 @@ table.header.date = Last Update Date
table.header.enabled = Enabled
#Form displayer resources
-formdisplayer.connection = Connection
+formdisplayer.link = link
formdisplayer.job = Job
formdisplayer.form = form
formdisplayer.name = Name
@@ -227,5 +227,5 @@ submission.progress_not_available = Progress is not available
submission.counters = Counters
submission.executed_success = Job executed successfully
submission.server_url = Server URL
-submission.connector_schema = Connector schema
-submission.hio_schema = Input/Output schema
+submission.from_schema = From schema
+submission.to_schema = To schema
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java b/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java
deleted file mode 100644
index 328f9b0..0000000
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/MetadataUpgrader.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sqoop.connector.spi;
-
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MJobForms;
-
-public abstract class MetadataUpgrader {
-
- /**
- * Upgrade the original connection and fill into the upgradeTarget. Note
- * that any metadata already in {@code upgradeTarget} maybe overwritten.
- * @param original - original connection metadata as in the repository
- * @param upgradeTarget - the instance that will be filled in with the
- * upgraded metadata.
- */
- public abstract void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget);
- /**
- * Upgrade the original job and fill into the upgradeTarget. Note
- * that any metadata already in {@code upgradeTarget} maybe overwritten.
- * This method must be called only after the connection metadata has
- * already been upgraded.
- * @param original - original connection metadata as in the repository
- * @param upgradeTarget - the instance that will be filled in with the
- * upgraded metadata.
- */
- public abstract void upgrade(MJobForms original, MJobForms upgradeTarget);
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java b/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
new file mode 100644
index 0000000..e00b404
--- /dev/null
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.connector.spi;
+
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MJobForms;
+
+/**
+ * Repository represents the sqoop entity store. Sqoop entities include
+ * the connectors, links, jobs and submissions and corresponding configs.
+ *
+ */
+public abstract class RepositoryUpgrader {
+
+ /**
+ * Upgrade the original connection and fill into the upgradeTarget. Note
+ * that any metadata already in {@code upgradeTarget} maybe overwritten.
+ * @param original - original connection metadata as in the repository
+ * @param upgradeTarget - the instance that will be filled in with the
+ * upgraded metadata.
+ */
+ public abstract void upgrade(MConnectionForms original,
+ MConnectionForms upgradeTarget);
+ /**
+ * Upgrade the original job and fill into the upgradeTarget. Note
+ * that any metadata already in {@code upgradeTarget} maybe overwritten.
+ * This method must be called only after the connection metadata has
+ * already been upgraded.
+ * @param original - original connection metadata as in the repository
+ * @param upgradeTarget - the instance that will be filled in with the
+ * upgraded metadata.
+ */
+ public abstract void upgrade(MJobForms original, MJobForms upgradeTarget);
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
index 2774e8b..7b78ba0 100644
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
@@ -60,7 +60,7 @@ public abstract class SqoopConnector {
/**
* @return Get connection configuration class
*/
- public abstract Class getConnectionConfigurationClass();
+ public abstract Class getLinkConfigurationClass();
/**
* @return Get job configuration class for given type or null if not supported
@@ -78,7 +78,7 @@ public abstract class SqoopConnector {
public abstract To getTo();
/**
- * Returns validation object that Sqoop framework can use to validate user
+ * Returns validation object that Sqoop can use to validate user
* supplied forms before accepting them. This object will be used both for
* connection and job forms.
*
@@ -87,11 +87,11 @@ public abstract class SqoopConnector {
public abstract Validator getValidator();
/**
- * Returns an {@linkplain MetadataUpgrader} object that can upgrade the
- * connection and job metadata.
- * @return MetadataUpgrader object
+ * Returns an {@linkplain RepositoryUpgrader} object that can upgrade the
+ * configs related to the link and job
+ * @return RespositoryUpgrader object
*/
- public abstract MetadataUpgrader getMetadataUpgrader();
+ public abstract RepositoryUpgrader getRepositoryUpgrader();
/**
* Returns the {@linkplain IntermediateDataFormat} this connector
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
index 149ad2c..f965bdd 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
@@ -21,17 +21,17 @@ package org.apache.sqoop.job.etl;
* This allows connector to define work to complete execution, for example,
* resource cleaning.
*/
-public abstract class Destroyer<ConnectionConfiguration, JobConfiguration> {
+public abstract class Destroyer<LinkConfiguration, JobConfiguration> {
/**
* Callback to clean up after job execution.
*
* @param context Destroyer context
- * @param connectionConfiguration Connection configuration object
+ * @param linkConfiguration link configuration object
* @param jobConfiguration Job configuration object
*/
public abstract void destroy(DestroyerContext context,
- ConnectionConfiguration connectionConfiguration,
+ LinkConfiguration linkConfiguration,
JobConfiguration jobConfiguration);
}
[46/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MConnector.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConnector.java b/common/src/main/java/org/apache/sqoop/model/MConnector.java
index 7999b08..2f42191 100644
--- a/common/src/main/java/org/apache/sqoop/model/MConnector.java
+++ b/common/src/main/java/org/apache/sqoop/model/MConnector.java
@@ -23,28 +23,27 @@ import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.common.SupportedDirections;
/**
- * Connector metadata.
- *
- * Includes unique id that identifies connector in metadata store, unique human
- * readable name, corresponding name and all forms for all supported job types.
+ * Connector entity supports the FROM/TO {@link Transferable} Includes unique id
+ * that identifies connector in the repository, unique human readable name,
+ * corresponding name and all configs to support the from and to data sources
*/
public final class MConnector extends MPersistableEntity implements MClonable {
private final String uniqueName;
private final String className;
- private final MConnectionForms connectionForms;
- private final MJobForms fromJobForms;
- private final MJobForms toJobForms;
- String version;
-
- public MConnector(String uniqueName, String className,
- String version, MConnectionForms connectionForms,
- MJobForms fromJobForms, MJobForms toJobForms) {
+ private final String version;
+ private final MLinkConfig linkConfig;
+ private final MFromConfig fromConfig;
+ private final MToConfig toConfig;
+
+ public MConnector(String uniqueName, String className, String version, MLinkConfig linkConfig,
+ MFromConfig fromConfig, MToConfig toConfig) {
this.version = version;
- this.connectionForms = connectionForms;
- this.fromJobForms = fromJobForms;
- this.toJobForms = toJobForms;
+ this.linkConfig = linkConfig;
+ this.fromConfig = fromConfig;
+ this.toConfig = toConfig;
+ // Why are we abusing NPE?
if (uniqueName == null || className == null) {
throw new NullPointerException();
}
@@ -63,17 +62,15 @@ public final class MConnector extends MPersistableEntity implements MClonable {
@Override
public String toString() {
- MJobForms fromJobForms = this.getJobForms(Direction.FROM);
- MJobForms toJobForms = this.getJobForms(Direction.TO);
StringBuilder sb = new StringBuilder("connector-");
sb.append(uniqueName).append(":").append(getPersistenceId()).append(":");
sb.append(className);
- sb.append(", ").append(getConnectionForms().toString());
- if (fromJobForms != null) {
- sb.append(", ").append(fromJobForms.toString());
+ sb.append(", ").append(getLinkConfig().toString());
+ if (getConfig(Direction.FROM) != null) {
+ sb.append(", ").append(getConfig(Direction.FROM).toString());
}
- if (toJobForms != null) {
- sb.append(", ").append(toJobForms.toString());
+ if (getConfig(Direction.TO) != null) {
+ sb.append(", ").append(getConfig(Direction.TO).toString());
}
return sb.toString();
}
@@ -94,41 +91,39 @@ public final class MConnector extends MPersistableEntity implements MClonable {
if (supportedDirections.isDirectionSupported(Direction.FROM)
&& mcSupportedDirections.isDirectionSupported(Direction.FROM)
- && !getJobForms(Direction.FROM).equals(mc.getJobForms(Direction.FROM))) {
+ && !getFromConfig().equals(mc.getFromConfig())) {
return false;
}
- if (supportedDirections.isDirectionSupported(Direction.FROM)
- != mcSupportedDirections.isDirectionSupported(Direction.FROM)) {
+ if (supportedDirections.isDirectionSupported(Direction.FROM) != mcSupportedDirections
+ .isDirectionSupported(Direction.FROM)) {
return false;
}
if (supportedDirections.isDirectionSupported(Direction.TO)
&& mcSupportedDirections.isDirectionSupported(Direction.TO)
- && !getJobForms(Direction.TO).equals(mc.getJobForms(Direction.TO))) {
+ && !getToConfig().equals(mc.getToConfig())) {
return false;
}
- if (supportedDirections.isDirectionSupported(Direction.TO)
- != mcSupportedDirections.isDirectionSupported(Direction.TO)) {
+ if (supportedDirections.isDirectionSupported(Direction.TO) != mcSupportedDirections
+ .isDirectionSupported(Direction.TO)) {
return false;
}
- return uniqueName.equals(mc.uniqueName)
- && className.equals(mc.className)
- && version.equals(mc.version)
- && connectionForms.equals(mc.getConnectionForms());
+ return uniqueName.equals(mc.uniqueName) && className.equals(mc.className)
+ && version.equals(mc.version) && linkConfig.equals((mc.getLinkConfig()));
}
@Override
public int hashCode() {
SupportedDirections supportedDirections = getSupportedDirections();
- int result = getConnectionForms().hashCode();
+ int result = getLinkConfig().hashCode();
if (supportedDirections.isDirectionSupported(Direction.FROM)) {
- result = 31 * result + getJobForms(Direction.FROM).hashCode();
+ result = 31 * result + getFromConfig().hashCode();
}
if (supportedDirections.isDirectionSupported(Direction.TO)) {
- result = 31 * result + getJobForms(Direction.TO).hashCode();
+ result = 31 * result + getToConfig().hashCode();
}
result = 31 * result + version.hashCode();
result = 31 * result + uniqueName.hashCode();
@@ -137,58 +132,57 @@ public final class MConnector extends MPersistableEntity implements MClonable {
}
public MConnector clone(boolean cloneWithValue) {
- //Connector never have any values filled
+ // Connector never have any values filled
cloneWithValue = false;
- MJobForms fromJobForms = this.getJobForms(Direction.FROM);
- MJobForms toJobForms = this.getJobForms(Direction.TO);
+ MFromConfig fromConfig = this.getFromConfig();
+ MToConfig toConfig = this.getToConfig();
- if (fromJobForms != null) {
- fromJobForms = fromJobForms.clone(cloneWithValue);
+ if (fromConfig != null) {
+ fromConfig = fromConfig.clone(cloneWithValue);
}
- if (toJobForms != null) {
- toJobForms = toJobForms.clone(cloneWithValue);
+ if (toConfig != null) {
+ toConfig = toConfig.clone(cloneWithValue);
}
- MConnector copy = new MConnector(
- this.getUniqueName(),
- this.getClassName(),
- this.getVersion(),
- this.getConnectionForms().clone(cloneWithValue),
- fromJobForms,
- toJobForms);
+ MConnector copy = new MConnector(this.getUniqueName(), this.getClassName(), this.getVersion(),
+ this.getLinkConfig().clone(cloneWithValue), fromConfig, toConfig);
copy.setPersistenceId(this.getPersistenceId());
return copy;
}
- public MConnectionForms getConnectionForms() {
- return connectionForms;
+ public MLinkConfig getLinkConfig() {
+ return linkConfig;
}
- public MJobForms getJobForms(Direction type) {
- switch(type) {
- case FROM:
- return fromJobForms;
+ public MConfigList getConfig(Direction type) {
+ switch (type) {
+ case FROM:
+ return fromConfig;
- case TO:
- return toJobForms;
+ case TO:
+ return toConfig;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
- public String getVersion() {
- return version;
+ public MFromConfig getFromConfig() {
+ return fromConfig;
}
- public void setVersion(String version) {
- this.version = version;
+ public MToConfig getToConfig() {
+ return toConfig;
+ }
+
+ public String getVersion() {
+ return version;
}
public SupportedDirections getSupportedDirections() {
- return new SupportedDirections(this.getJobForms(Direction.FROM) != null,
- this.getJobForms(Direction.TO) != null);
+ return new SupportedDirections(this.getConfig(Direction.FROM) != null,
+ this.getConfig(Direction.TO) != null);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MDriver.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MDriver.java b/common/src/main/java/org/apache/sqoop/model/MDriver.java
new file mode 100644
index 0000000..685439e
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MDriver.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.sql.Driver;
+
+/**
+ * Describes the configs associated with the {@link Driver} for executing sqoop jobs.
+ */
+public class MDriver extends MPersistableEntity implements MClonable {
+
+ private final MDriverConfig driverConfig;
+ private final String version;
+
+ public MDriver(MDriverConfig driverConfig, String version) {
+ this.driverConfig = driverConfig;
+ this.version = version;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("driver-");
+ sb.append(getPersistenceId()).append(":");
+ sb.append("version = " + version);
+ sb.append(", ").append(driverConfig.toString());
+
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (!(other instanceof MDriver)) {
+ return false;
+ }
+
+ MDriver driver = (MDriver) other;
+ return version.equals(driver.getVersion()) &&
+ driverConfig.equals(driver.driverConfig);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = driverConfig.hashCode();
+ result = 31 * result + version.hashCode();
+ return result;
+ }
+
+ public MDriverConfig getDriverConfig() {
+ return driverConfig;
+ }
+
+ @Override
+ public MDriver clone(boolean cloneWithValue) {
+ cloneWithValue = false;
+ MDriver copy = new MDriver(this.driverConfig.clone(cloneWithValue), this.version);
+ copy.setPersistenceId(this.getPersistenceId());
+ return copy;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java b/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java
index 62eb197..679859a 100644
--- a/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java
+++ b/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java
@@ -17,29 +17,23 @@
*/
package org.apache.sqoop.model;
-/**
- * Describes the configs associated with the {@link Driver} for executing sqoop jobs.
- */
-public class MDriverConfig extends MPersistableEntity implements MClonable {
+import java.util.List;
- private final MConnectionForms connectionForms;
- private final MJobForms jobForms;
- String version;
+/**
+ * Config describing all required information for the driver
+ * NOTE: It extends a config list since {@link MToConfig} could consist of a related config groups
+ * In future this could be simplified to hold a single list of all configs for the driver
- public MDriverConfig(MConnectionForms connectionForms, MJobForms jobForms, String version) {
- this.connectionForms = connectionForms;
- this.jobForms = jobForms;
- this.version = version;
+ */
+public class MDriverConfig extends MConfigList {
+ public MDriverConfig(List<MConfig> configs) {
+ super(configs);
}
@Override
public String toString() {
- StringBuilder sb = new StringBuilder("driver-");
- sb.append(getPersistenceId()).append(":");
- sb.append("version = " + version);
- sb.append(", ").append(connectionForms.toString());
- sb.append(jobForms.toString());
-
+ StringBuilder sb = new StringBuilder("Driver:");
+ sb.append(super.toString());
return sb.toString();
}
@@ -53,45 +47,18 @@ public class MDriverConfig extends MPersistableEntity implements MClonable {
return false;
}
- MDriverConfig mo = (MDriverConfig) other;
- return version.equals(mo.getVersion()) &&
- connectionForms.equals(mo.connectionForms) &&
- jobForms.equals(mo.jobForms);
+ MDriverConfig mDriver = (MDriverConfig) other;
+ return super.equals(mDriver);
}
@Override
public int hashCode() {
- int result = connectionForms.hashCode();
- result = 31 * result + jobForms.hashCode();
- result = 31 * result + version.hashCode();
- return result;
- }
-
- public MConnectionForms getConnectionForms() {
- return connectionForms;
- }
-
- public MJobForms getJobForms() {
- return jobForms;
+ return super.hashCode();
}
@Override
public MDriverConfig clone(boolean cloneWithValue) {
- //Framework never have any values filled
- cloneWithValue = false;
- MDriverConfig copy = new MDriverConfig(this.getConnectionForms().clone(cloneWithValue),
- this.getJobForms().clone(cloneWithValue), this.version);
- copy.setPersistenceId(this.getPersistenceId());
+ MDriverConfig copy = new MDriverConfig(super.clone(cloneWithValue).getConfigs());
return copy;
}
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
-}
-
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MForm.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MForm.java b/common/src/main/java/org/apache/sqoop/model/MForm.java
deleted file mode 100644
index ff94660..0000000
--- a/common/src/main/java/org/apache/sqoop/model/MForm.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import org.apache.sqoop.common.SqoopException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Represents a group of inputs that are processed together. This allows the
- * input gathering process to be broken down into multiple steps that can be
- * then paged through by the user interface.
- */
-public final class MForm extends MValidatedElement implements MClonable {
-
- private final List<MInput<?>> inputs;
-
- public MForm(String name, List<MInput<?>> inputs) {
- super(name);
-
- this.inputs = inputs;
- }
-
- public List<MInput<?>> getInputs() {
- return inputs;
- }
-
- public MInput<?> getInput(String inputName) {
- for(MInput<?> input: inputs) {
- if(inputName.equals(input.getName())) {
- return input;
- }
- }
-
- throw new SqoopException(ModelError.MODEL_011, "Input name: " + inputName);
- }
-
- public MStringInput getStringInput(String inputName) {
- return (MStringInput)getInput(inputName);
- }
-
- public MEnumInput getEnumInput(String inputName) {
- return (MEnumInput)getInput(inputName);
- }
-
- public MIntegerInput getIntegerInput(String inputName) {
- return (MIntegerInput)getInput(inputName);
- }
-
- public MBooleanInput getBooleanInput(String inputName) {
- return (MBooleanInput)getInput(inputName);
- }
-
- public MMapInput getMapInput(String inputName) {
- return (MMapInput)getInput(inputName);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("form-").append(getName());
- sb.append(":").append(getPersistenceId()).append(":").append(inputs);
-
- return sb.toString();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == this) {
- return true;
- }
-
- if (!(other instanceof MForm)) {
- return false;
- }
-
- MForm mf = (MForm) other;
- return getName().equals(mf.getName())
- && inputs.equals(mf.inputs);
- }
-
- @Override
- public int hashCode() {
- int result = 17;
- result = 31 * result + getName().hashCode();
- for (MInput<?> mi : inputs) {
- result = 31 * result + mi.hashCode();
- }
-
- return result;
- }
-
- @Override
- public MForm clone(boolean cloneWithValue) {
- List<MInput<?>> copyInputs = new ArrayList<MInput<?>>();
- for(MInput<?> itr : this.getInputs()) {
- copyInputs.add((MInput<?>)itr.clone(cloneWithValue));
- }
- MForm copyForm = new MForm(this.getName(), copyInputs);
- return copyForm;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MFormList.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MFormList.java b/common/src/main/java/org/apache/sqoop/model/MFormList.java
deleted file mode 100644
index 9130ada..0000000
--- a/common/src/main/java/org/apache/sqoop/model/MFormList.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import org.apache.sqoop.common.SqoopException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Arbitrary list of forms.
- */
-public class MFormList implements MClonable {
-
- private final List<MForm> forms;
-
- public MFormList(List<MForm> forms) {
- this.forms = forms;
- }
-
- public List<MForm> getForms() {
- return forms;
- }
-
- public MForm getForm(String formName) {
- for(MForm form: forms) {
- if(formName.equals(form.getName())) {
- return form;
- }
- }
-
- throw new SqoopException(ModelError.MODEL_010, "Form name: " + formName);
- }
-
- public MInput getInput(String name) {
- String []parts = name.split("\\.");
- if(parts.length != 2) {
- throw new SqoopException(ModelError.MODEL_009, name);
- }
-
- return getForm(parts[0]).getInput(name);
- }
-
- public MStringInput getStringInput(String name) {
- return (MStringInput)getInput(name);
- }
-
- public MEnumInput getEnumInput(String name) {
- return (MEnumInput)getInput(name);
- }
-
- public MIntegerInput getIntegerInput(String name) {
- return (MIntegerInput)getInput(name);
- }
-
- public MMapInput getMapInput(String name) {
- return (MMapInput)getInput(name);
- }
-
- public MBooleanInput getBooleanInput(String name) {
- return (MBooleanInput)getInput(name);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof MFormList)) return false;
-
- MFormList mFormList = (MFormList) o;
-
- if (!forms.equals(mFormList.forms)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- for(MForm form : forms) {
- result = 31 * result + form.hashCode();
- }
-
- return result;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("Forms: ");
- for(MForm form : forms) {
- sb.append(form.toString());
- }
- return sb.toString();
- }
-
- @Override
- public MFormList clone(boolean cloneWithValue) {
- List<MForm> copyForms = null;
- if(this.getForms() != null) {
- copyForms = new ArrayList<MForm>();
- for(MForm itr : this.getForms()) {
- MForm newForm = itr.clone(cloneWithValue);
- newForm.setPersistenceId(itr.getPersistenceId());
- copyForms.add(newForm);
- }
- }
- MFormList copyFormList = new MFormList(copyForms);
- return copyFormList;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MFormType.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MFormType.java b/common/src/main/java/org/apache/sqoop/model/MFormType.java
deleted file mode 100644
index 2f403df..0000000
--- a/common/src/main/java/org/apache/sqoop/model/MFormType.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-/**
- * Represents the various form types supported by the system.
- */
-public enum MFormType {
-
- /** Unknown form type */
- OTHER,
-
- /** Connection form type */
- CONNECTION,
-
- /** Job form type */
- JOB;
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MFromConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MFromConfig.java b/common/src/main/java/org/apache/sqoop/model/MFromConfig.java
new file mode 100644
index 0000000..1b450d6
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MFromConfig.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.util.List;
+
+/**
+ * Config describing all required information to build the FROM part of the job
+ * NOTE: It extends a config list since {@link MFromConfig} could consist of a related config groups
+ * In future this could be simplified to hold a single list of all configs for the FROM object
+
+ */
+public class MFromConfig extends MConfigList {
+ public MFromConfig(List<MConfig> configs) {
+ super(configs);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("From: ");
+ sb.append(super.toString());
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (!(other instanceof MFromConfig)) {
+ return false;
+ }
+
+ MFromConfig mj = (MFromConfig) other;
+ return super.equals(mj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public MFromConfig clone(boolean cloneWithValue) {
+ MFromConfig copy = new MFromConfig(super.clone(cloneWithValue).getConfigs());
+ return copy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MJob.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MJob.java b/common/src/main/java/org/apache/sqoop/model/MJob.java
index c9b45a5..b3dec27 100644
--- a/common/src/main/java/org/apache/sqoop/model/MJob.java
+++ b/common/src/main/java/org/apache/sqoop/model/MJob.java
@@ -22,29 +22,23 @@ import org.apache.sqoop.common.DirectionError;
import org.apache.sqoop.common.SqoopException;
/**
- * Model describing entire job object including both connector and
- * framework part.
+ * Model describing entire job object including the from/to and driver config information
+ * to execute the job
*/
public class MJob extends MAccountableEntity implements MClonable {
/**
- * Connector reference.
- *
- * Job object do not immediately depend on connector as there is indirect
+ * NOTE : Job object do not immediately depend on connector as there is indirect
* dependency through link object, but having this dependency explicitly
- * carried along helps a lot.
+ * carried along helps with not having to make the DB call everytime
*/
private final long fromConnectorId;
private final long toConnectorId;
-
- /**
- * Corresponding link objects for connector.
- */
private final long fromLinkId;
private final long toLinkId;
- private final MJobForms fromConnectorPart;
- private final MJobForms toConnectorPart;
- private final MJobForms frameworkPart;
+ private final MFromConfig fromConfig;
+ private final MToConfig toConfig;
+ private final MDriverConfig driverConfig;
/**
* Default constructor to build new MJob model.
@@ -53,24 +47,24 @@ public class MJob extends MAccountableEntity implements MClonable {
* @param toConnectorId TO Connector id
* @param fromLinkId FROM Link id
* @param toLinkId TO Link id
- * @param fromPart FROM Connector forms
- * @param toPart TO Connector forms
- * @param frameworkPart Framework forms
+ * @param fromConfig FROM job config
+ * @param toConfig TO job config
+ * @param driverConfig driver config
*/
public MJob(long fromConnectorId,
long toConnectorId,
- long fromConnectionId,
- long toConnectionId,
- MJobForms fromPart,
- MJobForms toPart,
- MJobForms frameworkPart) {
+ long fromLinkId,
+ long toLinkId,
+ MFromConfig fromConfig,
+ MToConfig toConfig,
+ MDriverConfig driverConfig) {
this.fromConnectorId = fromConnectorId;
this.toConnectorId = toConnectorId;
- this.fromLinkId = fromConnectionId;
- this.toLinkId = toConnectionId;
- this.fromConnectorPart = fromPart;
- this.toConnectorPart = toPart;
- this.frameworkPart = frameworkPart;
+ this.fromLinkId = fromLinkId;
+ this.toLinkId = toLinkId;
+ this.fromConfig = fromConfig;
+ this.toConfig = toConfig;
+ this.driverConfig = driverConfig;
}
/**
@@ -80,9 +74,9 @@ public class MJob extends MAccountableEntity implements MClonable {
*/
public MJob(MJob other) {
this(other,
- other.getConnectorPart(Direction.FROM).clone(true),
- other.getConnectorPart(Direction.TO).clone(true),
- other.frameworkPart.clone(true));
+ other.getFromJobConfig().clone(true),
+ other.getToJobConfig().clone(true),
+ other.driverConfig.clone(true));
}
/**
@@ -92,29 +86,29 @@ public class MJob extends MAccountableEntity implements MClonable {
* used otherwise.
*
* @param other MJob model to copy
- * @param fromPart FROM Connector forms
- * @param toPart TO Connector forms
- * @param frameworkPart Framework forms
+ * @param fromConfig FROM Job config
+ * @param toConfig TO Job config
+ * @param driverConfig driverConfig
*/
- public MJob(MJob other, MJobForms fromPart, MJobForms toPart, MJobForms frameworkPart) {
+ public MJob(MJob other, MFromConfig fromConfig, MToConfig toConfig, MDriverConfig driverConfig) {
super(other);
this.fromConnectorId = other.getConnectorId(Direction.FROM);
this.toConnectorId = other.getConnectorId(Direction.TO);
this.fromLinkId = other.getLinkId(Direction.FROM);
this.toLinkId = other.getLinkId(Direction.TO);
- this.fromConnectorPart = fromPart;
- this.toConnectorPart = toPart;
- this.frameworkPart = frameworkPart;
+ this.fromConfig = fromConfig;
+ this.toConfig = toConfig;
+ this.driverConfig = driverConfig;
this.setPersistenceId(other.getPersistenceId());
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("job");
- sb.append(" connector-from-part: ").append(getConnectorPart(Direction.FROM));
- sb.append(", connector-to-part: ").append(getConnectorPart(Direction.TO));
- sb.append(", framework-part: ").append(frameworkPart);
+ sb.append("From job config: ").append(getJobConfig(Direction.FROM));
+ sb.append(", To job config: ").append(getJobConfig(Direction.TO));
+ sb.append(", Driver config: ").append(driverConfig);
return sb.toString();
}
@@ -145,21 +139,29 @@ public class MJob extends MAccountableEntity implements MClonable {
}
}
- public MJobForms getConnectorPart(Direction type) {
+ public MConfigList getJobConfig(Direction type) {
switch(type) {
case FROM:
- return fromConnectorPart;
+ return fromConfig;
case TO:
- return toConnectorPart;
+ return toConfig;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
- public MJobForms getFrameworkPart() {
- return frameworkPart;
+ public MFromConfig getFromJobConfig() {
+ return fromConfig;
+ }
+
+ public MToConfig getToJobConfig() {
+ return toConfig;
+ }
+
+ public MDriverConfig getDriverConfig() {
+ return driverConfig;
}
@Override
@@ -172,9 +174,9 @@ public class MJob extends MAccountableEntity implements MClonable {
getConnectorId(Direction.TO),
getLinkId(Direction.FROM),
getLinkId(Direction.TO),
- getConnectorPart(Direction.FROM).clone(false),
- getConnectorPart(Direction.TO).clone(false),
- frameworkPart.clone(false));
+ getFromJobConfig().clone(false),
+ getToJobConfig().clone(false),
+ getDriverConfig().clone(false));
}
}
@@ -194,8 +196,8 @@ public class MJob extends MAccountableEntity implements MClonable {
&& (job.getLinkId(Direction.FROM) == this.getLinkId(Direction.FROM))
&& (job.getLinkId(Direction.TO) == this.getLinkId(Direction.TO))
&& (job.getPersistenceId() == this.getPersistenceId())
- && (job.getConnectorPart(Direction.FROM).equals(this.getConnectorPart(Direction.FROM)))
- && (job.getConnectorPart(Direction.TO).equals(this.getConnectorPart(Direction.TO)))
- && (job.frameworkPart.equals(this.frameworkPart));
+ && (job.getFromJobConfig().equals(this.getJobConfig(Direction.FROM)))
+ && (job.getToJobConfig().equals(this.getJobConfig(Direction.TO)))
+ && (job.getDriverConfig().equals(this.driverConfig));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MJobForms.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MJobForms.java b/common/src/main/java/org/apache/sqoop/model/MJobForms.java
deleted file mode 100644
index 08b9a78..0000000
--- a/common/src/main/java/org/apache/sqoop/model/MJobForms.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import java.util.List;
-
-/**
- * Metadata describing all required information to build a job
- * object with two connectors and a framework.
- */
-public class MJobForms extends MFormList {
- public MJobForms(List<MForm> forms) {
- super(forms);
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == this) {
- return true;
- }
-
- if (!(other instanceof MJobForms)) {
- return false;
- }
-
- MJobForms mj = (MJobForms) other;
- return super.equals(mj);
- }
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
-
- @Override
- public MJobForms clone(boolean cloneWithValue) {
- MJobForms copy = new MJobForms(super.clone(cloneWithValue).getForms());
- return copy;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MLink.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MLink.java b/common/src/main/java/org/apache/sqoop/model/MLink.java
index 6a8c424..7a9f538 100644
--- a/common/src/main/java/org/apache/sqoop/model/MLink.java
+++ b/common/src/main/java/org/apache/sqoop/model/MLink.java
@@ -22,57 +22,49 @@ package org.apache.sqoop.model;
*/
public class MLink extends MAccountableEntity implements MClonable {
private long connectorId;
-
- private final MConnectionForms connectorPart;
- private final MConnectionForms frameworkPart;
+ // NOTE: we hold this in the model for easy access to the link config object, it might as well be retrieved on the fly using the connectorId
+ private final MLinkConfig connectorLinkConfig;
/**
- * Default constructor to build new MConnection model.
+ * Default constructor to build new MLink model.
*
* @param connectorId Connector id
- * @param connectorPart Connector forms
- * @param frameworkPart Framework forms
+ * @param linkConfig Connector forms
*/
- public MLink(long connectorId,
- MConnectionForms connectorPart,
- MConnectionForms frameworkPart) {
+ public MLink(long connectorId, MLinkConfig linkConfig) {
this.connectorId = connectorId;
- this.connectorPart = connectorPart;
- this.frameworkPart = frameworkPart;
+ this.connectorLinkConfig = linkConfig;
}
/**
- * Constructor to create deep copy of another MConnection model.
+ * Constructor to create deep copy of another MLink model.
*
- * @param other MConnection model to copy
+ * @param other MLink model to copy
*/
public MLink(MLink other) {
- this(other, other.connectorPart.clone(true), other.frameworkPart.clone(true));
+ this(other, other.connectorLinkConfig.clone(true));
}
/**
- * Construct new MConnection model as a copy of another with replaced forms.
+ * Construct new MLink model as a copy of another with replaced forms.
*
- * This method is suitable only for metadata upgrade path and should not be
+ * This method is suitable only for link upgrade path and should not be
* used otherwise.
*
- * @param other MConnection model to copy
- * @param connectorPart Connector forms
- * @param frameworkPart Framework forms
+ * @param other MLink model to copy
+ * @param linkConfig link config
*/
- public MLink(MLink other, MConnectionForms connectorPart, MConnectionForms frameworkPart) {
+ public MLink(MLink other, MLinkConfig linkConfig) {
super(other);
this.connectorId = other.connectorId;
- this.connectorPart = connectorPart;
- this.frameworkPart = frameworkPart;
+ this.connectorLinkConfig = linkConfig;
this.setPersistenceId(other.getPersistenceId());
}
@Override
public String toString() {
- StringBuilder sb = new StringBuilder("connection: ").append(getName());
- sb.append(" connector-part: ").append(connectorPart);
- sb.append(", framework-part: ").append(frameworkPart);
+ StringBuilder sb = new StringBuilder("link: ").append(getName());
+ sb.append(" link-config: ").append(connectorLinkConfig);
return sb.toString();
}
@@ -85,20 +77,11 @@ public class MLink extends MAccountableEntity implements MClonable {
this.connectorId = connectorId;
}
- public MConnectionForms getConnectorPart() {
- return connectorPart;
- }
-
- public MConnectionForms getFrameworkPart() {
- return frameworkPart;
- }
-
- public MForm getConnectorForm(String formName) {
- return connectorPart.getForm(formName);
+ public MLinkConfig getConnectorLinkConfig() {
+ return connectorLinkConfig;
}
-
- public MForm getFrameworkForm(String formName) {
- return frameworkPart.getForm(formName);
+ public MConfig getConnectorLinkConfig(String formName) {
+ return connectorLinkConfig.getConfig(formName);
}
@Override
@@ -106,7 +89,7 @@ public class MLink extends MAccountableEntity implements MClonable {
if(cloneWithValue) {
return new MLink(this);
} else {
- return new MLink(connectorId, connectorPart.clone(false), frameworkPart.clone(false));
+ return new MLink(connectorId, connectorLinkConfig.clone(false));
}
}
@@ -120,10 +103,9 @@ public class MLink extends MAccountableEntity implements MClonable {
return false;
}
- MLink mc = (MLink)object;
- return (mc.connectorId == this.connectorId)
- && (mc.getPersistenceId() == this.getPersistenceId())
- && (mc.connectorPart.equals(this.connectorPart))
- && (mc.frameworkPart.equals(this.frameworkPart));
- }
+ MLink mLink = (MLink)object;
+ return (mLink.connectorId == this.connectorId)
+ && (mLink.getPersistenceId() == this.getPersistenceId())
+ && (mLink.connectorLinkConfig.equals(this.connectorLinkConfig));
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java b/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java
new file mode 100644
index 0000000..318b63c
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MLinkConfig.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.util.List;
+
+/**
+ * Config describing all required information to build up an link object
+ * NOTE: It extends a config list since {@link MLink} could consist of a related config groups
+ * In future this could be simplified to hold a single list of all configs for the link object
+ */
+public class MLinkConfig extends MConfigList {
+
+ public MLinkConfig(List<MConfig> configs) {
+ super(configs);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("Link: ");
+ sb.append(super.toString());
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ return super.equals(other);
+ }
+
+ @Override
+ public MLinkConfig clone(boolean cloneWithValue) {
+ MLinkConfig copy = new MLinkConfig(super.clone(cloneWithValue).getConfigs());
+ return copy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MToConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MToConfig.java b/common/src/main/java/org/apache/sqoop/model/MToConfig.java
new file mode 100644
index 0000000..b4fbe41
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MToConfig.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.util.List;
+
+/**
+ * Config describing all required information to build the TO part of the job
+ * NOTE: It extends a config list since {@link MToConfig} could consist of a related config groups
+ * In future this could be simplified to hold a single list of all configs for the TO object
+
+ */
+public class MToConfig extends MConfigList {
+ public MToConfig(List<MConfig> configs) {
+ super(configs);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("To: ");
+ sb.append(super.toString());
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (!(other instanceof MToConfig)) {
+ return false;
+ }
+
+ MToConfig mj = (MToConfig) other;
+ return super.equals(mj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public MToConfig clone(boolean cloneWithValue) {
+ MToConfig copy = new MToConfig(super.clone(cloneWithValue).getConfigs());
+ return copy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/ModelError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/ModelError.java b/common/src/main/java/org/apache/sqoop/model/ModelError.java
index 74e924e..35a8943 100644
--- a/common/src/main/java/org/apache/sqoop/model/ModelError.java
+++ b/common/src/main/java/org/apache/sqoop/model/ModelError.java
@@ -24,7 +24,7 @@ import org.apache.sqoop.common.ErrorCode;
*/
public enum ModelError implements ErrorCode {
- MODEL_001("Attempt to pass two different set of MForms for single job type."),
+ MODEL_001("Attempt to pass two different set of MConfigs for single job type."),
MODEL_002("Creating MJob of different job types"),
@@ -34,7 +34,7 @@ public enum ModelError implements ErrorCode {
MODEL_005("Can't get field value"),
- MODEL_006("Incompatible form list and configuration object"),
+ MODEL_006("Incompatible config list and configuration object"),
MODEL_007("Primitive types in configuration objects are not allowed"),
@@ -42,7 +42,7 @@ public enum ModelError implements ErrorCode {
MODEL_009("Invalid input name"),
- MODEL_010("Form do not exist"),
+ MODEL_010("Config do not exist"),
MODEL_011("Input do not exist"),
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
index eca9f7e..0be4d41 100644
--- a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
+++ b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
@@ -17,14 +17,12 @@
*/
package org.apache.sqoop.utils;
-import org.apache.log4j.Logger;
-
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
+
+import org.apache.log4j.Logger;
public final class ClassUtils {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/ConfigValidationError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ConfigValidationError.java b/common/src/main/java/org/apache/sqoop/validation/ConfigValidationError.java
new file mode 100644
index 0000000..3453648
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/validation/ConfigValidationError.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation;
+
+import org.apache.sqoop.common.ErrorCode;
+
+/**
+ *
+ */
+public enum ConfigValidationError implements ErrorCode {
+
+ VALIDATION_0000("Unknown error"),
+
+ VALIDATION_0001("Missing class declaration."),
+
+ VALIDATION_0002("Usage of missing field"),
+
+ VALIDATION_0003("Invalid representation of config and input field"),
+
+ VALIDATION_0004("Can't find validator class"),
+
+ ;
+
+ private final String message;
+
+ private ConfigValidationError(String message) {
+ this.message = message;
+ }
+
+ public String getCode() {
+ return name();
+ }
+
+ public String getMessage() {
+ return message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/ConfigValidationResult.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ConfigValidationResult.java b/common/src/main/java/org/apache/sqoop/validation/ConfigValidationResult.java
new file mode 100644
index 0000000..4c4d123
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/validation/ConfigValidationResult.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation;
+
+import org.apache.sqoop.validation.validators.AbstractValidator;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Result of validation execution.
+ */
+public class ConfigValidationResult {
+
+ /**
+ * All messages for each named item.
+ */
+ Map<String, List<Message>> messages;
+
+ /**
+ * Overall status.
+ */
+ Status status;
+
+ public ConfigValidationResult() {
+ messages = new HashMap<String, List<Message>>();
+ status = Status.getDefault();
+ }
+
+ /**
+ * Add given validator result to this instance.
+ *
+ * @param name Full name of the validated object
+ * @param validator Executed validator
+ */
+ public void addValidatorResult(String name, AbstractValidator<String> validator) {
+ if(validator.getStatus() == Status.getDefault()) {
+ return;
+ }
+
+ status = Status.getWorstStatus(status, validator.getStatus());
+ if(messages.containsKey(name)) {
+ messages.get(name).addAll(validator.getMessages());
+ } else {
+ messages.put(name, validator.getMessages());
+ }
+ }
+
+ /**
+ * Merge results with another validation result.
+ *
+ * @param result Other validation result
+ */
+ public void mergeValidatorResult(ConfigValidationResult result) {
+ messages.putAll(result.messages);
+ status = Status.getWorstStatus(status, result.status);
+ }
+
+ /**
+ * Method to directly add messages for given name.
+ *
+ * This method will replace previous messages for given name.
+ *
+ * @param name Name of the entity
+ * @param messages List of messages associated with the name
+ */
+ public void addMessages(String name, List<Message> messages) {
+ this.messages.put(name, messages);
+
+ for(Message message : messages) {
+ this.status = Status.getWorstStatus(status, message.getStatus());
+ }
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public Map<String, List<Message>> getMessages() {
+ return messages;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/ConfigValidationRunner.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ConfigValidationRunner.java b/common/src/main/java/org/apache/sqoop/validation/ConfigValidationRunner.java
new file mode 100644
index 0000000..8c66b3d
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/validation/ConfigValidationRunner.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation;
+
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.model.ConfigurationClass;
+import org.apache.sqoop.model.Config;
+import org.apache.sqoop.model.ConfigClass;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
+import org.apache.sqoop.utils.ClassUtils;
+import org.apache.sqoop.validation.validators.AbstractValidator;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Validation runner that will run validators associated with given configuration
+ * class or config object.
+ *
+ * Execution follows following rules:
+ * * Run children first (Inputs -> Config -> Class)
+ * * If any children is not suitable (canProceed = false), skip running parent
+ *
+ * Which means that config validator don't have to repeat it's input validators as it will
+ * be never called if the input's are not valid. Similarly Class validators won't be called
+ * unless all configs will pass validators.
+ *
+ */
+public class ConfigValidationRunner {
+
+ /**
+ * Private cache of instantiated validators.
+ *
+ * We're expecting that this cache will be very small as the number of possible validators
+ * is driven to high extent by the number of connectors and hence we don't have a cache
+ * eviction at the moment.
+ */
+ private Map<Class<? extends AbstractValidator>, AbstractValidator> cache;
+
+ public ConfigValidationRunner() {
+ cache = new HashMap<Class<? extends AbstractValidator>, AbstractValidator>();
+ }
+
+ /**
+ * Validate given configuration instance.
+ *
+ * @param config Configuration instance
+ * @return
+ */
+ public ConfigValidationResult validate(Object config) {
+ ConfigValidationResult result = new ConfigValidationResult();
+ ConfigurationClass globalAnnotation = ConfigUtils.getConfigurationClassAnnotation(config, true);
+
+ // Iterate over all declared config and call their validators
+ for (Field field : config.getClass().getDeclaredFields()) {
+ field.setAccessible(true);
+
+ Config configAnnotation = ConfigUtils.getConfigAnnotation(field, false);
+ if(configAnnotation == null) {
+ continue;
+ }
+
+ String configName = ConfigUtils.getName(field, configAnnotation);
+ ConfigValidationResult r = validateConfig(configName, ConfigUtils.getFieldValue(field, config));
+ result.mergeValidatorResult(r);
+ }
+
+ // Call class validator only as long as we are in suitable state
+ if(result.getStatus().canProceed()) {
+ ConfigValidationResult r = validateArray("", config, globalAnnotation.validators());
+ result.mergeValidatorResult(r);
+ }
+
+ return result;
+ }
+
+ /**
+ * Validate given config instance.
+ *
+ * @param configName Config's name to build full name for all inputs.
+ * @param config Config instance
+ * @return
+ */
+ public ConfigValidationResult validateConfig(String configName, Object config) {
+ ConfigValidationResult result = new ConfigValidationResult();
+ ConfigClass configAnnotation = ConfigUtils.getConfigClassAnnotation(config, true);
+
+ // Iterate over all declared inputs and call their validators
+ for (Field field : config.getClass().getDeclaredFields()) {
+ Input inputAnnotation = ConfigUtils.getInputAnnotation(field, false);
+ if(inputAnnotation == null) {
+ continue;
+ }
+
+ String name = configName + "." + ConfigUtils.getName(field, inputAnnotation);
+
+ ConfigValidationResult r = validateArray(name, ConfigUtils.getFieldValue(field, config), inputAnnotation.validators());
+ result.mergeValidatorResult(r);
+ }
+
+ // Call config validator only as long as we are in suitable state
+ if(result.getStatus().canProceed()) {
+ ConfigValidationResult r = validateArray(configName, config, configAnnotation.validators());
+ result.mergeValidatorResult(r);
+ }
+
+ return result;
+ }
+
+ /**
+ * Execute array of validators on given object (can be input/config/class).
+ *
+ * @param name Full name of the object
+ * @param object Input, Config or Class instance
+ * @param validators Validators array
+ * @return
+ */
+ private ConfigValidationResult validateArray(String name, Object object, Validator[] validators) {
+ ConfigValidationResult result = new ConfigValidationResult();
+
+ for (Validator validator : validators) {
+ AbstractValidator v = executeValidator(object, validator);
+ result.addValidatorResult(name, v);
+ }
+
+ return result;
+ }
+
+ /**
+ * Execute single validator.
+ *
+ * @param object Input, Config or Class instance
+ * @param validator Validator annotation
+ * @return
+ */
+ private AbstractValidator executeValidator(Object object, Validator validator) {
+ // Try to get validator instance from the cache
+ AbstractValidator instance = cache.get(validator.value());
+
+ if(instance == null) {
+ instance = (AbstractValidator) ClassUtils.instantiate(validator.value());
+
+ // This could happen if we would be missing some connector's jars on our classpath
+ if(instance == null) {
+ throw new SqoopException(ConfigValidationError.VALIDATION_0004, validator.value().getName());
+ }
+
+ cache.put(validator.value(), instance);
+ } else {
+ instance.reset();
+ }
+
+ instance.setStringArgument(validator.strArg());
+ instance.validate(object);
+ return instance;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/ConfigValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ConfigValidator.java b/common/src/main/java/org/apache/sqoop/validation/ConfigValidator.java
new file mode 100644
index 0000000..eac789e
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/validation/ConfigValidator.java
@@ -0,0 +1,228 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation;
+
+import org.apache.sqoop.common.SqoopException;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Config validators.
+ *
+ * This class represents validations for the sqoop objects
+ */
+public class ConfigValidator {
+
+ // Configuration class that belongs to this validation
+ Class klass;
+
+ // Entire validation status
+ Status status;
+
+ // Status messages for various fields
+ Map<ConfigInput, Message> messages;
+
+ public ConfigValidator(Class klass) {
+ this.klass = klass;
+ status = Status.getDefault();
+ messages = new HashMap<ConfigInput, Message>();
+ }
+
+ public ConfigValidator(Status status, Map<ConfigInput, Message> messages) {
+ this.status = status;
+ this.messages = messages;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public Map<ConfigInput, Message> getMessages() {
+ return messages;
+ }
+
+ /**
+ * Add message to config.
+ *
+ * @param status Severity of the message
+ * @param config Config name, must be defined in the class
+ * @param message Validation message
+ */
+ public void addMessage(Status status, String config, String message) {
+ addMessage(status, config, null, message);
+ }
+
+ /**
+ * Add message to input in one of the configs.
+ *
+ * @param status Severity of the message
+ * @param config Config name, must be defined in the class
+ * @param input Field name, must be defined in the config class
+ * @param message Validation message
+ */
+ public void addMessage(Status status, String config, String input, String message ) {
+ if( klass == null) {
+ throw new SqoopException(ConfigValidationError.VALIDATION_0001);
+ }
+
+ assert config != null;
+ assert message != null;
+
+ // Field for specified config
+ Field configField;
+
+ // Load the config field and verify that it exists
+ try {
+ configField = klass.getDeclaredField(config);
+ } catch (NoSuchFieldException e) {
+ throw new SqoopException(ConfigValidationError.VALIDATION_0002,
+ "Can't get config " + config + " from " + klass.getName(), e);
+ }
+
+ // If this is config message, just save the message and continue
+ if(input == null) {
+ setMessage(status, config, input, message);
+ return;
+ }
+
+ // Verify that specified input exists on the config
+ try {
+ configField.getType().getDeclaredField(input);
+ } catch (NoSuchFieldException e) {
+ throw new SqoopException(ConfigValidationError.VALIDATION_0002,
+ "Can't get input " + input + " from config" + configField.getType().getName(), e);
+ }
+
+ setMessage(status, config, input, message);
+ }
+
+ private void setMessage(Status status, String config, String input, String message) {
+ this.status = Status.getWorstStatus(this.status, status);
+ messages.put(new ConfigInput(config, input), new Message(status, message));
+ }
+
+ public static class Message {
+ private Status status;
+ private String message;
+
+ public Message(Status status, String message) {
+ this.status = status;
+ this.message = message;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof Message)) return false;
+
+ Message message1 = (Message) o;
+
+ if (message != null ? !message.equals(message1.message) : message1.message != null)
+ return false;
+ if (status != message1.status) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = status != null ? status.hashCode() : 0;
+ result = 31 * result + (message != null ? message.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "{" + status.name() + ": " + message + "}";
+ }
+ }
+
+ public static class ConfigInput{
+ private String config;
+ private String input;
+
+ public ConfigInput(String config, String input) {
+ this.config = config;
+ this.input = input;
+ }
+
+ public ConfigInput(String configInput) {
+ assert configInput != null;
+ String []parts = configInput.split("\\.");
+
+ if(configInput.isEmpty() || (parts.length != 1 && parts.length != 2)) {
+ throw new SqoopException(ConfigValidationError.VALIDATION_0003,
+ "Specification " + configInput + " is not in valid configat config.input");
+ }
+
+ this.config = parts[0];
+ if(parts.length == 2) {
+ this.input = parts[1];
+ }
+ }
+
+ public String getConfig() {
+ return config;
+ }
+
+ public String getInput() {
+ return input;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ConfigInput configInput = (ConfigInput) o;
+
+ if (config != null ? !config.equals(configInput.config) : configInput.config != null)
+ return false;
+ if (input != null ? !input.equals(configInput.input) : configInput.input != null)
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = config != null ? config.hashCode() : 0;
+ result = 31 * result + (input != null ? input.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ if(input == null) {
+ return config;
+ }
+
+ return config + "." + input;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/Message.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/Message.java b/common/src/main/java/org/apache/sqoop/validation/Message.java
index cb55b6a..3361b6f 100644
--- a/common/src/main/java/org/apache/sqoop/validation/Message.java
+++ b/common/src/main/java/org/apache/sqoop/validation/Message.java
@@ -21,7 +21,7 @@ package org.apache.sqoop.validation;
* Validation message.
*
* Validation message have always two parts - severity and textual information about what
- * is wrong. It can be associated with Input, Form or Configuration class.
+ * is wrong. It can be associated with Input, Config or ConfigurationGroup class.
*/
public class Message {
private Status status;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/Validation.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/Validation.java b/common/src/main/java/org/apache/sqoop/validation/Validation.java
deleted file mode 100644
index fce6e88..0000000
--- a/common/src/main/java/org/apache/sqoop/validation/Validation.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.validation;
-
-import org.apache.sqoop.common.SqoopException;
-
-import java.lang.reflect.Field;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Validation class.
- *
- * This class represents validations to given configuration object.
- */
-public class Validation {
-
- // Configuration class that belongs to this validation
- Class klass;
-
- // Entire validation status
- Status status;
-
- // Status messages for various fields
- Map<FormInput, Message> messages;
-
- public Validation(Class klass) {
- this.klass = klass;
- status = Status.getDefault();
- messages = new HashMap<FormInput, Message>();
- }
-
- public Validation(Status status, Map<FormInput, Message> messages) {
- this.status = status;
- this.messages = messages;
- }
-
- public Status getStatus() {
- return status;
- }
-
- public Map<FormInput, Message> getMessages() {
- return messages;
- }
-
- /**
- * Add message to form.
- *
- * @param status Severity of the message
- * @param form Form name, must be defined in the class
- * @param message Validation message
- */
- public void addMessage(Status status, String form, String message) {
- addMessage(status, form, null, message);
- }
-
- /**
- * Add message to input in one of the forms.
- *
- * @param status Severity of the message
- * @param form Form name, must be defined in the class
- * @param input Field name, must be defined in the form class
- * @param message Validation message
- */
- public void addMessage(Status status, String form, String input, String message ) {
- if( klass == null) {
- throw new SqoopException(ValidationError.VALIDATION_0001);
- }
-
- assert form != null;
- assert message != null;
-
- // Field for specified form
- Field formField;
-
- // Load the form field and verify that it exists
- try {
- formField = klass.getDeclaredField(form);
- } catch (NoSuchFieldException e) {
- throw new SqoopException(ValidationError.VALIDATION_0002,
- "Can't get form " + form + " from " + klass.getName(), e);
- }
-
- // If this is form message, just save the message and continue
- if(input == null) {
- setMessage(status, form, input, message);
- return;
- }
-
- // Verify that specified input exists on the form
- try {
- formField.getType().getDeclaredField(input);
- } catch (NoSuchFieldException e) {
- throw new SqoopException(ValidationError.VALIDATION_0002,
- "Can't get input " + input + " from form" + formField.getType().getName(), e);
- }
-
- setMessage(status, form, input, message);
- }
-
- private void setMessage(Status status, String form, String input, String message) {
- this.status = Status.getWorstStatus(this.status, status);
- messages.put(new FormInput(form, input), new Message(status, message));
- }
-
- public static class Message {
- private Status status;
- private String message;
-
- public Message(Status status, String message) {
- this.status = status;
- this.message = message;
- }
-
- public Status getStatus() {
- return status;
- }
-
- public String getMessage() {
- return message;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof Message)) return false;
-
- Message message1 = (Message) o;
-
- if (message != null ? !message.equals(message1.message) : message1.message != null)
- return false;
- if (status != message1.status) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = status != null ? status.hashCode() : 0;
- result = 31 * result + (message != null ? message.hashCode() : 0);
- return result;
- }
-
- @Override
- public String toString() {
- return "{" + status.name() + ": " + message + "}";
- }
- }
-
- public static class FormInput {
- private String form;
- private String input;
-
- public FormInput(String form, String input) {
- this.form = form;
- this.input = input;
- }
-
- public FormInput(String formInput) {
- assert formInput != null;
- String []parts = formInput.split("\\.");
-
- if(formInput.isEmpty() || (parts.length != 1 && parts.length != 2)) {
- throw new SqoopException(ValidationError.VALIDATION_0003,
- "Specification " + formInput + " is not in valid format form.input");
- }
-
- this.form = parts[0];
- if(parts.length == 2) {
- this.input = parts[1];
- }
- }
-
- public String getForm() {
- return form;
- }
-
- public String getInput() {
- return input;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- FormInput formInput = (FormInput) o;
-
- if (form != null ? !form.equals(formInput.form) : formInput.form != null)
- return false;
- if (input != null ? !input.equals(formInput.input) : formInput.input != null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = form != null ? form.hashCode() : 0;
- result = 31 * result + (input != null ? input.hashCode() : 0);
- return result;
- }
-
- @Override
- public String toString() {
- if(input == null) {
- return form;
- }
-
- return form + "." + input;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/ValidationError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationError.java b/common/src/main/java/org/apache/sqoop/validation/ValidationError.java
deleted file mode 100644
index ec64f10..0000000
--- a/common/src/main/java/org/apache/sqoop/validation/ValidationError.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.validation;
-
-import org.apache.sqoop.common.ErrorCode;
-
-/**
- *
- */
-public enum ValidationError implements ErrorCode {
-
- VALIDATION_0000("Unknown error"),
-
- VALIDATION_0001("Missing class declaration."),
-
- VALIDATION_0002("Usage of missing field"),
-
- VALIDATION_0003("Invalid representation of form and input field"),
-
- VALIDATION_0004("Can't find validator class"),
-
- ;
-
- private final String message;
-
- private ValidationError(String message) {
- this.message = message;
- }
-
- public String getCode() {
- return name();
- }
-
- public String getMessage() {
- return message;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java b/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java
deleted file mode 100644
index ae8f1d1..0000000
--- a/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.validation;
-
-import org.apache.sqoop.validation.validators.AbstractValidator;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Result of validation execution.
- */
-public class ValidationResult {
-
- /**
- * All messages for each named item.
- */
- Map<String, List<Message>> messages;
-
- /**
- * Overall status.
- */
- Status status;
-
- public ValidationResult() {
- messages = new HashMap<String, List<Message>>();
- status = Status.getDefault();
- }
-
- /**
- * Add given validator result to this instance.
- *
- * @param name Full name of the validated object
- * @param validator Executed validator
- */
- public void addValidator(String name, AbstractValidator validator) {
- if(validator.getStatus() == Status.getDefault()) {
- return;
- }
-
- status = Status.getWorstStatus(status, validator.getStatus());
- if(messages.containsKey(name)) {
- messages.get(name).addAll(validator.getMessages());
- } else {
- messages.put(name, validator.getMessages());
- }
- }
-
- /**
- * Merge results with another validation result.
- *
- * @param result Other validation result
- */
- public void merge(ValidationResult result) {
- messages.putAll(result.messages);
- status = Status.getWorstStatus(status, result.status);
- }
-
- /**
- * Method to directly add messages for given name.
- *
- * This method will replace previous messages for given name.
- *
- * @param name Name of the entity
- * @param messages List of messages associated with the name
- */
- public void addMessages(String name, List<Message> messages) {
- this.messages.put(name, messages);
-
- for(Message message : messages) {
- this.status = Status.getWorstStatus(status, message.getStatus());
- }
- }
-
- public Status getStatus() {
- return status;
- }
-
- public Map<String, List<Message>> getMessages() {
- return messages;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java b/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
deleted file mode 100644
index 8ffc0d4..0000000
--- a/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.validation;
-
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.Input;
-import org.apache.sqoop.model.Validator;
-import org.apache.sqoop.utils.ClassUtils;
-import org.apache.sqoop.validation.validators.AbstractValidator;
-
-import java.lang.reflect.Field;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Validation runner that will run validators associated with given configuration
- * class or form object.
- *
- * Execution follows following rules:
- * * Run children first (Inputs -> Form -> Class)
- * * If any children is not suitable (canProceed = false), skip running parent
- *
- * Which means that form validator don't have to repeat it's input validators as it will
- * be never called if the input's are not valid. Similarly Class validators won't be called
- * unless all forms will pass validators.
- */
-public class ValidationRunner {
-
- /**
- * Private cache of instantiated validators.
- *
- * We're expecting that this cache will be very small as the number of possible validators
- * is driven to high extent by the number of connectors and hence we don't have a cache
- * eviction at the moment.
- */
- private Map<Class<? extends AbstractValidator>, AbstractValidator> cache;
-
- public ValidationRunner() {
- cache = new HashMap<Class<? extends AbstractValidator>, AbstractValidator>();
- }
-
- /**
- * Validate given configuration instance.
- *
- * @param config Configuration instance
- * @return
- */
- public ValidationResult validate(Object config) {
- ValidationResult result = new ValidationResult();
- ConfigurationClass globalAnnotation = FormUtils.getConfigurationClassAnnotation(config, true);
-
- // Iterate over all declared form and call their validators
- for (Field field : config.getClass().getDeclaredFields()) {
- field.setAccessible(true);
-
- Form formAnnotation = FormUtils.getFormAnnotation(field, false);
- if(formAnnotation == null) {
- continue;
- }
-
- String formName = FormUtils.getName(field, formAnnotation);
- ValidationResult r = validateForm(formName, FormUtils.getFieldValue(field, config));
- result.merge(r);
- }
-
- // Call class validator only as long as we are in suitable state
- if(result.getStatus().canProceed()) {
- ValidationResult r = validateArray("", config, globalAnnotation.validators());
- result.merge(r);
- }
-
- return result;
- }
-
- /**
- * Validate given form instance.
- *
- * @param formName Form's name to build full name for all inputs.
- * @param form Form instance
- * @return
- */
- public ValidationResult validateForm(String formName, Object form) {
- ValidationResult result = new ValidationResult();
- FormClass formAnnotation = FormUtils.getFormClassAnnotation(form, true);
-
- // Iterate over all declared inputs and call their validators
- for (Field field : form.getClass().getDeclaredFields()) {
- Input inputAnnotation = FormUtils.getInputAnnotation(field, false);
- if(inputAnnotation == null) {
- continue;
- }
-
- String name = formName + "." + FormUtils.getName(field, inputAnnotation);
-
- ValidationResult r = validateArray(name, FormUtils.getFieldValue(field, form), inputAnnotation.validators());
- result.merge(r);
- }
-
- // Call form validator only as long as we are in suitable state
- if(result.getStatus().canProceed()) {
- ValidationResult r = validateArray(formName, form, formAnnotation.validators());
- result.merge(r);
- }
-
- return result;
- }
-
- /**
- * Execute array of validators on given object (can be input/form/class).
- *
- * @param name Full name of the object
- * @param object Input, Form or Class instance
- * @param validators Validators array
- * @return
- */
- private ValidationResult validateArray(String name, Object object, Validator[] validators) {
- ValidationResult result = new ValidationResult();
-
- for (Validator validator : validators) {
- AbstractValidator v = executeValidator(object, validator);
- result.addValidator(name, v);
- }
-
- return result;
- }
-
- /**
- * Execute single validator.
- *
- * @param object Input, Form or Class instance
- * @param validator Validator annotation
- * @return
- */
- private AbstractValidator executeValidator(Object object, Validator validator) {
- // Try to get validator instance from the cache
- AbstractValidator instance = cache.get(validator.value());
-
- if(instance == null) {
- instance = (AbstractValidator) ClassUtils.instantiate(validator.value());
-
- // This could happen if we would be missing some connector's jars on our classpath
- if(instance == null) {
- throw new SqoopException(ValidationError.VALIDATION_0004, validator.value().getName());
- }
-
- cache.put(validator.value(), instance);
- } else {
- instance.reset();
- }
-
- instance.setStringArgument(validator.strArg());
- instance.validate(object);
- return instance;
- }
-
-
-}
[51/52] [abbrv] git commit: SQOOP-1573: Sqoop2: From/To: Fix driver
labels
Posted by ab...@apache.org.
SQOOP-1573: Sqoop2: From/To: Fix driver labels
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/fc8eb9b5
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/fc8eb9b5
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/fc8eb9b5
Branch: refs/heads/SQOOP-1367
Commit: fc8eb9b54fef270a9b308ef5e87072a39dd9ff30
Parents: fc74316
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Oct 9 13:59:05 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 18:22:51 2014 -0700
----------------------------------------------------------------------
core/src/main/resources/driver-config.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc8eb9b5/core/src/main/resources/driver-config.properties
----------------------------------------------------------------------
diff --git a/core/src/main/resources/driver-config.properties b/core/src/main/resources/driver-config.properties
index 78c3b38..e005775 100644
--- a/core/src/main/resources/driver-config.properties
+++ b/core/src/main/resources/driver-config.properties
@@ -19,7 +19,7 @@
# Throttling From
#
-throttthrottlingConfigling.label = Throttling resources
+throttlingConfig.label = Throttling resources
throttlingConfig.help = Set throttling boundaries to not overload your systems
throttlingConfig.numExtractors.label = Extractors
[49/52] [abbrv] git commit: SQOOP-1562: Sqoop2: BIT handling in CSV
IDF
Posted by ab...@apache.org.
SQOOP-1562: Sqoop2: BIT handling in CSV IDF
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c78b75a6
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c78b75a6
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c78b75a6
Branch: refs/heads/SQOOP-1367
Commit: c78b75a645fa77e55809da57fedee3b0dea2fe60
Parents: 1e9db01
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Mon Oct 6 06:46:46 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 18:22:51 2014 -0700
----------------------------------------------------------------------
.../idf/CSVIntermediateDataFormat.java | 3 ++-
.../idf/TestCSVIntermediateDataFormat.java | 25 +++++++++++++++++++-
2 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c78b75a6/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
index 3c3543c..7eb1d8d 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
@@ -223,7 +223,8 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
out[i] = LocalDateTime.parse(fields[i]);
break;
case BIT:
- out[i] = fields[i];
+ out[i] = Boolean.valueOf(fields[i].equals("1")
+ || fields[i].toLowerCase().equals("true"));
break;
default:
throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0004, "Column type from schema was not recognized for " + colType);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c78b75a6/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
index 3159883..74f2e07 100644
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
@@ -19,6 +19,7 @@
package org.apache.sqoop.connector.idf;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -28,6 +29,7 @@ import java.util.Arrays;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Binary;
+import org.apache.sqoop.schema.type.Bit;
import org.apache.sqoop.schema.type.Date;
import org.apache.sqoop.schema.type.DateTime;
import org.apache.sqoop.schema.type.FixedPoint;
@@ -195,7 +197,7 @@ public class TestCSVIntermediateDataFormat {
Object[] in = {strData};
Object[] inCopy = new Object[1];
- System.arraycopy(in,0,inCopy,0,in.length);
+ System.arraycopy(in, 0, inCopy, 0, in.length);
// Modifies the input array, so we use the copy to confirm
data.setObjectData(in);
@@ -249,6 +251,27 @@ public class TestCSVIntermediateDataFormat {
}
}
+ @Test
+ public void testBit() {
+ Schema schema = new Schema("test");
+ schema.addColumn(new Bit("1"));
+ data.setSchema(schema);
+
+ for (String trueBit : new String[]{
+ "true", "TRUE", "1"
+ }) {
+ data.setTextData(trueBit);
+ assertTrue((Boolean) data.getObjectData()[0]);
+ }
+
+ for (String falseBit : new String[]{
+ "false", "FALSE", "0"
+ }) {
+ data.setTextData(falseBit);
+ assertFalse((Boolean) data.getObjectData()[0]);
+ }
+ }
+
@Test(expected=SqoopException.class)
public void testEmptySchema() {
String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
[31/52] [abbrv] git commit: SQOOP-1478: Sqoop2: Migrate remaining
tests from junit 3 to 4
Posted by ab...@apache.org.
SQOOP-1478: Sqoop2: Migrate remaining tests from junit 3 to 4
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/97da12aa
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/97da12aa
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/97da12aa
Branch: refs/heads/SQOOP-1367
Commit: 97da12aa5043e1d2f1fcfaf2531670c13aa0c2ae
Parents: 196346d
Author: Chiwan Park <ch...@icloud.com>
Authored: Mon Sep 29 11:04:48 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:59:24 2014 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/common/TestMapContext.java | 3 +-
.../apache/sqoop/json/TestConnectorBean.java | 3 +-
.../apache/sqoop/json/TestDriverConfigBean.java | 2 --
.../java/org/apache/sqoop/json/TestJobBean.java | 2 +-
.../org/apache/sqoop/json/TestLinkBean.java | 2 +-
.../org/apache/sqoop/json/TestSchemaBean.java | 2 --
.../apache/sqoop/json/TestSubmissionBean.java | 30 ++++++++++++++---
.../sqoop/json/TestValidationResultBean.java | 4 +--
.../org/apache/sqoop/model/TestFormUtils.java | 12 +++++--
.../sqoop/model/TestMAccountableEntity.java | 4 +--
.../apache/sqoop/model/TestMBooleanInput.java | 1 -
.../sqoop/model/TestMConnectionForms.java | 4 +--
.../org/apache/sqoop/model/TestMConnector.java | 4 +--
.../org/apache/sqoop/model/TestMEnumInput.java | 4 +--
.../java/org/apache/sqoop/model/TestMForm.java | 3 +-
.../org/apache/sqoop/model/TestMFormList.java | 2 +-
.../apache/sqoop/model/TestMIntegerInput.java | 4 +--
.../java/org/apache/sqoop/model/TestMJob.java | 4 +--
.../org/apache/sqoop/model/TestMJobForms.java | 4 +--
.../java/org/apache/sqoop/model/TestMLink.java | 4 +--
.../org/apache/sqoop/model/TestMMapInput.java | 10 +++---
.../apache/sqoop/model/TestMNamedElement.java | 4 +--
.../sqoop/model/TestMPersistableEntity.java | 3 +-
.../apache/sqoop/model/TestMStringInput.java | 4 +--
.../sqoop/model/TestMValidatedElement.java | 4 +--
.../sqoop/submission/TestSubmissionStatus.java | 10 ++++--
.../submission/counter/TestCounterGroup.java | 2 +-
.../sqoop/submission/counter/TestCounters.java | 2 +-
.../org/apache/sqoop/utils/TestClassUtils.java | 1 -
.../org/apache/sqoop/validation/TestStatus.java | 2 +-
.../apache/sqoop/validation/TestValidation.java | 8 +++--
.../validation/validators/TestContains.java | 2 +-
.../validators/TestNullOrContains.java | 2 +-
.../validation/validators/TestStartsWith.java | 2 +-
.../connector/jdbc/GenericJdbcExecutorTest.java | 12 +++++--
.../sqoop/connector/jdbc/TestExtractor.java | 20 ++++++++----
.../connector/jdbc/TestFromInitializer.java | 24 ++++++++++----
.../apache/sqoop/connector/jdbc/TestLoader.java | 6 ++--
.../sqoop/connector/jdbc/TestPartitioner.java | 25 +++++++++++---
.../sqoop/connector/jdbc/TestToInitializer.java | 26 ++++++++++++---
.../org/apache/sqoop/driver/TestJobManager.java | 14 ++++----
.../org/apache/sqoop/driver/TestJobRequest.java | 6 ++--
.../sqoop/repository/TestJdbcRepository.java | 34 ++++++++++----------
.../java/org/apache/sqoop/job/JobUtils.java | 2 +-
.../org/apache/sqoop/job/TestMapReduce.java | 15 +++++----
.../apache/sqoop/job/io/SqoopWritableTest.java | 10 ++++--
.../java/org/apache/sqoop/job/io/TestData.java | 9 +++---
.../mr/TestSqoopOutputFormatLoadExecutor.java | 2 +-
.../sqoop/repository/derby/DerbyTestCase.java | 16 ++++-----
.../repository/derby/TestConnectorHandling.java | 11 ++++++-
.../derby/TestDriverConfigHandling.java | 11 ++++++-
.../sqoop/repository/derby/TestInputTypes.java | 10 +++++-
.../sqoop/repository/derby/TestInternals.java | 11 ++++++-
.../sqoop/repository/derby/TestJobHandling.java | 14 +++++++-
.../repository/derby/TestLinkHandling.java | 14 +++++++-
.../derby/TestSubmissionHandling.java | 12 ++++++-
56 files changed, 310 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/common/TestMapContext.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/common/TestMapContext.java b/common/src/test/java/org/apache/sqoop/common/TestMapContext.java
index f4718c0..7ce1ccd 100644
--- a/common/src/test/java/org/apache/sqoop/common/TestMapContext.java
+++ b/common/src/test/java/org/apache/sqoop/common/TestMapContext.java
@@ -20,8 +20,7 @@ package org.apache.sqoop.common;
import java.util.HashMap;
import java.util.Map;
-import junit.framework.Assert;
-
+import org.junit.Assert;
import org.junit.Test;
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
index c3aa84f..fe75ee0 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
@@ -17,8 +17,6 @@
*/
package org.apache.sqoop.json;
-import static org.junit.Assert.*;
-
import org.apache.sqoop.model.MConnector;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
@@ -31,6 +29,7 @@ import java.util.Map;
import java.util.ResourceBundle;
import static org.apache.sqoop.json.TestUtil.*;
+import static org.junit.Assert.*;
/**
*
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java b/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
index 4309f10..fcce7b5 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
@@ -25,8 +25,6 @@ import org.junit.Test;
import java.util.ResourceBundle;
import static org.apache.sqoop.json.TestUtil.*;
-
-
import static org.junit.Assert.*;
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestJobBean.java b/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
index c6461bf..78a3420 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
@@ -27,8 +27,8 @@ import org.junit.Test;
import java.util.Date;
-import static junit.framework.Assert.assertEquals;
import static org.apache.sqoop.json.TestUtil.getJob;
+import static org.junit.Assert.assertEquals;
/**
*
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java b/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
index 8235275..9ca6b64 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
@@ -27,7 +27,7 @@ import org.junit.Test;
import java.util.Date;
-import static junit.framework.Assert.*;
+import static org.junit.Assert.*;
import static org.apache.sqoop.json.TestUtil.*;
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/json/TestSchemaBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestSchemaBean.java b/common/src/test/java/org/apache/sqoop/json/TestSchemaBean.java
index 5daab76..c7a7b2e 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestSchemaBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestSchemaBean.java
@@ -17,13 +17,11 @@
*/
package org.apache.sqoop.json;
-
import org.apache.sqoop.json.util.TestSchemaSerialization;
import org.apache.sqoop.schema.Schema;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
-
/**
* Run the same tests as TestSchemaSerialization, but using the SchamaBean
* as a means of transfer.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java b/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
index 518c9cb..e4d50bf 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
@@ -17,7 +17,6 @@
*/
package org.apache.sqoop.json;
-import junit.framework.TestCase;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Decimal;
@@ -28,16 +27,23 @@ import org.apache.sqoop.submission.counter.CounterGroup;
import org.apache.sqoop.submission.counter.Counters;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
+import org.junit.Test;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
/**
*
*/
-public class TestSubmissionBean extends TestCase {
+public class TestSubmissionBean {
+
+ private static final double EPSILON = 0.01;
+ @Test
public void testTransferUnknown() {
transfer(MSubmission.UNKNOWN);
@@ -47,6 +53,7 @@ public class TestSubmissionBean extends TestCase {
transfer(submissions);
}
+ @Test
public void testTransferJobId() {
MSubmission source = new MSubmission();
source.setJobId(666);
@@ -69,6 +76,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals(888, targets.get(1).getJobId());
}
+ @Test
public void testTransferCreationUser() {
String username = "admin";
MSubmission source = new MSubmission();
@@ -92,6 +100,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals("userB", targets.get(1).getCreationUser());
}
+ @Test
public void testTransferCreationDate() {
Date date = new Date();
MSubmission source = new MSubmission();
@@ -117,6 +126,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals(datey, targets.get(1).getCreationDate());
}
+ @Test
public void testTransferLastUpdateUser() {
String username = "admin";
MSubmission source = new MSubmission();
@@ -140,6 +150,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals("userB", targets.get(1).getLastUpdateUser());
}
+ @Test
public void testTransferLastUpdateDate() {
Date date = new Date();
MSubmission source = new MSubmission();
@@ -165,6 +176,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals(datey, targets.get(1).getLastUpdateDate());
}
+ @Test
public void testTransferStatus() {
MSubmission source = new MSubmission();
source.setStatus(SubmissionStatus.SUCCEEDED);
@@ -187,6 +199,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals(SubmissionStatus.BOOTING, targets.get(1).getStatus());
}
+ @Test
public void testTransferExternalId() {
MSubmission source = new MSubmission();
source.setExternalId("Job-x");
@@ -209,6 +222,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals("Job-z", targets.get(1).getExternalId());
}
+ @Test
public void testTransferExternalLink() {
MSubmission source = new MSubmission();
source.setExternalLink("http://");
@@ -231,6 +245,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals("http://localhost:8080", targets.get(1).getExternalLink());
}
+ @Test
public void testTransferException() {
MSubmission source = new MSubmission();
source.setExceptionInfo("EndOfTheWorldException");
@@ -253,6 +268,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals("EndOfTheWorldAgainException", targets.get(1).getExceptionInfo());
}
+ @Test
public void testTransferExceptionTrace() {
MSubmission source = new MSubmission();
source.setExceptionStackTrace("void.java(3): line infinity");
@@ -275,12 +291,13 @@ public class TestSubmissionBean extends TestCase {
assertEquals("void.java(5): core dumps in Java", targets.get(1).getExceptionStackTrace());
}
+ @Test
public void testTransferProgress() {
MSubmission source = new MSubmission();
source.setProgress(25.0);
MSubmission target = transfer(source);
- assertEquals(25.0, target.getProgress());
+ assertEquals(25.0, target.getProgress(), EPSILON);
List<MSubmission> sources = new ArrayList<MSubmission>();
MSubmission sourcex = new MSubmission();
@@ -292,11 +309,12 @@ public class TestSubmissionBean extends TestCase {
List<MSubmission> targets = transfer(sources);
assertNotNull(targets.get(0));
- assertEquals(50.0, targets.get(0).getProgress());
+ assertEquals(50.0, targets.get(0).getProgress(), EPSILON);
assertNotNull(targets.get(1));
- assertEquals(99.9, targets.get(1).getProgress());
+ assertEquals(99.9, targets.get(1).getProgress(), EPSILON);
}
+ @Test
public void testTransferCounters() {
Counters counters = new Counters();
counters.addCounterGroup(new CounterGroup("A")
@@ -405,6 +423,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals(222222, counter.getValue());
}
+ @Test
public void testTransferFromSchema() {
MSubmission source = new MSubmission();
source.setFromSchema(getSchema());
@@ -414,6 +433,7 @@ public class TestSubmissionBean extends TestCase {
assertEquals(getSchema(), target);
}
+ @Test
public void testTransferToSchema() {
MSubmission source = new MSubmission();
source.setToSchema(getSchema());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java b/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
index d9c2043..5c094fb 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
@@ -28,8 +28,8 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import static junit.framework.Assert.assertNull;
-import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java b/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
index 9fa2b6b..d2a3476 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
@@ -17,21 +17,24 @@
*/
package org.apache.sqoop.model;
-import junit.framework.TestCase;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
+import org.junit.Test;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import static org.junit.Assert.*;
+
/**
* Test form utils
*/
-public class TestFormUtils extends TestCase {
+public class TestFormUtils {
+ @Test
public void testToForms() {
Config config = new Config();
config.aForm.a1 = "value";
@@ -48,6 +51,7 @@ public class TestFormUtils extends TestCase {
assertEquals("value", formsByBoth.get(0).getInputs().get(0).getValue());
}
+ @Test
public void testToFormsMissingAnnotation() {
try {
FormUtils.toForms(ConfigWithout.class);
@@ -91,6 +95,7 @@ public class TestFormUtils extends TestCase {
fail("Correct exception wasn't thrown");
}
+ @Test
public void testFailureOnPrimitiveType() {
PrimitiveConfig config = new PrimitiveConfig();
@@ -102,6 +107,7 @@ public class TestFormUtils extends TestCase {
}
}
+ @Test
public void testFillValues() {
List<MForm> forms = getForms();
@@ -113,6 +119,7 @@ public class TestFormUtils extends TestCase {
assertEquals("value", config.aForm.a1);
}
+ @Test
public void testFillValuesObjectReuse() {
List<MForm> forms = getForms();
@@ -129,6 +136,7 @@ public class TestFormUtils extends TestCase {
assertNull(config.bForm.b2);
}
+ @Test
public void testJson() {
Config config = new Config();
config.aForm.a1 = "A";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java b/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
index d56e165..af0f450 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
@@ -17,14 +17,14 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.assertEquals;
-
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
/**
* Test class for org.apache.sqoop.model.MAccountableEntity
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMBooleanInput.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMBooleanInput.java b/common/src/test/java/org/apache/sqoop/model/TestMBooleanInput.java
index cf9cf24..433c160 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMBooleanInput.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMBooleanInput.java
@@ -19,7 +19,6 @@ package org.apache.sqoop.model;
import org.junit.Test;
-import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java b/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java
index e2d2717..243fff9 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MConnectionForms
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConnector.java b/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
index 0a5fd90..d8bc94d 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
@@ -17,8 +17,6 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -26,6 +24,8 @@ import java.util.List;
import org.apache.sqoop.common.Direction;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.TestMConnector
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMEnumInput.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMEnumInput.java b/common/src/test/java/org/apache/sqoop/model/TestMEnumInput.java
index a25016a..c76d031 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMEnumInput.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMEnumInput.java
@@ -17,10 +17,10 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MEnumInput
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMForm.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMForm.java b/common/src/test/java/org/apache/sqoop/model/TestMForm.java
index 0bd55d9..536b650 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMForm.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMForm.java
@@ -17,12 +17,13 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MForm
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMFormList.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMFormList.java b/common/src/test/java/org/apache/sqoop/model/TestMFormList.java
index c7ee682..b8d3d37 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMFormList.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMFormList.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
-import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertEquals;
/**
*
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMIntegerInput.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMIntegerInput.java b/common/src/test/java/org/apache/sqoop/model/TestMIntegerInput.java
index 1f38e6d..dd9227e 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMIntegerInput.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMIntegerInput.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.model;
+import org.junit.Test;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import org.junit.Test;
-
/**
* Test class for org.apache.sqoop.model.MInputInput
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMJob.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMJob.java b/common/src/test/java/org/apache/sqoop/model/TestMJob.java
index 96c316a..8f2943e 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMJob.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMJob.java
@@ -17,14 +17,14 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import java.util.ArrayList;
import java.util.List;
import org.apache.sqoop.common.Direction;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MJob
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java b/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java
index 81a8614..e59b282 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MJobForms
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMLink.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMLink.java b/common/src/test/java/org/apache/sqoop/model/TestMLink.java
index 1be37a1..77fa2a9 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMLink.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMLink.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MConnection
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java b/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java
index 120fb07..fbc08c7 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java
@@ -17,17 +17,17 @@
*/
package org.apache.sqoop.model;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.junit.Test;
-
/**
* Test class for org.apache.sqoop.model.MMapInput
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMNamedElement.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMNamedElement.java b/common/src/test/java/org/apache/sqoop/model/TestMNamedElement.java
index f336bab..6610784 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMNamedElement.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMNamedElement.java
@@ -17,10 +17,10 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MNamedElement
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMPersistableEntity.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMPersistableEntity.java b/common/src/test/java/org/apache/sqoop/model/TestMPersistableEntity.java
index 000c6be..8e1e5bd 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMPersistableEntity.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMPersistableEntity.java
@@ -17,9 +17,10 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
import org.junit.Test;
+import static org.junit.Assert.*;
+
public class TestMPersistableEntity {
@Test
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMStringInput.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMStringInput.java b/common/src/test/java/org/apache/sqoop/model/TestMStringInput.java
index 2fe0335..76e625e 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMStringInput.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMStringInput.java
@@ -17,10 +17,10 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MStringInput
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java b/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java
index cf34983..a75a7cd 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java
@@ -17,12 +17,12 @@
*/
package org.apache.sqoop.model;
-import static org.junit.Assert.*;
-
import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Test class for org.apache.sqoop.model.MValidatedElement
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/submission/TestSubmissionStatus.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/submission/TestSubmissionStatus.java b/common/src/test/java/org/apache/sqoop/submission/TestSubmissionStatus.java
index 99f4767..948c0a6 100644
--- a/common/src/test/java/org/apache/sqoop/submission/TestSubmissionStatus.java
+++ b/common/src/test/java/org/apache/sqoop/submission/TestSubmissionStatus.java
@@ -17,19 +17,23 @@
*/
package org.apache.sqoop.submission;
+import org.junit.Test;
+
import java.util.Arrays;
import java.util.List;
-import junit.framework.TestCase;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
/**
* Test class for org.apache.sqoop.submission.SubmissionStatus
*/
-public class TestSubmissionStatus extends TestCase {
+public class TestSubmissionStatus {
/**
* unfinished() test
*/
+ @Test
public void testUnfinished() {
SubmissionStatus subStatus[] = SubmissionStatus.unfinished();
SubmissionStatus subStatusTest[] = new SubmissionStatus[] {
@@ -43,6 +47,7 @@ public class TestSubmissionStatus extends TestCase {
/**
* isRunning() test
*/
+ @Test
public void testIsRunning() {
assertTrue(SubmissionStatus.RUNNING.isRunning());
assertTrue(SubmissionStatus.BOOTING.isRunning());
@@ -54,6 +59,7 @@ public class TestSubmissionStatus extends TestCase {
/**
* isFailure() test
*/
+ @Test
public void testIsFailure() {
assertTrue(SubmissionStatus.FAILED.isFailure());
assertTrue(SubmissionStatus.UNKNOWN.isFailure());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/submission/counter/TestCounterGroup.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/submission/counter/TestCounterGroup.java b/common/src/test/java/org/apache/sqoop/submission/counter/TestCounterGroup.java
index 985009a..ae7aaf0 100644
--- a/common/src/test/java/org/apache/sqoop/submission/counter/TestCounterGroup.java
+++ b/common/src/test/java/org/apache/sqoop/submission/counter/TestCounterGroup.java
@@ -17,7 +17,7 @@
*/
package org.apache.sqoop.submission.counter;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/submission/counter/TestCounters.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/submission/counter/TestCounters.java b/common/src/test/java/org/apache/sqoop/submission/counter/TestCounters.java
index 8f8d617..90a35c3 100644
--- a/common/src/test/java/org/apache/sqoop/submission/counter/TestCounters.java
+++ b/common/src/test/java/org/apache/sqoop/submission/counter/TestCounters.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.submission.counter;
import org.junit.Test;
-import junit.framework.Assert;
+import org.junit.Assert;
/**
* Test Class for org.apache.sqoop.submission.counter.TestCounters
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
index 16c2015..53ece87 100644
--- a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
+++ b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
-
/**
*
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/validation/TestStatus.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/TestStatus.java b/common/src/test/java/org/apache/sqoop/validation/TestStatus.java
index 5b17a4e..99d8076 100644
--- a/common/src/test/java/org/apache/sqoop/validation/TestStatus.java
+++ b/common/src/test/java/org/apache/sqoop/validation/TestStatus.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.validation;
import org.junit.Test;
-import static junit.framework.Assert.*;
+import static org.junit.Assert.*;
/**
*
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/validation/TestValidation.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/TestValidation.java b/common/src/test/java/org/apache/sqoop/validation/TestValidation.java
index bf0ade5..85e9e1c 100644
--- a/common/src/test/java/org/apache/sqoop/validation/TestValidation.java
+++ b/common/src/test/java/org/apache/sqoop/validation/TestValidation.java
@@ -20,20 +20,22 @@ package org.apache.sqoop.validation;
import java.util.HashMap;
import java.util.Map;
-import junit.framework.TestCase;
-
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.validation.Validation.FormInput;
import org.apache.sqoop.validation.Validation.Message;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
/**
* Test class for org.apache.sqoop.validation.Validation
*/
-public class TestValidation extends TestCase {
+public class TestValidation {
/**
* Initialization test
*/
+ @Test
public void testInitialization() {
/* Check initialization with class */
Validation validation = new Validation(Class.class);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java
index b5dce62..e63a69a 100644
--- a/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java
@@ -20,7 +20,7 @@ package org.apache.sqoop.validation.validators;
import org.apache.sqoop.validation.Status;
import org.junit.Test;
-import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertEquals;
/**
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java
index 4bd1559..88347eb 100644
--- a/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java
@@ -20,7 +20,7 @@ package org.apache.sqoop.validation.validators;
import org.apache.sqoop.validation.Status;
import org.junit.Test;
-import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertEquals;
/**
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java
index 92409d9..8c4f9e0 100644
--- a/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java
@@ -20,7 +20,7 @@ package org.apache.sqoop.validation.validators;
import org.apache.sqoop.validation.Status;
import org.junit.Test;
-import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertEquals;
/**
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
index e10a5b4..1022370 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
@@ -17,9 +17,12 @@
*/
package org.apache.sqoop.connector.jdbc;
-import junit.framework.TestCase;
+import org.junit.Before;
+import org.junit.Test;
-public class GenericJdbcExecutorTest extends TestCase {
+import static org.junit.Assert.assertEquals;
+
+public class GenericJdbcExecutorTest {
private final String table;
private final String emptyTable;
private final GenericJdbcExecutor executor;
@@ -34,7 +37,7 @@ public class GenericJdbcExecutorTest extends TestCase {
GenericJdbcTestConstants.URL, null, null);
}
- @Override
+ @Before
public void setUp() {
if(executor.existTable(emptyTable)) {
executor.executeUpdate("DROP TABLE " + emptyTable);
@@ -56,6 +59,7 @@ public class GenericJdbcExecutorTest extends TestCase {
}
}
+ @Test
@SuppressWarnings("unchecked")
public void testDeleteTableData() throws Exception {
executor.deleteTableData(table);
@@ -63,6 +67,7 @@ public class GenericJdbcExecutorTest extends TestCase {
0, executor.getTableRowCount(table));
}
+ @Test
@SuppressWarnings("unchecked")
public void testMigrateData() throws Exception {
assertEquals("Table " + emptyTable + " is expected to be empty.",
@@ -80,6 +85,7 @@ public class GenericJdbcExecutorTest extends TestCase {
executor.getTableRowCount(emptyTable));
}
+ @Test
@SuppressWarnings("unchecked")
public void testGetTableRowCount() throws Exception {
assertEquals("Table " + table + " is expected to be empty.",
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
index 5f091de..fbdf9c6 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
@@ -17,8 +17,6 @@
*/
package org.apache.sqoop.connector.jdbc;
-import junit.framework.TestCase;
-
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
@@ -26,8 +24,14 @@ import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.job.etl.Extractor;
import org.apache.sqoop.job.etl.ExtractorContext;
import org.apache.sqoop.etl.io.DataWriter;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
-public class TestExtractor extends TestCase {
+public class TestExtractor {
private final String tableName;
@@ -36,11 +40,13 @@ public class TestExtractor extends TestCase {
private static final int START = -50;
private static final int NUMBER_OF_ROWS = 101;
+ private static final double EPSILON = 0.01;
+
public TestExtractor() {
tableName = getClass().getSimpleName().toUpperCase();
}
- @Override
+ @Before
public void setUp() {
executor = new GenericJdbcExecutor(GenericJdbcTestConstants.DRIVER,
GenericJdbcTestConstants.URL, null, null);
@@ -59,11 +65,12 @@ public class TestExtractor extends TestCase {
}
}
- @Override
+ @After
public void tearDown() {
executor.close();
}
+ @Test
public void testQuery() throws Exception {
MutableContext context = new MutableMapContext();
@@ -96,6 +103,7 @@ public class TestExtractor extends TestCase {
extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
}
+ @Test
public void testSubquery() throws Exception {
MutableContext context = new MutableMapContext();
@@ -139,7 +147,7 @@ public class TestExtractor extends TestCase {
if (array[i] instanceof Integer) {
assertEquals(indx, ((Integer)array[i]).intValue());
} else if (array[i] instanceof Double) {
- assertEquals((double)indx, ((Double)array[i]).doubleValue());
+ assertEquals((double)indx, ((Double)array[i]).doubleValue(), EPSILON);
} else {
assertEquals(String.valueOf(indx), array[i].toString());
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
index bf66bb1..54e6acf 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
@@ -19,8 +19,6 @@ package org.apache.sqoop.connector.jdbc;
import java.sql.Types;
-import junit.framework.TestCase;
-
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
@@ -32,8 +30,13 @@ import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.FloatingPoint;
import org.apache.sqoop.schema.type.Text;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
-public class TestFromInitializer extends TestCase {
+public class TestFromInitializer {
private final String schemaName;
private final String tableName;
@@ -56,7 +59,7 @@ public class TestFromInitializer extends TestCase {
tableColumns = "ICOL,VCOL";
}
- @Override
+ @Before
public void setUp() {
executor = new GenericJdbcExecutor(GenericJdbcTestConstants.DRIVER,
GenericJdbcTestConstants.URL, null, null);
@@ -105,11 +108,12 @@ public class TestFromInitializer extends TestCase {
;
}
- @Override
+ @After
public void tearDown() {
executor.close();
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableName() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -136,6 +140,7 @@ public class TestFromInitializer extends TestCase {
String.valueOf(START+NUMBER_OF_ROWS-1));
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumns() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -163,6 +168,7 @@ public class TestFromInitializer extends TestCase {
String.valueOf(START+NUMBER_OF_ROWS-1));
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableSql() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -190,6 +196,7 @@ public class TestFromInitializer extends TestCase {
String.valueOf((double)(START+NUMBER_OF_ROWS-1)));
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableSqlWithTableColumns() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -219,6 +226,7 @@ public class TestFromInitializer extends TestCase {
String.valueOf((double)(START+NUMBER_OF_ROWS-1)));
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableNameWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -248,6 +256,7 @@ public class TestFromInitializer extends TestCase {
String.valueOf(START+NUMBER_OF_ROWS-1));
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumnsWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -278,6 +287,7 @@ public class TestFromInitializer extends TestCase {
String.valueOf(START+NUMBER_OF_ROWS-1));
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableSqlWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -308,7 +318,7 @@ public class TestFromInitializer extends TestCase {
String.valueOf((double)(START+NUMBER_OF_ROWS-1)));
}
-
+ @Test
@SuppressWarnings("unchecked")
public void testGetSchemaForTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -330,6 +340,7 @@ public class TestFromInitializer extends TestCase {
assertEquals(getSchema(jobConf.fromJobConfig.schemaName + "." + tableName), schema);
}
+ @Test
@SuppressWarnings("unchecked")
public void testGetSchemaForSql() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -351,6 +362,7 @@ public class TestFromInitializer extends TestCase {
assertEquals(getSchema("Query"), schema);
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableSqlWithTableColumnsWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
index c349676..144b92a 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
@@ -17,9 +17,6 @@
*/
package org.apache.sqoop.connector.jdbc;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
import java.sql.ResultSet;
import java.util.Arrays;
import java.util.Collection;
@@ -38,6 +35,9 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
@RunWith(Parameterized.class)
public class TestLoader {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
index 949877b..ec75e1e 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
@@ -25,8 +25,6 @@ import java.sql.Types;
import java.util.Iterator;
import java.util.List;
-import junit.framework.TestCase;
-
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
@@ -34,12 +32,17 @@ import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.job.etl.Partition;
import org.apache.sqoop.job.etl.Partitioner;
import org.apache.sqoop.job.etl.PartitionerContext;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
-public class TestPartitioner extends TestCase {
+public class TestPartitioner {
private static final int START = -5;
private static final int NUMBER_OF_ROWS = 11;
+ @Test
public void testIntegerEvenPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(
@@ -71,6 +74,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testIntegerUnevenPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(
@@ -100,6 +104,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testIntegerOverPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(
@@ -136,6 +141,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testFloatingPointEvenPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(
@@ -167,6 +173,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testFloatingPointUnevenPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(
@@ -196,6 +203,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testNumericEvenPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME, "ICOL");
@@ -219,6 +227,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testNumericUnevenPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME, "DCOL");
@@ -240,6 +249,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testNumericSinglePartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME, "DCOL");
@@ -259,7 +269,7 @@ public class TestPartitioner extends TestCase {
});
}
-
+ @Test
public void testDatePartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME, "DCOL");
@@ -288,6 +298,7 @@ public class TestPartitioner extends TestCase {
}
+ @Test
public void testTimePartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants
@@ -314,6 +325,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testTimestampPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants
@@ -338,6 +350,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testBooleanPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants
@@ -361,6 +374,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testVarcharPartition() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants
@@ -408,6 +422,7 @@ public class TestPartitioner extends TestCase {
});
}
+ @Test
public void testVarcharPartition2() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants
@@ -431,6 +446,7 @@ public class TestPartitioner extends TestCase {
assertTrue(partitions.get(4).toString().contains("Warty Warthog"));
}
+ @Test
public void testVarcharPartitionWithCommonPrefix() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants
@@ -460,6 +476,7 @@ public class TestPartitioner extends TestCase {
}
+ @Test
public void testPatitionWithNullValues() throws Exception {
MutableContext context = new MutableMapContext();
context.setString(GenericJdbcConnectorConstants
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
index 00c2aab..a87ce7a 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
@@ -17,7 +17,6 @@
*/
package org.apache.sqoop.connector.jdbc;
-import junit.framework.TestCase;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.common.SqoopException;
@@ -28,8 +27,15 @@ import org.apache.sqoop.job.etl.InitializerContext;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.ValidationResult;
import org.apache.sqoop.validation.ValidationRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
-public class TestToInitializer extends TestCase {
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestToInitializer {
private final String schemaName;
private final String tableName;
private final String schemalessTableName;
@@ -51,7 +57,7 @@ public class TestToInitializer extends TestCase {
tableColumns = "ICOL,VCOL";
}
- @Override
+ @Before
public void setUp() {
executor = new GenericJdbcExecutor(GenericJdbcTestConstants.DRIVER,
GenericJdbcTestConstants.URL, null, null);
@@ -68,11 +74,12 @@ public class TestToInitializer extends TestCase {
}
}
- @Override
+ @After
public void tearDown() {
executor.close();
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableName() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -94,6 +101,7 @@ public class TestToInitializer extends TestCase {
verifyResult(context, "INSERT INTO " + fullTableName + " VALUES (?,?,?)");
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumns() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -116,6 +124,7 @@ public class TestToInitializer extends TestCase {
verifyResult(context, "INSERT INTO " + fullTableName + " (" + tableColumns + ") VALUES (?,?)");
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableSql() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -135,6 +144,7 @@ public class TestToInitializer extends TestCase {
verifyResult(context, "INSERT INTO " + executor.delimitIdentifier(schemalessTableName) + " VALUES (?,?,?)");
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableNameWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -157,6 +167,7 @@ public class TestToInitializer extends TestCase {
verifyResult(context, "INSERT INTO " + fullTableName + " VALUES (?,?,?)");
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumnsWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -180,6 +191,7 @@ public class TestToInitializer extends TestCase {
verifyResult(context, "INSERT INTO " + fullTableName + " (" + tableColumns + ") VALUES (?,?)");
}
+ @Test
@SuppressWarnings("unchecked")
public void testTableSqlWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -215,6 +227,7 @@ public class TestToInitializer extends TestCase {
"(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20))");
}
+ @Test
public void testNonExistingStageTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
@@ -237,6 +250,7 @@ public class TestToInitializer extends TestCase {
}
}
+ @Test
@SuppressWarnings("unchecked")
public void testNonEmptyStageTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -264,6 +278,7 @@ public class TestToInitializer extends TestCase {
}
}
+ @Test
public void testClearStageTableValidation() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
@@ -293,6 +308,7 @@ public class TestToInitializer extends TestCase {
"toJobConfig"));
}
+ @Test
public void testStageTableWithoutTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
@@ -311,6 +327,7 @@ public class TestToInitializer extends TestCase {
"toJobConfig"));
}
+ @Test
@SuppressWarnings("unchecked")
public void testClearStageTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
@@ -336,6 +353,7 @@ public class TestToInitializer extends TestCase {
executor.getTableRowCount(stageTableName));
}
+ @Test
@SuppressWarnings("unchecked")
public void testStageTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
index 80f45e5..aa3af89 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
@@ -17,13 +17,6 @@
*/
package org.apache.sqoop.driver;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
import java.util.Arrays;
import java.util.List;
@@ -41,6 +34,13 @@ import org.apache.sqoop.request.HttpEventContext;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class TestJobManager {
private JobManager jobManager;
private SqoopConnector sqoopConnectorMock;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
index 768c978..9eff8a4 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
@@ -17,9 +17,6 @@
*/
package org.apache.sqoop.driver;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
import java.util.Arrays;
import org.apache.sqoop.driver.JobRequest;
@@ -27,6 +24,9 @@ import org.apache.sqoop.utils.ClassUtils;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
/**
*
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
index 796b021..19b0023 100644
--- a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
+++ b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
@@ -17,23 +17,6 @@
*/
package org.apache.sqoop.repository;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.inOrder;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Collections;
@@ -63,6 +46,23 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.InOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
public class TestJdbcRepository {
private JdbcRepository repoSpy;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
index 09e5ec5..b5435ff 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.job;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
index 869c727..5662120 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
@@ -23,9 +23,6 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -57,13 +54,17 @@ import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.FloatingPoint;
import org.apache.sqoop.schema.type.Text;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
-public class TestMapReduce extends TestCase {
+public class TestMapReduce {
private static final int START_PARTITION = 1;
private static final int NUMBER_OF_PARTITIONS = 9;
private static final int NUMBER_OF_ROWS_PER_PARTITION = 10;
+ @Test
public void testInputFormat() throws Exception {
Configuration conf = new Configuration();
conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
@@ -82,6 +83,7 @@ public class TestMapReduce extends TestCase {
}
}
+ @Test
public void testMapper() throws Exception {
Configuration conf = new Configuration();
conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
@@ -98,6 +100,7 @@ public class TestMapReduce extends TestCase {
DummyOutputFormat.class);
}
+ @Test
public void testOutputFormat() throws Exception {
Configuration conf = new Configuration();
conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
@@ -117,8 +120,8 @@ public class TestMapReduce extends TestCase {
SqoopNullOutputFormat.class);
// Make sure both destroyers get called.
- Assert.assertEquals(1, DummyFromDestroyer.count);
- Assert.assertEquals(1, DummyToDestroyer.count);
+ assertEquals(1, DummyFromDestroyer.count);
+ assertEquals(1, DummyToDestroyer.count);
}
public static class DummyPartition extends Partition {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/SqoopWritableTest.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/SqoopWritableTest.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/SqoopWritableTest.java
index b78b140..f5742a2 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/SqoopWritableTest.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/SqoopWritableTest.java
@@ -19,8 +19,6 @@
package org.apache.sqoop.job.io;
import com.google.common.base.Charsets;
-import junit.framework.Assert;
-import junit.framework.TestCase;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -34,17 +32,21 @@ import java.io.InputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.connector.idf.CSVIntermediateDataFormat;
import org.apache.sqoop.job.JobConstants;
+import org.junit.Assert;
+import org.junit.Test;
-public class SqoopWritableTest extends TestCase {
+public class SqoopWritableTest {
private final SqoopWritable writable = new SqoopWritable();
+ @Test
public void testStringInStringOut() {
String testData = "Live Long and prosper";
writable.setString(testData);
Assert.assertEquals(testData,writable.getString());
}
+ @Test
public void testDataWritten() throws IOException {
String testData = "One ring to rule them all";
byte[] testDataBytes = testData.getBytes(Charsets.UTF_8);
@@ -59,6 +61,7 @@ public class SqoopWritableTest extends TestCase {
Assert.assertEquals(testData, readData);
}
+ @Test
public void testDataRead() throws IOException {
String testData = "Brandywine Bridge - 20 miles!";
ByteArrayOutputStream ostream = new ByteArrayOutputStream();
@@ -70,6 +73,7 @@ public class SqoopWritableTest extends TestCase {
Assert.assertEquals(testData, writable.getString());
}
+ @Test
public void testWriteReadUsingStream() throws IOException {
String testData = "You shall not pass";
ByteArrayOutputStream ostream = new ByteArrayOutputStream();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestData.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestData.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestData.java
index 91df426..4e23bcb 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestData.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/io/TestData.java
@@ -19,11 +19,10 @@ package org.apache.sqoop.job.io;
import java.util.Arrays;
-import junit.framework.TestCase;
-
+import org.junit.Assert;
import org.junit.Test;
-public class TestData extends TestCase {
+public class TestData {
private static final double TEST_NUMBER = Math.PI + 100;
@Test
@@ -108,10 +107,10 @@ public class TestData extends TestCase {
public static void assertEquals(Object expected, Object actual) {
if (expected instanceof byte[]) {
- assertEquals(Arrays.toString((byte[])expected),
+ Assert.assertEquals(Arrays.toString((byte[])expected),
Arrays.toString((byte[])actual));
} else {
- TestCase.assertEquals(expected, actual);
+ Assert.assertEquals(expected, actual);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java
index c2ebd7e..1f411d2 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestSqoopOutputFormatLoadExecutor.java
@@ -18,7 +18,6 @@
*/
package org.apache.sqoop.job.mr;
-import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.RecordWriter;
@@ -29,6 +28,7 @@ import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.etl.Loader;
import org.apache.sqoop.job.etl.LoaderContext;
import org.apache.sqoop.job.io.SqoopWritable;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index 998f5b7..4b95687 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -17,8 +17,6 @@
*/
package org.apache.sqoop.repository.derby;
-import junit.framework.TestCase;
-
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.model.MLink;
@@ -31,6 +29,8 @@ import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
+import org.junit.After;
+import org.junit.Before;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -43,11 +43,12 @@ import java.util.LinkedList;
import java.util.List;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.*;
+import static org.junit.Assert.assertEquals;
/**
* Abstract class with convenience methods for testing derby repository.
*/
-abstract public class DerbyTestCase extends TestCase {
+abstract public class DerbyTestCase {
private static int LATEST_SYSTEM_VERSION = 4;
@@ -59,16 +60,14 @@ abstract public class DerbyTestCase extends TestCase {
private Connection connection;
- @Override
+ @Before
public void setUp() throws Exception {
- super.setUp();
-
// Create link to the database
Class.forName(DERBY_DRIVER).newInstance();
connection = DriverManager.getConnection(getStartJdbcUrl());
}
- @Override
+ @After
public void tearDown() throws Exception {
// Close active link
if(connection != null) {
@@ -81,9 +80,6 @@ abstract public class DerbyTestCase extends TestCase {
} catch (SQLException ex) {
// Dropping Derby database leads always to exception
}
-
- // Call parent tear down
- super.tearDown();
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
index 8a5823d..15306c2 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
@@ -18,9 +18,15 @@
package org.apache.sqoop.repository.derby;
import org.apache.sqoop.model.MConnector;
+import org.junit.Before;
+import org.junit.Test;
import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
/**
* Test connector methods on Derby repository.
*/
@@ -28,7 +34,7 @@ public class TestConnectorHandling extends DerbyTestCase {
DerbyRepositoryHandler handler;
- @Override
+ @Before
public void setUp() throws Exception {
super.setUp();
@@ -38,6 +44,7 @@ public class TestConnectorHandling extends DerbyTestCase {
createSchema();
}
+ @Test
public void testFindConnector() throws Exception {
// On empty repository, no connectors should be there
assertNull(handler.findConnector("A", getDerbyDatabaseConnection()));
@@ -57,6 +64,7 @@ public class TestConnectorHandling extends DerbyTestCase {
assertEquals(original, connector);
}
+ @Test
public void testFindAllConnectors() throws Exception {
// No connectors in an empty repository, we expect an empty list
assertEquals(handler.findConnectors(getDerbyDatabaseConnection()).size(),0);
@@ -74,6 +82,7 @@ public class TestConnectorHandling extends DerbyTestCase {
}
+ @Test
public void testRegisterConnector() throws Exception {
MConnector connector = getConnector();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
index 567dda6..9b18bd3 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
@@ -19,11 +19,17 @@ package org.apache.sqoop.repository.derby;
import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.model.MDriverConfig;
+import org.junit.Before;
+import org.junit.Test;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
/**
* Test driver config methods on Derby repository.
*/
@@ -31,7 +37,7 @@ public class TestDriverConfigHandling extends DerbyTestCase {
DerbyRepositoryHandler handler;
- @Override
+ @Before
public void setUp() throws Exception {
super.setUp();
@@ -41,6 +47,7 @@ public class TestDriverConfigHandling extends DerbyTestCase {
createSchema();
}
+ @Test
public void testFindDriverConfig() throws Exception {
// On empty repository, no driverConfig should be there
assertNull(handler.findDriverConfig(getDerbyDatabaseConnection()));
@@ -57,6 +64,7 @@ public class TestDriverConfigHandling extends DerbyTestCase {
assertEquals(originalDriverConfig, driverConfig);
}
+ @Test
public void testRegisterConnector() throws Exception {
MDriverConfig driverConfig = getDriverConfig();
handler.registerDriverConfig(driverConfig, getDerbyDatabaseConnection());
@@ -106,6 +114,7 @@ public class TestDriverConfigHandling extends DerbyTestCase {
}
}
+ @Test
public void testDriverVersion() throws Exception {
handler.registerDriverConfig(getDriverConfig(), getDerbyDatabaseConnection());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
index c4a5829..d744693 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
@@ -29,12 +29,18 @@ import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MStringInput;
+import org.junit.Before;
+import org.junit.Test;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+
/**
* Test proper support of all available model types.
*/
@@ -42,7 +48,7 @@ public class TestInputTypes extends DerbyTestCase {
DerbyRepositoryHandler handler;
- @Override
+ @Before
public void setUp() throws Exception {
super.setUp();
@@ -56,6 +62,7 @@ public class TestInputTypes extends DerbyTestCase {
* Ensure that metadata with all various data types can be successfully
* serialized into repository and retrieved back.
*/
+ @Test
public void testEntitySerialization() throws Exception {
MConnector connector = getConnector();
@@ -76,6 +83,7 @@ public class TestInputTypes extends DerbyTestCase {
/**
* Test that serializing actual data is not an issue.
*/
+ @Test
public void testEntityDataSerialization() throws Exception {
MConnector connector = getConnector();
MDriverConfig driverConfig = getDriverConfig();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
index 37aa8d4..1292168 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
@@ -17,8 +17,14 @@
*/
package org.apache.sqoop.repository.derby;
+import org.junit.Before;
+import org.junit.Test;
+
import java.sql.Connection;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
/**
*
*/
@@ -26,25 +32,28 @@ public class TestInternals extends DerbyTestCase {
DerbyRepositoryHandler handler;
- @Override
+ @Before
public void setUp() throws Exception {
super.setUp();
handler = new TestDerbyRepositoryHandler();
}
+ @Test
public void testSuitableInternals() throws Exception {
assertFalse(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
createSchema(); // Test code is building the structures
assertTrue(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
}
+ @Test
public void testCreateorUpdateInternals() throws Exception {
assertFalse(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
handler.createOrUpdateInternals(getDerbyDatabaseConnection());
assertTrue(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
}
+ @Test
public void testUpgradeVersion2ToVersion4() throws Exception {
createSchema(2);
assertFalse(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/97da12aa/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
index f22c351..47350ea 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
@@ -23,11 +23,15 @@ import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
+import org.junit.Before;
+import org.junit.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.junit.Assert.*;
+
/**
* Test job methods on Derby repository.
*/
@@ -35,7 +39,7 @@ public class TestJobHandling extends DerbyTestCase {
DerbyRepositoryHandler handler;
- @Override
+ @Before
public void setUp() throws Exception {
super.setUp();
@@ -51,6 +55,7 @@ public class TestJobHandling extends DerbyTestCase {
loadLinks();
}
+ @Test
public void testFindJob() throws Exception {
// Let's try to find non existing job
try {
@@ -94,6 +99,7 @@ public class TestJobHandling extends DerbyTestCase {
assertNull(forms.get(1).getInputs().get(1).getValue());
}
+ @Test
public void testFindJobs() throws Exception {
List<MJob> list;
@@ -116,6 +122,7 @@ public class TestJobHandling extends DerbyTestCase {
assertEquals("JD", list.get(3).getName());
}
+ @Test
public void testExistsJob() throws Exception {
// There shouldn't be anything on empty repository
assertFalse(handler.existsJob(1, getDerbyDatabaseConnection()));
@@ -133,6 +140,7 @@ public class TestJobHandling extends DerbyTestCase {
assertFalse(handler.existsJob(5, getDerbyDatabaseConnection()));
}
+ @Test
public void testInUseJob() throws Exception {
loadJobs();
loadSubmissions();
@@ -143,6 +151,7 @@ public class TestJobHandling extends DerbyTestCase {
assertFalse(handler.inUseJob(4, getDerbyDatabaseConnection()));
}
+ @Test
public void testCreateJob() throws Exception {
MJob job = getJob();
@@ -183,6 +192,7 @@ public class TestJobHandling extends DerbyTestCase {
assertCountForTable("SQOOP.SQ_JOB_INPUT", 12);
}
+ @Test
public void testUpdateJob() throws Exception {
loadJobs();
@@ -233,6 +243,7 @@ public class TestJobHandling extends DerbyTestCase {
assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
}
+ @Test
public void testEnableAndDisableJob() throws Exception {
loadJobs();
@@ -251,6 +262,7 @@ public class TestJobHandling extends DerbyTestCase {
assertEquals(true, retrieved.getEnabled());
}
+ @Test
public void testDeleteJob() throws Exception {
loadJobs();
[05/52] [abbrv] git commit: SQOOP-1468: Sqoop2: Validations:
Introduce advanced validators that can be parametrized
Posted by ab...@apache.org.
SQOOP-1468: Sqoop2: Validations: Introduce advanced validators that can be parametrized
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/e71dd757
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/e71dd757
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/e71dd757
Branch: refs/heads/SQOOP-1367
Commit: e71dd75793982b1d6e5109b8ef2209c835a79b68
Parents: c0b22b1
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Sep 3 11:17:48 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:17 2014 -0700
----------------------------------------------------------------------
.../apache/sqoop/model/ConfigurationClass.java | 4 +-
.../java/org/apache/sqoop/model/FormClass.java | 4 +-
.../main/java/org/apache/sqoop/model/Input.java | 4 +-
.../java/org/apache/sqoop/model/Validator.java | 45 ++++++++
.../sqoop/validation/ValidationResult.java | 4 +-
.../sqoop/validation/ValidationRunner.java | 18 ++--
.../validators/AbstractValidator.java | 103 +++++++++++++++++++
.../validation/validators/ClassAvailable.java | 2 +-
.../sqoop/validation/validators/Contains.java | 32 ++++++
.../sqoop/validation/validators/NotEmpty.java | 2 +-
.../sqoop/validation/validators/NotNull.java | 2 +-
.../sqoop/validation/validators/StartsWith.java | 32 ++++++
.../sqoop/validation/validators/Validator.java | 80 --------------
.../sqoop/validation/TestValidationRunner.java | 42 ++++++--
.../validators/TestClassAvailable.java | 2 +-
.../validation/validators/TestContains.java | 90 ++++++++++++++++
.../validation/validators/TestNotEmpty.java | 2 +-
.../validation/validators/TestNotNull.java | 2 +-
.../validation/validators/TestStartsWith.java | 74 +++++++++++++
.../validation/validators/TestValidator.java | 2 +-
.../jdbc/configuration/ConnectionForm.java | 12 ++-
.../jdbc/configuration/FromTableForm.java | 19 ++--
.../jdbc/configuration/ToTableForm.java | 7 +-
23 files changed, 450 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java b/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
index 5323bd9..73374d8 100644
--- a/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
+++ b/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
@@ -17,8 +17,6 @@
*/
package org.apache.sqoop.model;
-import org.apache.sqoop.validation.validators.Validator;
-
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -37,5 +35,5 @@ public @interface ConfigurationClass {
*
* @return
*/
- Class<? extends Validator>[] validators() default {};
+ Validator[] validators() default {};
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/model/FormClass.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/FormClass.java b/common/src/main/java/org/apache/sqoop/model/FormClass.java
index 48bff3c..6048d03 100644
--- a/common/src/main/java/org/apache/sqoop/model/FormClass.java
+++ b/common/src/main/java/org/apache/sqoop/model/FormClass.java
@@ -17,8 +17,6 @@
*/
package org.apache.sqoop.model;
-import org.apache.sqoop.validation.validators.Validator;
-
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -43,5 +41,5 @@ public @interface FormClass {
*
* @return
*/
- Class<? extends Validator>[] validators() default {};
+ Validator[] validators() default {};
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/model/Input.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/Input.java b/common/src/main/java/org/apache/sqoop/model/Input.java
index 61fc01a..2b32e43 100644
--- a/common/src/main/java/org/apache/sqoop/model/Input.java
+++ b/common/src/main/java/org/apache/sqoop/model/Input.java
@@ -17,8 +17,6 @@
*/
package org.apache.sqoop.model;
-import org.apache.sqoop.validation.validators.Validator;
-
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -51,5 +49,5 @@ public @interface Input {
*
* @return
*/
- Class<? extends Validator>[] validators() default {};
+ Validator[] validators() default {};
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/model/Validator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/Validator.java b/common/src/main/java/org/apache/sqoop/model/Validator.java
new file mode 100644
index 0000000..6368a9d
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/Validator.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import org.apache.sqoop.validation.validators.AbstractValidator;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Annotation for specifying validators
+ *
+ * Usage without any parameters:
+ * @Validator(ClassName.class)
+ *
+ * To specify string parameter call:
+ * @Validator(value = ClassName.class, strArg = "Hello World!")
+ */
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Validator {
+ /**
+ * Validator implementation that should be executed.
+ */
+ Class<? extends AbstractValidator> value();
+
+ /**
+ * Optional argument that should be given to the validator before execution.
+ */
+ String strArg() default AbstractValidator.DEFAULT_STRING_ARGUMENT;
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java b/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java
index 2782fac..ae8f1d1 100644
--- a/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java
+++ b/common/src/main/java/org/apache/sqoop/validation/ValidationResult.java
@@ -17,7 +17,7 @@
*/
package org.apache.sqoop.validation;
-import org.apache.sqoop.validation.validators.Validator;
+import org.apache.sqoop.validation.validators.AbstractValidator;
import java.util.HashMap;
import java.util.List;
@@ -49,7 +49,7 @@ public class ValidationResult {
* @param name Full name of the validated object
* @param validator Executed validator
*/
- public void addValidator(String name, Validator validator) {
+ public void addValidator(String name, AbstractValidator validator) {
if(validator.getStatus() == Status.getDefault()) {
return;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java b/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
index 46e2d56..f36faf2 100644
--- a/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
+++ b/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
@@ -22,8 +22,9 @@ import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.FormUtils;
import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
import org.apache.sqoop.utils.ClassUtils;
-import org.apache.sqoop.validation.validators.Validator;
+import org.apache.sqoop.validation.validators.AbstractValidator;
import java.lang.reflect.Field;
@@ -114,14 +115,14 @@ public class ValidationRunner {
*
* @param name Full name of the object
* @param object Input, Form or Class instance
- * @param classes Validators array
+ * @param validators Validators array
* @return
*/
- private ValidationResult validateArray(String name, Object object, Class<? extends Validator> []classes) {
+ private ValidationResult validateArray(String name, Object object, Validator[] validators) {
ValidationResult result = new ValidationResult();
- for (Class<? extends Validator> klass : classes) {
- Validator v = executeValidator(object, klass);
+ for (Validator validator : validators) {
+ AbstractValidator v = executeValidator(object, validator);
result.addValidator(name, v);
}
@@ -132,11 +133,12 @@ public class ValidationRunner {
* Execute single validator.
*
* @param object Input, Form or Class instance
- * @param klass Validator's clas
+ * @param validator Validator annotation
* @return
*/
- private Validator executeValidator(Object object, Class<? extends Validator> klass) {
- Validator instance = (Validator) ClassUtils.instantiate(klass);
+ private AbstractValidator executeValidator(Object object, Validator validator) {
+ AbstractValidator instance = (AbstractValidator) ClassUtils.instantiate(validator.value());
+ instance.setStringArgument(validator.strArg());
instance.validate(object);
return instance;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/validators/AbstractValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/AbstractValidator.java b/common/src/main/java/org/apache/sqoop/validation/validators/AbstractValidator.java
new file mode 100644
index 0000000..9dbb44c
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/validation/validators/AbstractValidator.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation.validators;
+
+import org.apache.sqoop.validation.Message;
+import org.apache.sqoop.validation.Status;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Abstract validator class.
+ *
+ * Can be used to validate inputs, forms and configuration classes.
+ */
+abstract public class AbstractValidator<T> {
+
+ /**
+ * Default value of String argument.
+ */
+ public static final String DEFAULT_STRING_ARGUMENT = "";
+
+ /**
+ * Validation check.
+ *
+ * To be implemented by our children.
+ *
+ * @param instance Object to validate (depending on what we are validating)
+ */
+ abstract public void validate(T instance);
+
+ /**
+ * Messages generated during validation.
+ */
+ private List<Message> messages;
+
+ /**
+ * Overall status of the validation.
+ */
+ private Status status;
+
+ /**
+ * Optional String argument that can be defined for the Validator
+ */
+ private String stringArgument;
+
+ public AbstractValidator() {
+ reset();
+ }
+
+ protected void addMessage(Message msg) {
+ status = Status.getWorstStatus(status, msg.getStatus());
+ messages.add(msg);
+ }
+
+ protected void addMessage(Status status, String msg) {
+ addMessage(new Message(status, msg));
+ }
+
+ public List<Message> getMessages() {
+ return messages;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStringArgument(String arg) {
+ this.stringArgument = arg;
+ }
+
+ public String getStringArgument() {
+ return stringArgument;
+ }
+
+ /**
+ * Reset validator state.
+ *
+ * * Previous messages
+ * * Status
+ * * Any stored arguments
+ */
+ public void reset() {
+ messages = new LinkedList<Message>();
+ status = Status.getDefault();
+ stringArgument = DEFAULT_STRING_ARGUMENT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/validators/ClassAvailable.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/ClassAvailable.java b/common/src/main/java/org/apache/sqoop/validation/validators/ClassAvailable.java
index 2adfe6c..52a8fdd 100644
--- a/common/src/main/java/org/apache/sqoop/validation/validators/ClassAvailable.java
+++ b/common/src/main/java/org/apache/sqoop/validation/validators/ClassAvailable.java
@@ -23,7 +23,7 @@ import org.apache.sqoop.validation.Status;
/**
* Ensure that given String Input is a class that is available to this JVM.
*/
-public class ClassAvailable extends Validator<String> {
+public class ClassAvailable extends AbstractValidator<String> {
@Override
public void validate(String klass) {
try {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/validators/Contains.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/Contains.java b/common/src/main/java/org/apache/sqoop/validation/validators/Contains.java
new file mode 100644
index 0000000..8920c72
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/validation/validators/Contains.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation.validators;
+
+import org.apache.sqoop.validation.Status;
+
+/**
+ * String validator to verify presence of a sub string (provided as a String argument)
+ */
+public class Contains extends AbstractValidator<String> {
+ @Override
+ public void validate(String str) {
+ if(str == null || !str.contains(getStringArgument())) {
+ addMessage(Status.UNACCEPTABLE, "String must contain substring: " + getStringArgument());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/validators/NotEmpty.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/NotEmpty.java b/common/src/main/java/org/apache/sqoop/validation/validators/NotEmpty.java
index 520beea..248a2fa 100644
--- a/common/src/main/java/org/apache/sqoop/validation/validators/NotEmpty.java
+++ b/common/src/main/java/org/apache/sqoop/validation/validators/NotEmpty.java
@@ -24,7 +24,7 @@ import org.apache.sqoop.validation.Status;
*
* Will also ensure that the string is not null.
*/
-public class NotEmpty extends Validator<String> {
+public class NotEmpty extends AbstractValidator<String> {
@Override
public void validate(String instance) {
if (instance == null || instance.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/validators/NotNull.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/NotNull.java b/common/src/main/java/org/apache/sqoop/validation/validators/NotNull.java
index fb8a926..93b5fa7 100644
--- a/common/src/main/java/org/apache/sqoop/validation/validators/NotNull.java
+++ b/common/src/main/java/org/apache/sqoop/validation/validators/NotNull.java
@@ -22,7 +22,7 @@ import org.apache.sqoop.validation.Status;
/**
* Ensure that given object is never null.
*/
-public class NotNull<T> extends Validator<T> {
+public class NotNull<T> extends AbstractValidator<T> {
@Override
public void validate(T instance) {
if (instance == null) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/validators/StartsWith.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/StartsWith.java b/common/src/main/java/org/apache/sqoop/validation/validators/StartsWith.java
new file mode 100644
index 0000000..808e1f7
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/validation/validators/StartsWith.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation.validators;
+
+import org.apache.sqoop.validation.Status;
+
+/**
+ * String validator to verify if Input starts with given string (String argument).
+ */
+public class StartsWith extends AbstractValidator<String> {
+ @Override
+ public void validate(String str) {
+ if(str == null || !str.startsWith(getStringArgument())) {
+ addMessage(Status.UNACCEPTABLE, "String must start with: " + getStringArgument());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java b/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java
deleted file mode 100644
index bdb7c20..0000000
--- a/common/src/main/java/org/apache/sqoop/validation/validators/Validator.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.validation.validators;
-
-import org.apache.sqoop.validation.Message;
-import org.apache.sqoop.validation.Status;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Abstract validator class.
- *
- * Can be used to validate inputs, forms and configuration classes.
- */
-abstract public class Validator<T> {
-
- /**
- * Validation check.
- *
- * To be implemented by our children.
- *
- * @param instance Object to validate (depending on what we are validating)
- */
- abstract public void validate(T instance);
-
- /**
- * Messages generated during validation.
- */
- private List<Message> messages;
-
- /**
- * Overall status of the validation.
- */
- private Status status;
-
- public Validator() {
- reset();
- }
-
- protected void addMessage(Message msg) {
- status = Status.getWorstStatus(status, msg.getStatus());
- messages.add(msg);
- }
-
- protected void addMessage(Status status, String msg) {
- addMessage(new Message(status, msg));
- }
-
- public List<Message> getMessages() {
- return messages;
- }
-
- public Status getStatus() {
- return status;
- }
-
- /**
- * Reset validator state (all previous messages).
- */
- public void reset() {
- messages = new LinkedList<Message>();
- status = Status.getDefault();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java b/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
index 1961425..647abe0 100644
--- a/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
+++ b/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
@@ -21,9 +21,11 @@ import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
+import org.apache.sqoop.validation.validators.Contains;
import org.apache.sqoop.validation.validators.NotEmpty;
import org.apache.sqoop.validation.validators.NotNull;
-import org.apache.sqoop.validation.validators.Validator;
+import org.apache.sqoop.validation.validators.AbstractValidator;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -33,12 +35,12 @@ import static org.junit.Assert.assertTrue;
*/
public class TestValidationRunner {
- @FormClass(validators = {FormA.FormValidator.class})
+ @FormClass(validators = {@Validator(FormA.FormValidator.class)})
public static class FormA {
- @Input(validators = {NotNull.class})
+ @Input(validators = {@Validator(NotNull.class)})
String notNull;
- public static class FormValidator extends Validator<FormA> {
+ public static class FormValidator extends AbstractValidator<FormA> {
@Override
public void validate(FormA form) {
if(form.notNull == null) {
@@ -80,7 +82,13 @@ public class TestValidationRunner {
@FormClass
public static class FormB {
- @Input(validators = {NotNull.class, NotEmpty.class})
+ @Input(validators = {@Validator(NotNull.class), @Validator(NotEmpty.class)})
+ String str;
+ }
+
+ @FormClass
+ public static class FormC {
+ @Input(validators = {@Validator(value = Contains.class, strArg = "findme")})
String str;
}
@@ -98,14 +106,34 @@ public class TestValidationRunner {
assertEquals(2, result.getMessages().get("formName.str").size());
}
- @ConfigurationClass(validators = {ConfigurationA.ClassValidator.class})
+ @Test
+ public void testValidatorWithParameters() {
+ FormC form = new FormC();
+ ValidationRunner runner = new ValidationRunner();
+ ValidationResult result;
+
+ // Sub string not found
+ form.str = "Mordor";
+ result = runner.validateForm("formName", form);
+ assertEquals(Status.UNACCEPTABLE, result.getStatus());
+ assertEquals(1, result.getMessages().size());
+ assertTrue(result.getMessages().containsKey("formName.str"));
+
+ // Sub string found
+ form.str = "Morfindmedor";
+ result = runner.validateForm("formName", form);
+ assertEquals(Status.FINE, result.getStatus());
+ assertEquals(0, result.getMessages().size());
+ }
+
+ @ConfigurationClass(validators = {@Validator(ConfigurationA.ClassValidator.class)})
public static class ConfigurationA {
@Form FormA formA;
public ConfigurationA() {
formA = new FormA();
}
- public static class ClassValidator extends Validator<ConfigurationA> {
+ public static class ClassValidator extends AbstractValidator<ConfigurationA> {
@Override
public void validate(ConfigurationA conf) {
if("error".equals(conf.formA.notNull)) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
index 511b3b4..62b2e0a 100644
--- a/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
@@ -29,7 +29,7 @@ import static org.junit.Assert.assertEquals;
*/
public class TestClassAvailable {
- Validator validator = new ClassAvailable();
+ AbstractValidator validator = new ClassAvailable();
@Test
public void test() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java
new file mode 100644
index 0000000..b5dce62
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestContains.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation.validators;
+
+import org.apache.sqoop.validation.Status;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ */
+public class TestContains {
+
+ AbstractValidator validator = new Contains();
+
+ @Test
+ public void test() {
+ assertEquals(0, validator.getMessages().size());
+
+ // Default, no string argument set
+ validator.validate("str");
+ assertEquals(Status.FINE, validator.getStatus());
+ assertEquals(0, validator.getMessages().size());
+
+ // Searched substring is entire string
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("str");
+ assertEquals(Status.FINE, validator.getStatus());
+ assertEquals(0, validator.getMessages().size());
+
+ // Just starts with
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("straaaaa");
+ assertEquals(Status.FINE, validator.getStatus());
+ assertEquals(0, validator.getMessages().size());
+
+ // Ends with
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("aaastr");
+ assertEquals(Status.FINE, validator.getStatus());
+ assertEquals(0, validator.getMessages().size());
+
+ // In the middle
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("aaastraaa");
+ assertEquals(Status.FINE, validator.getStatus());
+ assertEquals(0, validator.getMessages().size());
+
+ // Null string
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate(null);
+ assertEquals(Status.UNACCEPTABLE, validator.getStatus());
+ assertEquals(1, validator.getMessages().size());
+
+ // Empty string
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("");
+ assertEquals(Status.UNACCEPTABLE, validator.getStatus());
+ assertEquals(1, validator.getMessages().size());
+
+ // "Random" string
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("Ahoj tady je meduza");
+ assertEquals(Status.UNACCEPTABLE, validator.getStatus());
+ assertEquals(1, validator.getMessages().size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/test/java/org/apache/sqoop/validation/validators/TestNotEmpty.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestNotEmpty.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestNotEmpty.java
index d225b06..5c9169a 100644
--- a/common/src/test/java/org/apache/sqoop/validation/validators/TestNotEmpty.java
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestNotEmpty.java
@@ -29,7 +29,7 @@ import static org.junit.Assert.assertEquals;
*/
public class TestNotEmpty {
- Validator validator = new NotEmpty();
+ AbstractValidator validator = new NotEmpty();
@Test
public void test() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/test/java/org/apache/sqoop/validation/validators/TestNotNull.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestNotNull.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestNotNull.java
index 9c5bed5..91e5398 100644
--- a/common/src/test/java/org/apache/sqoop/validation/validators/TestNotNull.java
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestNotNull.java
@@ -29,7 +29,7 @@ import static org.junit.Assert.assertEquals;
*/
public class TestNotNull {
- Validator validator = new NotNull();
+ AbstractValidator validator = new NotNull();
@Test
public void test() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java
new file mode 100644
index 0000000..92409d9
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestStartsWith.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.validation.validators;
+
+import org.apache.sqoop.validation.Status;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ */
+public class TestStartsWith {
+
+ AbstractValidator validator = new StartsWith();
+
+ @Test
+ public void test() {
+ assertEquals(0, validator.getMessages().size());
+
+ // Default, no string argument set
+ validator.validate("str");
+ assertEquals(Status.FINE, validator.getStatus());
+ assertEquals(0, validator.getMessages().size());
+
+ // Searched substring is entire string
+ validator.validate("str");
+ assertEquals(Status.FINE, validator.getStatus());
+ assertEquals(0, validator.getMessages().size());
+
+ // Just starts with
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("strstr");
+ assertEquals(Status.FINE, validator.getStatus());
+ assertEquals(0, validator.getMessages().size());
+
+ // Null string
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate(null);
+ assertEquals(Status.UNACCEPTABLE, validator.getStatus());
+ assertEquals(1, validator.getMessages().size());
+
+ // Empty string
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("");
+ assertEquals(Status.UNACCEPTABLE, validator.getStatus());
+ assertEquals(1, validator.getMessages().size());
+
+ // "Random" string
+ validator.reset();
+ validator.setStringArgument("str");
+ validator.validate("Ahoj tady je meduza");
+ assertEquals(Status.UNACCEPTABLE, validator.getStatus());
+ assertEquals(1, validator.getMessages().size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/common/src/test/java/org/apache/sqoop/validation/validators/TestValidator.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestValidator.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestValidator.java
index 1a5dbdd..3f60ee6 100644
--- a/common/src/test/java/org/apache/sqoop/validation/validators/TestValidator.java
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestValidator.java
@@ -27,7 +27,7 @@ import static org.junit.Assert.assertEquals;
*
*/
public class TestValidator {
- public static class ValidatorImpl extends Validator<String> {
+ public static class ValidatorImpl extends AbstractValidator<String> {
@Override
public void validate(String msg) {
addMessage(Status.FINE, msg);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java
index e513770..3b5d70a 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ConnectionForm.java
@@ -19,10 +19,12 @@ package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.validators.AbstractValidator;
import org.apache.sqoop.validation.validators.NotEmpty;
-import org.apache.sqoop.validation.validators.Validator;
import org.apache.sqoop.validation.validators.ClassAvailable;
+import org.apache.sqoop.validation.validators.StartsWith;
import java.sql.DriverManager;
import java.sql.SQLException;
@@ -31,12 +33,12 @@ import java.util.Map;
/**
*
*/
-@FormClass(validators = {ConnectionForm.FormValidator.class})
+@FormClass(validators = {@Validator(ConnectionForm.FormValidator.class)})
public class ConnectionForm {
- @Input(size = 128, validators = {NotEmpty.class, ClassAvailable.class} )
+ @Input(size = 128, validators = {@Validator(NotEmpty.class), @Validator(ClassAvailable.class)} )
public String jdbcDriver;
- @Input(size = 128, validators = {NotEmpty.class} )
+ @Input(size = 128, validators = {@Validator(value = StartsWith.class, strArg = "jdbc:")} )
public String connectionString;
@Input(size = 40)
@@ -48,7 +50,7 @@ public class ConnectionForm {
@Input
public Map<String, String> jdbcProperties;
- public static class FormValidator extends Validator<ConnectionForm> {
+ public static class FormValidator extends AbstractValidator<ConnectionForm> {
@Override
public void validate(ConnectionForm form) {
// See if we can connect to the database
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java
index 1c0b429..ac72546 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java
@@ -20,13 +20,15 @@ package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.connector.jdbc.GenericJdbcConnectorConstants;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.validators.Validator;
+import org.apache.sqoop.validation.validators.AbstractValidator;
+import org.apache.sqoop.validation.validators.Contains;
/**
*
*/
-@FormClass( validators = {FromTableForm.FormValidator.class})
+@FormClass( validators = {@Validator(FromTableForm.FormValidator.class)})
public class FromTableForm {
@Input(size = 50)
public String schemaName;
@@ -34,7 +36,7 @@ public class FromTableForm {
@Input(size = 50)
public String tableName;
- @Input(size = 2000, validators = {SqlConditionTokenValidator.class})
+ @Input(size = 2000, validators = {@Validator(value = Contains.class, strArg = GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN)})
public String sql;
@Input(size = 50)
@@ -49,7 +51,7 @@ public class FromTableForm {
@Input(size = 50)
public String boundaryQuery;
- public static class FormValidator extends Validator<FromTableForm> {
+ public static class FormValidator extends AbstractValidator<FromTableForm> {
@Override
public void validate(FromTableForm form) {
if(form.tableName == null && form.sql == null) {
@@ -63,13 +65,4 @@ public class FromTableForm {
}
}
}
-
- public static class SqlConditionTokenValidator extends Validator<String> {
- @Override
- public void validate(String sql) {
- if(sql != null && !sql.contains(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN)) {
- addMessage(Status.UNACCEPTABLE, "SQL statement must contain placeholder for auto generated conditions - " + GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN);
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e71dd757/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java
index 0601a39..642da11 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToTableForm.java
@@ -19,13 +19,14 @@ package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.Input;
+import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.validators.Validator;
+import org.apache.sqoop.validation.validators.AbstractValidator;
/**
*
*/
-@FormClass(validators = {ToTableForm.FormValidator.class})
+@FormClass(validators = {@Validator(ToTableForm.FormValidator.class)})
public class ToTableForm {
@Input(size = 50) public String schemaName;
@Input(size = 2000) public String tableName;
@@ -34,7 +35,7 @@ public class ToTableForm {
@Input(size = 2000) public String stageTableName;
@Input public Boolean clearStageTable;
- public static class FormValidator extends Validator<ToTableForm> {
+ public static class FormValidator extends AbstractValidator<ToTableForm> {
@Override
public void validate(ToTableForm form) {
if(form.tableName == null && form.sql == null) {
[07/52] [abbrv] git commit: SQOOP-1542: Sqoop2: From/To: MConnector
should handle null forms
Posted by ab...@apache.org.
SQOOP-1542: Sqoop2: From/To: MConnector should handle null forms
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d0153621
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d0153621
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d0153621
Branch: refs/heads/SQOOP-1367
Commit: d0153621d38c4c8478276417b12e36bd67ed6121
Parents: 049994a
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Sep 24 16:16:54 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:18 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/sqoop/model/MConnector.java | 64 +++++++++++++++++---
.../org/apache/sqoop/model/TestMConnector.java | 52 ++++++++++++++++
2 files changed, 107 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d0153621/common/src/main/java/org/apache/sqoop/model/MConnector.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConnector.java b/common/src/main/java/org/apache/sqoop/model/MConnector.java
index 3dc1014..7999b08 100644
--- a/common/src/main/java/org/apache/sqoop/model/MConnector.java
+++ b/common/src/main/java/org/apache/sqoop/model/MConnector.java
@@ -63,12 +63,18 @@ public final class MConnector extends MPersistableEntity implements MClonable {
@Override
public String toString() {
+ MJobForms fromJobForms = this.getJobForms(Direction.FROM);
+ MJobForms toJobForms = this.getJobForms(Direction.TO);
StringBuilder sb = new StringBuilder("connector-");
sb.append(uniqueName).append(":").append(getPersistenceId()).append(":");
sb.append(className);
sb.append(", ").append(getConnectionForms().toString());
- sb.append(", ").append(getJobForms(Direction.FROM).toString());
- sb.append(", ").append(getJobForms(Direction.TO).toString());
+ if (fromJobForms != null) {
+ sb.append(", ").append(fromJobForms.toString());
+ }
+ if (toJobForms != null) {
+ sb.append(", ").append(toJobForms.toString());
+ }
return sb.toString();
}
@@ -83,19 +89,47 @@ public final class MConnector extends MPersistableEntity implements MClonable {
}
MConnector mc = (MConnector) other;
+ SupportedDirections supportedDirections = this.getSupportedDirections();
+ SupportedDirections mcSupportedDirections = mc.getSupportedDirections();
+
+ if (supportedDirections.isDirectionSupported(Direction.FROM)
+ && mcSupportedDirections.isDirectionSupported(Direction.FROM)
+ && !getJobForms(Direction.FROM).equals(mc.getJobForms(Direction.FROM))) {
+ return false;
+ }
+
+ if (supportedDirections.isDirectionSupported(Direction.FROM)
+ != mcSupportedDirections.isDirectionSupported(Direction.FROM)) {
+ return false;
+ }
+
+ if (supportedDirections.isDirectionSupported(Direction.TO)
+ && mcSupportedDirections.isDirectionSupported(Direction.TO)
+ && !getJobForms(Direction.TO).equals(mc.getJobForms(Direction.TO))) {
+ return false;
+ }
+
+ if (supportedDirections.isDirectionSupported(Direction.TO)
+ != mcSupportedDirections.isDirectionSupported(Direction.TO)) {
+ return false;
+ }
+
return uniqueName.equals(mc.uniqueName)
&& className.equals(mc.className)
&& version.equals(mc.version)
- && connectionForms.equals(mc.getConnectionForms())
- && fromJobForms.equals(mc.getJobForms(Direction.FROM))
- && toJobForms.equals(mc.getJobForms(Direction.TO));
+ && connectionForms.equals(mc.getConnectionForms());
}
@Override
public int hashCode() {
+ SupportedDirections supportedDirections = getSupportedDirections();
int result = getConnectionForms().hashCode();
- result = 31 * result + getJobForms(Direction.FROM).hashCode();
- result = 31 * result + getJobForms(Direction.TO).hashCode();
+ if (supportedDirections.isDirectionSupported(Direction.FROM)) {
+ result = 31 * result + getJobForms(Direction.FROM).hashCode();
+ }
+ if (supportedDirections.isDirectionSupported(Direction.TO)) {
+ result = 31 * result + getJobForms(Direction.TO).hashCode();
+ }
result = 31 * result + version.hashCode();
result = 31 * result + uniqueName.hashCode();
result = 31 * result + className.hashCode();
@@ -105,13 +139,25 @@ public final class MConnector extends MPersistableEntity implements MClonable {
public MConnector clone(boolean cloneWithValue) {
//Connector never have any values filled
cloneWithValue = false;
+
+ MJobForms fromJobForms = this.getJobForms(Direction.FROM);
+ MJobForms toJobForms = this.getJobForms(Direction.TO);
+
+ if (fromJobForms != null) {
+ fromJobForms = fromJobForms.clone(cloneWithValue);
+ }
+
+ if (toJobForms != null) {
+ toJobForms = toJobForms.clone(cloneWithValue);
+ }
+
MConnector copy = new MConnector(
this.getUniqueName(),
this.getClassName(),
this.getVersion(),
this.getConnectionForms().clone(cloneWithValue),
- this.getJobForms(Direction.FROM).clone(cloneWithValue),
- this.getJobForms(Direction.TO).clone(cloneWithValue));
+ fromJobForms,
+ toJobForms);
copy.setPersistenceId(this.getPersistenceId());
return copy;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d0153621/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConnector.java b/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
index 3fde47b..0a5fd90 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
@@ -132,6 +132,58 @@ public class TestMConnector {
}
@Test
+ public void testFromDirection() {
+ MConnector connector = createConnector(Arrays.asList(Direction.FROM));
+
+ // Clone should clone only one job form.
+ MConnector clone = connector.clone(true);
+ assertNotNull(clone.getJobForms(Direction.FROM));
+ assertNull(clone.getJobForms(Direction.TO));
+ assertEquals(connector, clone);
+ assertEquals(connector.toString(), clone.toString());
+ assertNotEquals(connector.hashCode(), clone.hashCode());
+ }
+
+ @Test
+ public void testToDirection() {
+ MConnector connector = createConnector(Arrays.asList(Direction.TO));
+
+ // Clone should clone only one job form.
+ MConnector clone = connector.clone(true);
+ assertNull(clone.getJobForms(Direction.FROM));
+ assertNotNull(clone.getJobForms(Direction.TO));
+ assertEquals(connector, clone);
+ assertEquals(connector.toString(), clone.toString());
+ assertNotEquals(connector.hashCode(), clone.hashCode());
+ }
+
+ @Test
+ public void testNoDirection() {
+ MConnector connector = createConnector(Arrays.asList(new Direction[0]));
+
+ // Clone should clone only one job form.
+ MConnector clone = connector.clone(true);
+ assertNull(clone.getJobForms(Direction.FROM));
+ assertNull(clone.getJobForms(Direction.TO));
+ assertEquals(connector, clone);
+ assertEquals(connector.toString(), clone.toString());
+ assertNotEquals(connector.hashCode(), clone.hashCode());
+ }
+
+ @Test
+ public void testBothDirections() {
+ MConnector connector = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
+
+ // Clone should clone only one job form.
+ MConnector clone = connector.clone(true);
+ assertNotNull(clone.getJobForms(Direction.FROM));
+ assertNotNull(clone.getJobForms(Direction.TO));
+ assertEquals(connector, clone);
+ assertEquals(connector.toString(), clone.toString());
+ assertNotEquals(connector.hashCode(), clone.hashCode());
+ }
+
+ @Test
public void testGetSupportedDirections() {
MConnector connector = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
assertTrue(connector.getSupportedDirections().isDirectionSupported(Direction.FROM));
[12/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
index 0fcae75..796b021 100644
--- a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
+++ b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
@@ -17,6 +17,23 @@
*/
package org.apache.sqoop.repository;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Collections;
@@ -26,39 +43,35 @@ import java.util.List;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.framework.FrameworkManager;
-import org.apache.sqoop.framework.configuration.JobConfiguration;
+import org.apache.sqoop.driver.Driver;
+import org.apache.sqoop.driver.configuration.JobConfiguration;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnection;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFramework;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
-
import org.apache.sqoop.validation.Validator;
import org.junit.Before;
import org.junit.Test;
import org.mockito.InOrder;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
public class TestJdbcRepository {
private JdbcRepository repoSpy;
private JdbcRepositoryTransaction repoTransactionMock;
private ConnectorManager connectorMgrMock;
- private FrameworkManager frameworkMgrMock;
+ private Driver driverMock;
private JdbcRepositoryHandler repoHandlerMock;
private Validator validatorMock;
- private MetadataUpgrader upgraderMock;
+ private RepositoryUpgrader upgraderMock;
private Validation validRepoMock;
private Validation invalidRepoMock;
@@ -67,16 +80,16 @@ public class TestJdbcRepository {
public void setUp() throws Exception {
repoTransactionMock = mock(JdbcRepositoryTransaction.class);
connectorMgrMock = mock(ConnectorManager.class);
- frameworkMgrMock = mock(FrameworkManager.class);
+ driverMock = mock(Driver.class);
repoHandlerMock = mock(JdbcRepositoryHandler.class);
validatorMock = mock(Validator.class);
- upgraderMock = mock(MetadataUpgrader.class);
+ upgraderMock = mock(RepositoryUpgrader.class);
repoSpy = spy(new JdbcRepository(repoHandlerMock, null));
// setup transaction and connector manager
doReturn(repoTransactionMock).when(repoSpy).getTransaction();
ConnectorManager.setInstance(connectorMgrMock);
- FrameworkManager.setInstance(frameworkMgrMock);
+ Driver.setInstance(driverMock);
validRepoMock = mock(Validation.class);
when(validRepoMock.getStatus()).thenReturn(Status.ACCEPTABLE);
@@ -138,26 +151,26 @@ public class TestJdbcRepository {
}
/**
- * Test the procedure when the framework auto upgrade option is enabled
+ * Test the procedure when the driverConfig auto upgrade option is enabled
*/
@Test
- public void testFrameworkEnableAutoUpgrade() {
- MFramework newFramework = framework();
- MFramework oldFramework = anotherFramework();
+ public void testDriverConfigEnableAutoUpgrade() {
+ MDriverConfig newDriverConfig = driverConfig();
+ MDriverConfig oldDriverConfig = anotherDriverConfig();
- when(repoHandlerMock.findFramework(any(Connection.class))).thenReturn(oldFramework);
+ when(repoHandlerMock.findDriverConfig(any(Connection.class))).thenReturn(oldDriverConfig);
- // make the upgradeFramework to throw an exception to prove that it has been called
+ // make the upgradeDriverConfig to throw an exception to prove that it has been called
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "upgradeFramework() has been called.");
- doThrow(exception).when(repoHandlerMock).findConnections(any(Connection.class));
+ "upgradeDriverConfig() has been called.");
+ doThrow(exception).when(repoHandlerMock).findLinks(any(Connection.class));
try {
- repoSpy.registerFramework(newFramework, true);
+ repoSpy.registerDriverConfig(newDriverConfig, true);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findFramework(any(Connection.class));
- verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findDriverConfig(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -166,20 +179,20 @@ public class TestJdbcRepository {
}
/**
- * Test the procedure when the framework auto upgrade option is disabled
+ * Test the procedure when the driverConfig auto upgrade option is disabled
*/
@Test
- public void testFrameworkDisableAutoUpgrade() {
- MFramework newFramework = framework();
- MFramework oldFramework = anotherFramework();
+ public void testDriverConfigDisableAutoUpgrade() {
+ MDriverConfig newDriverConfig = driverConfig();
+ MDriverConfig oldDriverConfig = anotherDriverConfig();
- when(repoHandlerMock.findFramework(any(Connection.class))).thenReturn(oldFramework);
+ when(repoHandlerMock.findDriverConfig(any(Connection.class))).thenReturn(oldDriverConfig);
try {
- repoSpy.registerFramework(newFramework, false);
+ repoSpy.registerDriverConfig(newDriverConfig, false);
} catch (SqoopException ex) {
assertEquals(ex.getErrorCode(), RepositoryError.JDBCREPO_0026);
- verify(repoHandlerMock, times(1)).findFramework(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findDriverConfig(any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -188,32 +201,32 @@ public class TestJdbcRepository {
}
/**
- * Test the connector upgrade procedure, when all the connections and
+ * Test the connector upgrade procedure, when all the links and
* jobs using the old connector are still valid for the new connector
*/
@Test
- public void testConnectorUpgradeWithValidConnectionsAndJobs() {
+ public void testConnectorUpgradeWithValidLinksAndJobs() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
// prepare the sqoop connector
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
when(sqconnector.getValidator()).thenReturn(validatorMock);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
- when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
+ when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
- // prepare the connections and jobs
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ // prepare the links and jobs
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
// mock necessary methods for upgradeConnector() procedure
- doReturn(connectionList).when(repoSpy).findConnectionsForConnector(anyLong());
+ doReturn(linkList).when(repoSpy).findLinksForConnector(anyLong());
doReturn(jobList).when(repoSpy).findJobsForConnector(anyLong());
- doNothing().when(repoSpy).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateLink(any(MLink.class), any(RepositoryTransaction.class));
doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
doNothing().when(repoSpy).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
@@ -224,15 +237,15 @@ public class TestJdbcRepository {
InOrder upgraderOrder = inOrder(upgraderMock);
InOrder validatorOrder = inOrder(validatorMock);
- repoOrder.verify(repoSpy, times(1)).findConnectionsForConnector(anyLong());
+ repoOrder.verify(repoSpy, times(1)).findLinksForConnector(anyLong());
repoOrder.verify(repoSpy, times(1)).findJobsForConnector(anyLong());
repoOrder.verify(repoSpy, times(1)).getTransaction();
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(1, repoTransactionMock);
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(1, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(2, repoTransactionMock);
repoOrder.verify(repoSpy, times(1)).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
- repoOrder.verify(repoSpy, times(2)).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(2)).updateLink(any(MLink.class), any(RepositoryTransaction.class));
repoOrder.verify(repoSpy, times(4)).updateJob(any(MJob.class), any(RepositoryTransaction.class));
repoOrder.verifyNoMoreInteractions();
txOrder.verify(repoTransactionMock, times(1)).begin();
@@ -242,7 +255,7 @@ public class TestJdbcRepository {
upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
upgraderOrder.verify(upgraderMock, times(4)).upgrade(any(MJobForms.class), any(MJobForms.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validatorMock, times(2)).validateConnection(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateLink(anyObject());
// @TODO(Abe): Re-enable job validation?
validatorOrder.verify(validatorMock, times(0)).validateJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
@@ -250,31 +263,31 @@ public class TestJdbcRepository {
/**
* @TODO(Abe): To re-enable with Validation in Repository upgrade.
- * Test the connector upgrade procedure, when all the connections and
+ * Test the connector upgrade procedure, when all the links and
* jobs using the old connector are invalid for the new connector
*/
// @Test
-// public void testConnectorUpgradeWithInvalidConnectionsAndJobs() {
+// public void testConnectorUpgradeWithInvalidLinksAndJobs() {
// MConnector newConnector = connector(1, "1.1");
// MConnector oldConnector = connector(1);
//
// // prepare the sqoop connector
// SqoopConnector sqconnector = mock(SqoopConnector.class);
-// when(validator.validateConnection(any(MConnection.class))).thenReturn(invalid);
+// when(validator.validateLink(any(MLink.class))).thenReturn(invalid);
// when(validator.validateJob(any(MJob.class))).thenReturn(invalid);
// when(sqconnector.getValidator()).thenReturn(validator);
-// when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
-// when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+// when(sqconnector.getDriverConfigRepositoryUpgrader()).thenReturn(upgrader);
+// when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
// when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
// when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
//
-// // prepare the connections and jobs
-// List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+// // prepare the links and jobs
+// List<MLink> linkList = links(link(1,1), link(2,1));
// List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
//
-// doReturn(connectionList).when(repo).findConnectionsForConnector(anyLong());
+// doReturn(linkList).when(repo).findLinksForConnector(anyLong());
// doReturn(jobList).when(repo).findJobsForConnector(anyLong());
-// doNothing().when(repo).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+// doNothing().when(repo).updateLink(any(MLink.class), any(RepositoryTransaction.class));
// doNothing().when(repo).updateJob(any(MJob.class), any(RepositoryTransaction.class));
// doNothing().when(repo).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
//
@@ -288,13 +301,13 @@ public class TestJdbcRepository {
// InOrder upgraderOrder = inOrder(upgrader);
// InOrder validatorOrder = inOrder(validator);
//
-// repoOrder.verify(repo, times(1)).findConnectionsForConnector(anyLong());
+// repoOrder.verify(repo, times(1)).findLinksForConnector(anyLong());
// repoOrder.verify(repo, times(1)).findJobsForConnector(anyLong());
// repoOrder.verify(repo, times(1)).getTransaction();
// repoOrder.verify(repo, times(1)).deleteJobInputs(1, tx);
// repoOrder.verify(repo, times(1)).deleteJobInputs(2, tx);
-// repoOrder.verify(repo, times(1)).deleteConnectionInputs(1, tx);
-// repoOrder.verify(repo, times(1)).deleteConnectionInputs(2, tx);
+// repoOrder.verify(repo, times(1)).deleteLinkInputs(1, tx);
+// repoOrder.verify(repo, times(1)).deleteLinkInputs(2, tx);
// repoOrder.verify(repo, times(1)).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
// repoOrder.verifyNoMoreInteractions();
// txOrder.verify(tx, times(1)).begin();
@@ -304,7 +317,7 @@ public class TestJdbcRepository {
// upgraderOrder.verify(upgrader, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
// upgraderOrder.verify(upgrader, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
// upgraderOrder.verifyNoMoreInteractions();
-// validatorOrder.verify(validator, times(2)).validateConnection(anyObject());
+// validatorOrder.verify(validator, times(2)).validateLink(anyObject());
// validatorOrder.verify(validator, times(2)).validateJob(anyObject());
// validatorOrder.verifyNoMoreInteractions();
// return ;
@@ -314,45 +327,45 @@ public class TestJdbcRepository {
// }
/**
- * Test the framework upgrade procedure, when all the connections and
+ * Test the driverConfig upgrade procedure, when all the links and
* jobs using the old connector are still valid for the new connector
*/
@Test
- public void testFrameworkUpgradeWithValidConnectionsAndJobs() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeWithValidLinksAndJobs() {
+ MDriverConfig newDriverConfig = driverConfig();
- when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
- when(frameworkMgrMock.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(frameworkMgrMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(driverMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoSpy).findConnections();
+ doReturn(linkList).when(repoSpy).findLinks();
doReturn(jobList).when(repoSpy).findJobs();
- doNothing().when(repoSpy).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateLink(any(MLink.class), any(RepositoryTransaction.class));
doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
- doNothing().when(repoSpy).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateDriverConfig(any(MDriverConfig.class), any(RepositoryTransaction.class));
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(newDriverConfig);
InOrder repoOrder = inOrder(repoSpy);
InOrder txOrder = inOrder(repoTransactionMock);
InOrder upgraderOrder = inOrder(upgraderMock);
InOrder validatorOrder = inOrder(validatorMock);
- repoOrder.verify(repoSpy, times(1)).findConnections();
+ repoOrder.verify(repoSpy, times(1)).findLinks();
repoOrder.verify(repoSpy, times(1)).findJobs();
repoOrder.verify(repoSpy, times(1)).getTransaction();
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(1, repoTransactionMock);
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(1, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
- repoOrder.verify(repoSpy, times(2)).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).updateDriverConfig(any(MDriverConfig.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(2)).updateLink(any(MLink.class), any(RepositoryTransaction.class));
repoOrder.verify(repoSpy, times(2)).updateJob(any(MJob.class), any(RepositoryTransaction.class));
repoOrder.verifyNoMoreInteractions();
txOrder.verify(repoTransactionMock, times(1)).begin();
@@ -362,37 +375,37 @@ public class TestJdbcRepository {
upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validatorMock, times(2)).validateConnection(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateLink(anyObject());
validatorOrder.verify(validatorMock, times(2)).validateJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
}
/**
- * Test the framework upgrade procedure, when all the connections and
+ * Test the driverConfig upgrade procedure, when all the links and
* jobs using the old connector are invalid for the new connector
*/
@Test
- public void testFrameworkUpgradeWithInvalidConnectionsAndJobs() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeWithInvalidLinksAndJobs() {
+ MDriverConfig newDriverConfig = driverConfig();
- when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(invalidRepoMock);
+ when(validatorMock.validateLink(any(MLink.class))).thenReturn(invalidRepoMock);
when(validatorMock.validateJob(any(MJob.class))).thenReturn(invalidRepoMock);
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
- when(frameworkMgrMock.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(frameworkMgrMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(driverMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoSpy).findConnections();
+ doReturn(linkList).when(repoSpy).findLinks();
doReturn(jobList).when(repoSpy).findJobs();
- doNothing().when(repoSpy).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateLink(any(MLink.class), any(RepositoryTransaction.class));
doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
- doNothing().when(repoSpy).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateDriverConfig(any(MDriverConfig.class), any(RepositoryTransaction.class));
try {
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getErrorCode(), RepositoryError.JDBCREPO_0027);
@@ -401,14 +414,14 @@ public class TestJdbcRepository {
InOrder upgraderOrder = inOrder(upgraderMock);
InOrder validatorOrder = inOrder(validatorMock);
- repoOrder.verify(repoSpy, times(1)).findConnections();
+ repoOrder.verify(repoSpy, times(1)).findLinks();
repoOrder.verify(repoSpy, times(1)).findJobs();
repoOrder.verify(repoSpy, times(1)).getTransaction();
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(1, repoTransactionMock);
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(1, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).updateDriverConfig(any(MDriverConfig.class), any(RepositoryTransaction.class));
repoOrder.verifyNoMoreInteractions();
txOrder.verify(repoTransactionMock, times(1)).begin();
txOrder.verify(repoTransactionMock, times(1)).rollback();
@@ -417,7 +430,7 @@ public class TestJdbcRepository {
upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validatorMock, times(2)).validateConnection(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateLink(anyObject());
validatorOrder.verify(validatorMock, times(2)).validateJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
return ;
@@ -428,27 +441,27 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * find connections for a given connector
+ * find links for a given connector
*/
@Test
- public void testConnectorUpgradeHandlerFindConnectionsForConnectorError() {
+ public void testConnectorUpgradeHandlerFindLinksForConnectorError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
when(sqconnector.getValidator()).thenReturn(validatorMock);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "find connections for connector error.");
- doThrow(exception).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ "find links for connector error.");
+ doThrow(exception).when(repoHandlerMock).findLinksForConnector(anyLong(), any(Connection.class));
try {
repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinksForConnector(anyLong(), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -467,11 +480,11 @@ public class TestJdbcRepository {
SqoopConnector sqconnector = mock(SqoopConnector.class);
when(sqconnector.getValidator()).thenReturn(validatorMock);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ List<MLink> linkList = links(link(1,1), link(2,1));
+ doReturn(linkList).when(repoHandlerMock).findLinksForConnector(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"find jobs for connector error.");
@@ -481,7 +494,7 @@ public class TestJdbcRepository {
repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinksForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
@@ -501,12 +514,12 @@ public class TestJdbcRepository {
SqoopConnector sqconnector = mock(SqoopConnector.class);
when(sqconnector.getValidator()).thenReturn(validatorMock);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinksForConnector(anyLong(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
@@ -517,7 +530,7 @@ public class TestJdbcRepository {
repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinksForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).deleteJobInputs(anyLong(), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
@@ -529,36 +542,36 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * delete connection inputs for a given connector
+ * delete link inputs for a given connector
*/
@Test
- public void testConnectorUpgradeHandlerDeleteConnectionInputsError() {
+ public void testConnectorUpgradeHandlerDeleteLinkInputsError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
when(sqconnector.getValidator()).thenReturn(validatorMock);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinksForConnector(anyLong(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "delete connection inputs for connector error.");
- doThrow(exception).when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ "delete link inputs for connector error.");
+ doThrow(exception).when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
try {
repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinksForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).deleteLinkInputs(anyLong(), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -568,7 +581,7 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * update the connector metadata
+ * update the connector entity
*/
@Test
public void testConnectorUpgradeHandlerUpdateConnectorError() {
@@ -577,15 +590,15 @@ public class TestJdbcRepository {
SqoopConnector sqconnector = mock(SqoopConnector.class);
when(sqconnector.getValidator()).thenReturn(validatorMock);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinksForConnector(anyLong(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update connector error.");
@@ -595,10 +608,10 @@ public class TestJdbcRepository {
repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinksForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
@@ -609,46 +622,46 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * update the connection metadata
+ * update the link entity
*/
@Test
- public void testConnectorUpgradeHandlerUpdateConnectionError() {
+ public void testConnectorUpgradeHandlerUpdateLinkError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
when(sqconnector.getValidator()).thenReturn(validatorMock);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
- when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
+ when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinksForConnector(anyLong(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).updateConnector(any(MConnector.class), any(Connection.class));
- doReturn(true).when(repoHandlerMock).existsConnection(anyLong(), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsLink(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "update connection error.");
- doThrow(exception).when(repoHandlerMock).updateConnection(any(MConnection.class), any(Connection.class));
+ "update link error.");
+ doThrow(exception).when(repoHandlerMock).updateLink(any(MLink.class), any(Connection.class));
try {
repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinksForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
- verify(repoHandlerMock, times(1)).existsConnection(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateConnection(any(MConnection.class), any(Connection.class));
+ verify(repoHandlerMock, times(1)).existsLink(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateLink(any(MLink.class), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -658,7 +671,7 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * update the job metadata
+ * update the job entity
*/
@Test
public void testConnectorUpgradeHandlerUpdateJobError() {
@@ -666,23 +679,23 @@ public class TestJdbcRepository {
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
when(sqconnector.getValidator()).thenReturn(validatorMock);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
- when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
+ when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinksForConnector(anyLong(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).updateConnector(any(MConnector.class), any(Connection.class));
- doNothing().when(repoHandlerMock).updateConnection(any(MConnection.class), any(Connection.class));
- doReturn(true).when(repoHandlerMock).existsConnection(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateLink(any(MLink.class), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsLink(anyLong(), any(Connection.class));
doReturn(true).when(repoHandlerMock).existsJob(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
@@ -693,13 +706,13 @@ public class TestJdbcRepository {
repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinksForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
- verify(repoHandlerMock, times(2)).existsConnection(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).updateConnection(any(MConnection.class), any(Connection.class));
+ verify(repoHandlerMock, times(2)).existsLink(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).updateLink(any(MLink.class), any(Connection.class));
verify(repoHandlerMock, times(1)).existsJob(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).updateJob(any(MJob.class), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
@@ -711,24 +724,24 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * find connections for framework
+ * find links for driverConfig
*/
@Test
- public void testFrameworkUpgradeHandlerFindConnectionsError() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeHandlerFindLinksError() {
+ MDriverConfig newDriverConfig = driverConfig();
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "find connections error.");
- doThrow(exception).when(repoHandlerMock).findConnections(any(Connection.class));
+ "find links error.");
+ doThrow(exception).when(repoHandlerMock).findLinks(any(Connection.class));
try {
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -738,27 +751,27 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * find jobs for framework
+ * find jobs for driverConfig
*/
@Test
- public void testFrameworkUpgradeHandlerFindJobsError() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeHandlerFindJobsError() {
+ MDriverConfig newDriverConfig = driverConfig();
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ List<MLink> linkList = links(link(1,1), link(2,1));
+ doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"find jobs error.");
doThrow(exception).when(repoHandlerMock).findJobs(any(Connection.class));
try {
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
@@ -769,18 +782,18 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * delete job inputs for framework upgrade
+ * delete job inputs for driverConfig upgrade
*/
@Test
- public void testFrameworkUpgradeHandlerDeleteJobInputsError() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeHandlerDeleteJobInputsError() {
+ MDriverConfig newDriverConfig = driverConfig();
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
@@ -788,10 +801,10 @@ public class TestJdbcRepository {
doThrow(exception).when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
try {
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verify(repoHandlerMock, times(1)).deleteJobInputs(anyLong(), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
@@ -803,33 +816,33 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * delete connection inputs for framework upgrade
+ * delete link inputs for driverConfig upgrade
*/
@Test
- public void testFrameworkUpgradeHandlerDeleteConnectionInputsError() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeHandlerDeleteConnectionInputsError() {
+ MDriverConfig newDriverConfig = driverConfig();
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "delete connection inputs error.");
- doThrow(exception).when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ "delete link inputs error.");
+ doThrow(exception).when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
try {
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).deleteLinkInputs(anyLong(), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -839,35 +852,35 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * update the framework metadata
+ * update the driverConfig entity
*/
@Test
- public void testFrameworkUpgradeHandlerUpdateFrameworkError() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeHandlerUpdateDriverConfigError() {
+ MDriverConfig newDriverConfig = driverConfig();
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "update framework metadata error.");
- doThrow(exception).when(repoHandlerMock).updateFramework(any(MFramework.class), any(Connection.class));
+ "update driverConfig entity error.");
+ doThrow(exception).when(repoHandlerMock).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
try {
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -877,43 +890,43 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * update the connection metadata
+ * update the link entity
*/
@Test
- public void testFrameworkUpgradeHandlerUpdateConnectionError() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeHandlerUpdateConnectionError() {
+ MDriverConfig newDriverConfig = driverConfig();
- when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
- when(frameworkMgrMock.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(frameworkMgrMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(driverMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).updateFramework(any(MFramework.class), any(Connection.class));
- doReturn(true).when(repoHandlerMock).existsConnection(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsLink(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "update connection error.");
- doThrow(exception).when(repoHandlerMock).updateConnection(any(MConnection.class), any(Connection.class));
+ "update link error.");
+ doThrow(exception).when(repoHandlerMock).updateLink(any(MLink.class), any(Connection.class));
try {
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
- verify(repoHandlerMock, times(1)).existsConnection(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateConnection(any(MConnection.class), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
+ verify(repoHandlerMock, times(1)).existsLink(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateLink(any(MLink.class), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -923,45 +936,45 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * update the job metadata
+ * update the job entity
*/
@Test
- public void testFrameworkUpgradeHandlerUpdateJobError() {
- MFramework newFramework = framework();
+ public void testDriverConfigUpgradeHandlerUpdateJobError() {
+ MDriverConfig driverConfig = driverConfig();
- when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
- when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
- when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
- when(frameworkMgrMock.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(frameworkMgrMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(driverMock.getValidator()).thenReturn(validatorMock);
+ when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
+ when(driverMock.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(driverMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
- List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
+ List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).updateFramework(any(MFramework.class), any(Connection.class));
- doReturn(true).when(repoHandlerMock).existsConnection(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsLink(anyLong(), any(Connection.class));
doReturn(true).when(repoHandlerMock).existsJob(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).updateConnection(any(MConnection.class), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateLink(any(MLink.class), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update job error.");
doThrow(exception).when(repoHandlerMock).updateJob(any(MJob.class), any(Connection.class));
try {
- repoSpy.upgradeFramework(newFramework);
+ repoSpy.upgradeDriverConfig(driverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
- verify(repoHandlerMock, times(2)).existsConnection(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).updateConnection(any(MConnection.class), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
+ verify(repoHandlerMock, times(2)).existsLink(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).updateLink(any(MLink.class), any(Connection.class));
verify(repoHandlerMock, times(1)).existsJob(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).updateJob(any(MJob.class), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
@@ -971,44 +984,44 @@ public class TestJdbcRepository {
fail("Should throw out an exception with message: " + exception.getMessage());
}
- private MConnector connector(long id, String version) {
- MConnector connector = new MConnector("A" + id, "A" + id, version + id,
+ private MConnector connector(long connectorId, String version) {
+ MConnector connector = new MConnector("A" + connectorId, "A" + connectorId, version + connectorId,
new MConnectionForms(new LinkedList<MForm>()),
new MJobForms(FormUtils.toForms(JobConfiguration.class)),
new MJobForms(FormUtils.toForms(JobConfiguration.class)));
- connector.setPersistenceId(id);
+ connector.setPersistenceId(connectorId);
return connector;
}
- private MConnector connector(long id) {
- return connector(id, "1.0");
+ private MConnector connector(long connectoId) {
+ return connector(connectoId, "1.0");
}
- private MFramework framework() {
- MFramework framework = new MFramework(
+ private MDriverConfig driverConfig() {
+ MDriverConfig driverConfig = new MDriverConfig(
new MConnectionForms(new LinkedList<MForm>()),
new MJobForms(FormUtils.toForms(JobConfiguration.class)),
- FrameworkManager.CURRENT_FRAMEWORK_VERSION);
- framework.setPersistenceId(1);
- return framework;
+ Driver.CURRENT_DRIVER_VERSION);
+ driverConfig.setPersistenceId(1);
+ return driverConfig;
}
- private MFramework anotherFramework() {
- MFramework framework = new MFramework(null, null,
- FrameworkManager.CURRENT_FRAMEWORK_VERSION);
- framework.setPersistenceId(1);
- return framework;
+ private MDriverConfig anotherDriverConfig() {
+ MDriverConfig driverConfig = new MDriverConfig(null, null,
+ Driver.CURRENT_DRIVER_VERSION);
+ driverConfig.setPersistenceId(1);
+ return driverConfig;
}
- private MConnection connection(long id, long cid) {
- MConnection connection = new MConnection(cid, new MConnectionForms(new LinkedList<MForm>()),
+ private MLink link(long linkId, long connectorId) {
+ MLink link = new MLink(connectorId, new MConnectionForms(new LinkedList<MForm>()),
new MConnectionForms(new LinkedList<MForm>()));
- connection.setPersistenceId(id);
- return connection;
+ link.setPersistenceId(linkId);
+ return link;
}
- private MJob job(long id, long fromConnectorId, long toConnectorId, long fromConnectionId, long toConnectionId) {
- MJob job = new MJob(fromConnectorId, toConnectorId, fromConnectionId, toConnectionId,
+ private MJob job(long id, long fromConnectorId, long toConnectorId, long fromLinkId, long toLinkId) {
+ MJob job = new MJob(fromConnectorId, toConnectorId, fromLinkId, toLinkId,
new MJobForms(new LinkedList<MForm>()),
new MJobForms(new LinkedList<MForm>()),
new MJobForms(new LinkedList<MForm>()));
@@ -1016,10 +1029,10 @@ public class TestJdbcRepository {
return job;
}
- private List<MConnection> connections(MConnection ... cs) {
- List<MConnection> connections = new ArrayList<MConnection>();
- Collections.addAll(connections, cs);
- return connections;
+ private List<MLink> links(MLink ... ls) {
+ List<MLink> links = new ArrayList<MLink>();
+ Collections.addAll(links, ls);
+ return links;
}
private List<MJob> jobs(MJob ... js) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java
index 2d53dd2..5248b7b 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java
@@ -21,7 +21,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.sqoop.framework.JobRequest;
+import org.apache.sqoop.driver.JobRequest;
/**
* Map-reduce specific submission request containing all extra information
@@ -99,4 +99,4 @@ public class MRJobRequest extends JobRequest {
public void setOutputValueClass(Class<? extends Writable> outputValueClass) {
this.outputValueClass = outputValueClass;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
index 049d183..47f8478 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
@@ -19,8 +19,8 @@ package org.apache.sqoop.execution.mapreduce;
import org.apache.hadoop.io.NullWritable;
import org.apache.sqoop.common.MutableMapContext;
-import org.apache.sqoop.framework.ExecutionEngine;
-import org.apache.sqoop.framework.JobRequest;
+import org.apache.sqoop.driver.ExecutionEngine;
+import org.apache.sqoop.driver.JobRequest;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.etl.From;
import org.apache.sqoop.job.etl.To;
@@ -63,8 +63,7 @@ public class MapreduceExecutionEngine extends ExecutionEngine {
From from = (From) mrJobRequest.getFrom();
To to = (To) mrJobRequest.getTo();
-
- MutableMapContext context = mrJobRequest.getFrameworkContext();
+ MutableMapContext context = mrJobRequest.getDriverContext();
context.setString(JobConstants.JOB_ETL_PARTITIONER, from.getPartitioner().getName());
context.setString(JobConstants.JOB_ETL_EXTRACTOR, from.getExtractor().getName());
context.setString(JobConstants.JOB_ETL_LOADER, to.getLoader().getName());
@@ -85,10 +84,10 @@ public class MapreduceExecutionEngine extends ExecutionEngine {
* at mapreduce job time. This method will register all dependencies in the request
* object.
*
- * @param request Active request object.
+ * @param jobrequest Active job request object.
*/
- protected void addDependencies(MRJobRequest request) {
+ protected void addDependencies(MRJobRequest jobrequest) {
// Guava
- request.addJarForClass(ThreadFactoryBuilder.class);
+ jobrequest.addJarForClass(ThreadFactoryBuilder.class);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
index 542607a..349bb60 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.job;
import org.apache.sqoop.core.ConfigurationConstants;
-import org.apache.sqoop.framework.FrameworkConstants;
+import org.apache.sqoop.driver.DriverConstants;
public final class JobConstants extends Constants {
/**
@@ -73,7 +73,7 @@ public final class JobConstants extends Constants {
"mapred.output.compression.codec";
public static final String INTERMEDIATE_DATA_FORMAT =
- FrameworkConstants.PREFIX_EXECUTION_CONFIG + "intermediate.format";
+ DriverConstants.PREFIX_EXECUTION_CONFIG + "intermediate.format";
private JobConstants() {
// Disable explicit object creation
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
index 2996275..eea0623 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
@@ -250,7 +250,7 @@ public class SqoopOutputFormatLoadExecutor {
} catch (Throwable t) {
readerFinished = true;
LOG.error("Error while loading data out of MR job.", t);
- // Release so that the writer can tell the framework something went
+ // Release so that the writer can tell Sqoop something went
// wrong.
free.release();
throw new SqoopException(MapreduceExecutionError.MAPRED_EXEC_0018, t);
@@ -262,7 +262,7 @@ public class SqoopOutputFormatLoadExecutor {
// throw exception if data are not all consumed
readerFinished = true;
LOG.error("Reader terminated, but writer is still running!");
- // Release so that the writer can tell the framework something went
+ // Release so that the writer can tell Sqoop something went
// wrong.
free.release();
throw new SqoopException(MapreduceExecutionError.MAPRED_EXEC_0019);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java
index fdcecf2..74e41df 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java
@@ -23,7 +23,7 @@ public final class DerbyRepoConstants {
public static final String SYSKEY_VERSION = "version";
- public static final String SYSKEY_FRAMEWORK_VERSION = "framework.version";
+ public static final String SYSKEY_DRIVER_VERSION = "driver.version";
/**
* Expected version of the repository structures.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
index aae89fe..61dc3b4 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
@@ -1,7 +1,7 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
+ * distributed with this work for additional inconfigation
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
@@ -33,35 +33,35 @@ public enum DerbyRepoError implements ErrorCode {
/** The system was unable to run the specified query. */
DERBYREPO_0003("Unable to run specified query"),
- /** The system was unable to query the repository for metadata. */
- DERBYREPO_0004("Unable to retrieve metadata"),
+ /** The system was unable to query the repository for given entity */
+ DERBYREPO_0004("Unable to retrieve entity data"),
- /** The metadata repository contains more than one connector with same name */
- DERBYREPO_0005("Invalid metadata state - multiple connectors with name"),
+ /** The repository contains more than one connector with same name */
+ DERBYREPO_0005("Invalid entity state - multiple connectors with name"),
/** The system does not support the given input type.*/
DERBYREPO_0006("Unknown input type encountered"),
- /** The system does not support the given form type.*/
- DERBYREPO_0007("Unknown form type encountered"),
+ /** The system does not support the given config type.*/
+ DERBYREPO_0007("Unknown config type encountered"),
- /** No input metadata was found for the given form. */
- DERBYREPO_0008("The form contains no input metadata"),
+ /** No input was found for the given config. */
+ DERBYREPO_0008("The config contains no input"),
- /** The system could not load the form due to unexpected position of input.*/
- DERBYREPO_0009("The form input retrieved does not match expected position"),
+ /** The system could not load the config due to unexpected position of input.*/
+ DERBYREPO_0009("The config input retrieved does not match expected position"),
/**
* The system could not load the connector due to unexpected position
- * of form.
+ * of config.
*/
- DERBYREPO_0010("The form retrieved does not match expected position"),
+ DERBYREPO_0010("The config retrieved does not match expected position"),
/**
- * The system was not able to register metadata due to a pre-assigned
+ * The system was not able to register entity due to a pre-assigned
* persistence identifier.
*/
- DERBYREPO_0011("Metadata cannot have preassigned persistence id"),
+ DERBYREPO_0011("Entity cannot have preassigned persistence id"),
/**
* The system was unable to register various entities.
@@ -69,91 +69,91 @@ public enum DerbyRepoError implements ErrorCode {
DERBYREPO_0012("Unexpected update count when registering entity"),
/**
- * The system was unable to register metadata due to a failure to retrieve
+ * The system was unable to register entity due to a failure to retrieve
* the generated identifier.
*/
DERBYREPO_0013("Unable to retrieve generated identifier"),
/**
- * The system was unable to register connector metadata due to a server
+ * The system was unable to register connector due to a server
* error.
*/
- DERBYREPO_0014("Registration of connector metadata failed"),
+ DERBYREPO_0014("Registration of connector failed"),
/**
- * The system was not able to register connector metadata due to an unexpected
+ * The system was not able to register connector due to an unexpected
* update count.
*/
- DERBYREPO_0015("Unexpected update count on form registration"),
+ DERBYREPO_0015("Unexpected update count on config registration"),
/**
- * The system was unable to register connector metadata due to a failure to
- * retrieve the generated identifier for a form.
+ * The system was unable to register connector due to a failure to
+ * retrieve the generated identifier for a config.
*/
- DERBYREPO_0016("Unable to retrieve generated identifier for form"),
+ DERBYREPO_0016("Unable to retrieve generated identifier for config"),
/**
- * The system was unable to register connector metadata due to an unexpected
- * update count for form input registration.
+ * The system was unable to register connector due to an unexpected
+ * update count for config input registration.
*/
- DERBYREPO_0017("Unexpected update count for form input"),
+ DERBYREPO_0017("Unexpected update count for config input"),
/**
- * The system was unable to register connector metadata due to a failure to
- * retrieve the generated identifier for a form input.
+ * The system was unable to register connector due to a failure to
+ * retrieve the generated identifier for a config input.
*/
- DERBYREPO_0018("Unable to retrieve generated identifier for form input"),
+ DERBYREPO_0018("Unable to retrieve generated identifier for config input"),
- /** We cant create new connection in metastore **/
- DERBYREPO_0019("Unable to create new connection data"),
+ /** We cant create new link in repository **/
+ DERBYREPO_0019("Unable to create new link data"),
- /** We can't save values for input to metastore **/
- DERBYREPO_0020("Unable to save input values to metadata repository"),
+ /** We can't save values for input to repository **/
+ DERBYREPO_0020("Unable to save input values to the repository"),
- /** We can't update connection in metastore **/
- DERBYREPO_0021("Unable to update connection metadata in repository"),
+ /** We can't update link in repository **/
+ DERBYREPO_0021("Unable to update link in repository"),
- /** We can't delete connection in metastore **/
- DERBYREPO_0022("Unable to delete connection metadata in repository"),
+ /** We can't delete link in repository **/
+ DERBYREPO_0022("Unable to delete link in repository"),
- /** We can't restore connection metadata from metastore **/
- DERBYREPO_0023("Unable to load connection metadata from repository"),
+ /** We can't restore link from repository **/
+ DERBYREPO_0023("Unable to load link from repository"),
- /** We can't restore specific connection metadata from metastore **/
- DERBYREPO_0024("Unable to load specific connection metadata from repository"),
+ /** We can't restore specific link from repository **/
+ DERBYREPO_0024("Unable to load specific link from repository"),
- /** We're unable to check if given connection already exists */
- DERBYREPO_0025("Unable to check if given connection exists"),
+ /** We're unable to check if given link already exists */
+ DERBYREPO_0025("Unable to check if given link exists"),
- /** We cant create new job in metastore **/
+ /** We cant create new job in repository **/
DERBYREPO_0026("Unable to create new job data"),
- /** We can't update job in metastore **/
- DERBYREPO_0027("Unable to update job metadata in repository"),
+ /** We can't update job in repository **/
+ DERBYREPO_0027("Unable to update job in repository"),
- /** We can't delete job in metastore **/
- DERBYREPO_0028("Unable to delete job metadata in repository"),
+ /** We can't delete job in repository **/
+ DERBYREPO_0028("Unable to delete job in repository"),
/** We're unable to check if given job already exists */
DERBYREPO_0029("Unable to check if given job exists"),
- /** We can't restore specific job metadata from metastore **/
- DERBYREPO_0030("Unable to load specific job metadata from repository"),
+ /** We can't restore specific job from repository **/
+ DERBYREPO_0030("Unable to load specific job from repository"),
- /** We can't restore job metadata from metastore **/
- DERBYREPO_0031("Unable to load job metadata from repository"),
+ /** We can't restore job from repository **/
+ DERBYREPO_0031("Unable to load job from repository"),
- /** Can't verify if connection is referenced from somewhere **/
- DERBYREPO_0032("Unable to check if connection is in use"),
+ /** Can't verify if link is referenced from somewhere **/
+ DERBYREPO_0032("Unable to check if link is in use"),
/** We're unable to check if given submission already exists */
DERBYREPO_0033("Unable to check if given submission exists"),
- /** We cant create new submission in metastore **/
+ /** We cant create new submission in repository **/
DERBYREPO_0034("Unable to create new submission data"),
- /** We can't update submission in metastore **/
- DERBYREPO_0035("Unable to update submission metadata in repository"),
+ /** We can't update submission in repository **/
+ DERBYREPO_0035("Unable to update submission in the repository"),
/** Can't purge old submissions **/
DERBYREPO_0036("Unable to purge old submissions"),
@@ -172,15 +172,15 @@ public enum DerbyRepoError implements ErrorCode {
/** Can't detect version of the database structures **/
DERBYREPO_0041("Can't detect version of repository storage"),
- /** Can't enable/disable connection **/
- DERBYREPO_0042("Can't enable/disable connection"),
+ /** Can't enable/disable link **/
+ DERBYREPO_0042("Can't enable/disable link"),
/** Can't enable/disable job **/
DERBYREPO_0043("Can't enable/disable job"),
- DERBYREPO_0044("Update of framework failed"),
+ DERBYREPO_0044("Update of driver config failed"),
- DERBYREPO_0045("Can't retrieve all connector metadata"),
+ DERBYREPO_0045("Can't retrieve all connectors"),
;
private final String message;
[27/52] [abbrv] git commit: SQOOP-1556: Sqoop2: Add documentation
clarifying connectors vs. engines
Posted by ab...@apache.org.
SQOOP-1556: Sqoop2: Add documentation clarifying connectors vs. engines
(Gwen Shapira via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/196346d5
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/196346d5
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/196346d5
Branch: refs/heads/SQOOP-1367
Commit: 196346d5ccbb7cd6f4f2627cbd937728c61301d2
Parents: 35a060e
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Fri Sep 26 16:12:22 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:59:24 2014 -0700
----------------------------------------------------------------------
docs/src/site/sphinx/ConnectorDevelopment.rst | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/196346d5/docs/src/site/sphinx/ConnectorDevelopment.rst
----------------------------------------------------------------------
diff --git a/docs/src/site/sphinx/ConnectorDevelopment.rst b/docs/src/site/sphinx/ConnectorDevelopment.rst
index 5121382..ae4f721 100644
--- a/docs/src/site/sphinx/ConnectorDevelopment.rst
+++ b/docs/src/site/sphinx/ConnectorDevelopment.rst
@@ -31,6 +31,20 @@ Connector reads data from databases for import,
and write data to databases for export.
Interaction with Hadoop is taken cared by common modules of Sqoop 2 framework.
+When do we add a new connector?
+===============================
+You add a new connector when you need to extract data from a new data source, or load
+data to a new target.
+In addition to the connector API, Sqoop 2 also has an engine interface.
+At the moment the only engine is MapReduce,but we may support additional engines in the future.
+Since many parallel execution engines are capable of reading/writing data
+there may be a question of whether support for specific data stores should be done
+through a new connector or new engine.
+
+**Our guideline is:** Connectors should manage all data extract/load. Engines manage job
+life cycles. If you need to support a new data store and don't care how jobs run -
+you are looking to add a connector.
+
Connector Implementation
++++++++++++++++++++++++
[50/52] [abbrv] git commit: SQOOP-1477: Sqoop2: Make link and job
name unique identifier
Posted by ab...@apache.org.
SQOOP-1477: Sqoop2: Make link and job name unique identifier
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/4c964a97
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/4c964a97
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/4c964a97
Branch: refs/heads/SQOOP-1367
Commit: 4c964a97be674ead165844274d48fea9ff69fe65
Parents: c78b75a
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Tue Oct 7 11:38:09 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 18:22:51 2014 -0700
----------------------------------------------------------------------
.../sqoop/repository/RepositoryError.java | 2 +-
.../derby/DerbyRepositoryHandler.java | 18 +++--
.../repository/derby/DerbySchemaConstants.java | 10 +++
.../repository/derby/DerbySchemaQuery.java | 44 +++++-------
.../sqoop/repository/derby/DerbyTestCase.java | 19 ++++--
.../sqoop/repository/derby/TestInternals.java | 71 ++++++++++++++++++++
.../sqoop/repository/derby/TestJobHandling.java | 23 +++++--
.../repository/derby/TestLinkHandling.java | 12 ++++
8 files changed, 156 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/4c964a97/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
index 54e37d9..0c93b50 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
@@ -122,7 +122,7 @@ public enum RepositoryError implements ErrorCode {
JDBCREPO_0026("Upgrade required but not allowed"),
/** Invalid links or jobs when upgrading connector **/
- JDBCREPO_0027("Invalid links or jobs when upgrading connector");
+ JDBCREPO_0027("Invalid links or jobs when upgrading connector")
;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/4c964a97/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index 39702ca..73d8387 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -415,6 +415,10 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
// Wait to remove SQB_TYPE (IMPORT/EXPORT) until we update data.
// Data updates depend on knowledge of the type of job.
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE, conn);
+
+ // Add unique constraints on job and links.
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_UNIQUE_CONSTRAINT_NAME, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_UNIQUE_CONSTRAINT_NAME, conn);
}
ResultSet rs = null;
@@ -849,7 +853,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
PreparedStatement stmt = null;
ResultSet rs = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_LINK_CHECK);
+ stmt = conn.prepareStatement(STMT_SELECT_LINK_CHECK_BY_ID);
stmt.setLong(1, id);
rs = stmt.executeQuery();
@@ -1164,9 +1168,9 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
job.getJobConfig(Direction.TO).getConfigs(),
conn);
createInputValues(STMT_INSERT_JOB_INPUT,
- job.getPersistenceId(),
- job.getDriverConfig().getConfigs(),
- conn);
+ job.getPersistenceId(),
+ job.getDriverConfig().getConfigs(),
+ conn);
} catch (SQLException ex) {
logException(ex, job);
@@ -1184,7 +1188,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
PreparedStatement stmt = null;
ResultSet rs = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_JOB_CHECK);
+ stmt = conn.prepareStatement(STMT_SELECT_JOB_CHECK_BY_ID);
stmt.setLong(1, id);
rs = stmt.executeQuery();
@@ -1279,7 +1283,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
public MJob findJob(long id, Connection conn) {
PreparedStatement stmt = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_JOB_SINGLE);
+ stmt = conn.prepareStatement(STMT_SELECT_JOB_SINGLE_BY_ID);
stmt.setLong(1, id);
List<MJob> jobs = loadJobs(stmt, conn);
@@ -1307,7 +1311,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
public List<MJob> findJobs(Connection conn) {
PreparedStatement stmt = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_JOB_ALL);
+ stmt = conn.prepareStatement(STMT_SELECT_JOB);
return loadJobs(stmt, conn);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/4c964a97/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
index fc3ec18..ad749ed 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
@@ -133,6 +133,10 @@ public final class DerbySchemaConstants {
public static final String CONSTRAINT_SQ_LNK_SQC = SCHEMA_PREFIX + CONSTRAINT_SQ_LNK_SQC_NAME;
+ public static final String CONSTRAINT_SQ_LNK_NAME_UNIQUE_NAME = CONSTRAINT_PREFIX + "SQ_LNK_NAME_UNIQUE";
+
+ public static final String CONSTRAINT_SQ_LNK_NAME_UNIQUE = SCHEMA_PREFIX + CONSTRAINT_SQ_LNK_NAME_UNIQUE_NAME;
+
// SQ_JOB
public static final String TABLE_SQ_JOB_NAME = "SQ_JOB";
@@ -174,6 +178,12 @@ public final class DerbySchemaConstants {
public static final String CONSTRAINT_SQB_SQ_LNK_TO = SCHEMA_PREFIX + CONSTRAINT_SQB_SQ_LNK_TO_NAME;
+ public static final String CONSTRAINT_SQB_NAME_UNIQUE_NAME = CONSTRAINT_PREFIX + "SQB_NAME_UNIQUE";
+
+ public static final String CONSTRAINT_SQB_NAME_UNIQUE = SCHEMA_PREFIX + CONSTRAINT_SQB_NAME_UNIQUE_NAME;
+
+ // SQ_CONNECTION_INPUT
+
public static final String TABLE_SQ_LINK_INPUT_NAME =
"SQ_LINK_INPUT";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/4c964a97/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
index 951d9b4..478afe2 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
@@ -694,7 +694,7 @@ public final class DerbySchemaQuery {
+ " WHERE " + COLUMN_SQ_LNK_CONNECTOR + " = ?";
// DML: Check if given link exists
- public static final String STMT_SELECT_LINK_CHECK =
+ public static final String STMT_SELECT_LINK_CHECK_BY_ID =
"SELECT count(*) FROM " + TABLE_SQ_LINK
+ " WHERE " + COLUMN_SQ_LNK_ID + " = ?";
@@ -743,7 +743,7 @@ public final class DerbySchemaQuery {
+ " WHERE " + COLUMN_SQB_ID + " = ?";
// DML: Check if given job exists
- public static final String STMT_SELECT_JOB_CHECK =
+ public static final String STMT_SELECT_JOB_CHECK_BY_ID =
"SELECT count(*) FROM " + TABLE_SQ_JOB
+ " WHERE " + COLUMN_SQB_ID + " = ?";
@@ -756,29 +756,8 @@ public final class DerbySchemaQuery {
+ " ON " + COLUMN_SQB_FROM_LINK + " = " + COLUMN_SQ_LNK_ID
+ " WHERE " + COLUMN_SQ_LNK_ID + " = ? ";
- // DML: Select one specific job
- public static final String STMT_SELECT_JOB_SINGLE =
- "SELECT "
- + "FROM_CONNECTOR." + COLUMN_SQ_LNK_CONNECTOR + ", "
- + "TO_CONNECTOR." + COLUMN_SQ_LNK_CONNECTOR + ", "
- + "job." + COLUMN_SQB_ID + ", "
- + "job." + COLUMN_SQB_NAME + ", "
- + "job." + COLUMN_SQB_FROM_LINK + ", "
- + "job." + COLUMN_SQB_TO_LINK + ", "
- + "job." + COLUMN_SQB_ENABLED + ", "
- + "job." + COLUMN_SQB_CREATION_USER + ", "
- + "job." + COLUMN_SQB_CREATION_DATE + ", "
- + "job." + COLUMN_SQB_UPDATE_USER + ", "
- + "job." + COLUMN_SQB_UPDATE_DATE
- + " FROM " + TABLE_SQ_JOB + " job"
- + " LEFT JOIN " + TABLE_SQ_LINK
- + " FROM_CONNECTOR ON " + COLUMN_SQB_FROM_LINK + " = FROM_CONNECTOR." + COLUMN_SQ_LNK_ID
- + " LEFT JOIN " + TABLE_SQ_LINK
- + " TO_CONNECTOR ON " + COLUMN_SQB_TO_LINK + " = TO_CONNECTOR." + COLUMN_SQ_LNK_ID
- + " WHERE " + COLUMN_SQB_ID + " = ?";
-
// DML: Select all jobs
- public static final String STMT_SELECT_JOB_ALL =
+ public static final String STMT_SELECT_JOB =
"SELECT "
+ "FROM_LINK." + COLUMN_SQ_LNK_CONNECTOR + ", "
+ "TO_LINK." + COLUMN_SQ_LNK_CONNECTOR + ", "
@@ -797,9 +776,14 @@ public final class DerbySchemaQuery {
+ " LEFT JOIN " + TABLE_SQ_LINK + " TO_LINK"
+ " ON " + COLUMN_SQB_TO_LINK + " = TO_LINK." + COLUMN_SQ_LNK_ID;
+ // DML: Select one specific job
+ public static final String STMT_SELECT_JOB_SINGLE_BY_ID =
+ STMT_SELECT_JOB + " WHERE " + COLUMN_SQB_ID + " = ?";
+
// DML: Select all jobs for a Connector
- public static final String STMT_SELECT_ALL_JOBS_FOR_CONNECTOR = STMT_SELECT_JOB_ALL
- + " WHERE FROM_LINK." + COLUMN_SQ_LNK_CONNECTOR + " = ? OR TO_LINK." + COLUMN_SQ_LNK_CONNECTOR + " = ?";
+ public static final String STMT_SELECT_ALL_JOBS_FOR_CONNECTOR = STMT_SELECT_JOB
+ + " WHERE FROM_LINK." + COLUMN_SQ_LNK_CONNECTOR + " = ? OR TO_LINK."
+ + COLUMN_SQ_LNK_CONNECTOR + " = ?";
// DML: Insert new submission
public static final String STMT_INSERT_SUBMISSION =
@@ -1066,6 +1050,14 @@ public final class DerbySchemaQuery {
public static final String QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE =
"ALTER TABLE " + TABLE_SQ_JOB + " DROP COLUMN " + COLUMN_SQB_TYPE;
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_UNIQUE_CONSTRAINT_NAME =
+ "ALTER TABLE " + TABLE_SQ_JOB + " ADD CONSTRAINT "
+ + CONSTRAINT_SQB_NAME_UNIQUE + " UNIQUE (" + COLUMN_SQB_NAME + ")";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_ADD_UNIQUE_CONSTRAINT_NAME =
+ "ALTER TABLE " + TABLE_SQ_LINK + " ADD CONSTRAINT "
+ + CONSTRAINT_SQ_LNK_NAME_UNIQUE + " UNIQUE (" + COLUMN_SQ_LNK_NAME + ")";
+
private DerbySchemaQuery() {
// Disable explicit object creation
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/4c964a97/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index bf72626..2da084f 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -37,12 +37,14 @@ import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_T
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_UPDATE_USER;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_FROM;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_TO;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_UNIQUE_CONSTRAINT_NAME;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQ_LNK;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_LINK_TO_SQB_FROM_LINK;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_CREATION_USER;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_ENABLED;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_UPDATE_USER;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_LINK_ADD_UNIQUE_CONSTRAINT_NAME;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_CREATION_USER;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_UPDATE_USER;
import static org.junit.Assert.assertEquals;
@@ -152,11 +154,13 @@ abstract public class DerbyTestCase {
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_FROM);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_TO);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_UNIQUE_CONSTRAINT_NAME);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_UNIQUE_CONSTRAINT_NAME);
}
runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) VALUES('version', '" + version + "')");
runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) " +
- "VALUES('" + DerbyRepoConstants.SYSKEY_DRIVER_VERSION + "', '1')");
+ "VALUES('" + DerbyRepoConstants.SYSKEY_DRIVER_VERSION + "', '1')");
}
protected void createSchema() throws Exception {
@@ -169,13 +173,19 @@ abstract public class DerbyTestCase {
* @param query Query to execute
* @throws Exception
*/
- protected void runQuery(String query, String... args) throws Exception {
+ protected void runQuery(String query, Object... args) throws Exception {
PreparedStatement stmt = null;
try {
stmt = getDerbyDatabaseConnection().prepareStatement(query);
for (int i = 0; i < args.length; ++i) {
- stmt.setString(i + 1, args[i]);
+ if (args[i] instanceof String) {
+ stmt.setString(i + 1, (String)args[i]);
+ } else if (args[i] instanceof Long) {
+ stmt.setLong(i + 1, (Long)args[i]);
+ } else {
+ stmt.setString(i + 1, args[i].toString());
+ }
}
stmt.execute();
@@ -436,6 +446,7 @@ abstract public class DerbyTestCase {
* @throws Exception
*/
public void loadJobs(int version) throws Exception {
+ int index = 0;
switch (version) {
case 2:
for(String type : new String[] {"IMPORT", "EXPORT"}) {
@@ -468,7 +479,7 @@ abstract public class DerbyTestCase {
case 4:
for (String name : new String[]{"JA", "JB", "JC", "JD"}) {
runQuery("INSERT INTO SQOOP.SQ_JOB(SQB_NAME, SQB_FROM_LINK, SQB_TO_LINK)"
- + " VALUES('" + name + "', 1, 1)");
+ + " VALUES('" + name + index + "', 1, 1)");
}
// Odd IDs inputs have values
http://git-wip-us.apache.org/repos/asf/sqoop/blob/4c964a97/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
index 1292168..0eb9df4 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
@@ -17,10 +17,18 @@
*/
package org.apache.sqoop.repository.derby;
+import org.apache.sqoop.common.SqoopException;
import org.junit.Before;
import org.junit.Test;
import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -53,6 +61,17 @@ public class TestInternals extends DerbyTestCase {
assertTrue(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
}
+ @Test(expected=SqoopException.class)
+ public void testUpgradeVersion2ToVersion4DuplicateFailure() throws Exception {
+ createSchema(2);
+ assertFalse(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
+ loadConnectorAndDriverConfig(2);
+ loadLinks(2);
+ loadJobs(2);
+ handler.createOrUpdateInternals(getDerbyDatabaseConnection());
+ assertTrue(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
+ }
+
@Test
public void testUpgradeVersion2ToVersion4() throws Exception {
createSchema(2);
@@ -60,10 +79,62 @@ public class TestInternals extends DerbyTestCase {
loadConnectorAndDriverConfig(2);
loadLinks(2);
loadJobs(2);
+ renameLinks();
+ renameJobs();
handler.createOrUpdateInternals(getDerbyDatabaseConnection());
assertTrue(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
}
+ private Map<String, List<Long>> getNameIdMap(PreparedStatement ps) throws SQLException {
+ Map<String, List<Long>> nameIdMap = new TreeMap<String, List<Long>>();
+ ResultSet rs = null;
+
+ try {
+ rs = ps.executeQuery();
+ while(rs.next()) {
+ if (!nameIdMap.containsKey(rs.getString(1))) {
+ nameIdMap.put(rs.getString(1), new LinkedList<Long>());
+ }
+ nameIdMap.get(rs.getString(1)).add(rs.getLong(2));
+ }
+ } finally {
+ if(rs != null) {
+ rs.close();
+ }
+ if(ps != null) {
+ ps.close();
+ }
+ }
+
+ return nameIdMap;
+ }
+
+ private void renameLinks() throws Exception {
+ Map<String, List<Long>> nameIdMap =
+ getNameIdMap(getDerbyDatabaseConnection().prepareStatement("SELECT SQ_LNK_NAME, SQ_LNK_ID FROM SQOOP.SQ_LINK"));;
+
+ for (String name : nameIdMap.keySet()) {
+ if (nameIdMap.get(name).size() > 1) {
+ for (Long id : nameIdMap.get(name)) {
+ runQuery("UPDATE SQOOP.SQ_LINK SET SQ_LNK_NAME=? WHERE SQ_LNK_ID=?", name + "-" + id, id);
+ }
+ }
+ }
+ }
+
+ private void renameJobs() throws Exception {
+ Map<String, List<Long>> nameIdMap =
+ getNameIdMap(getDerbyDatabaseConnection().prepareStatement("SELECT SQB_NAME, SQB_ID FROM SQOOP.SQ_JOB"));;
+
+ for (String name : nameIdMap.keySet()) {
+ if (nameIdMap.get(name).size() > 1) {
+ for (Long id : nameIdMap.get(name)) {
+ runQuery("UPDATE SQOOP.SQ_JOB SET SQB_NAME=? WHERE SQB_ID=?", name + "-" + id, id);
+ }
+ }
+ }
+ }
+
private class TestDerbyRepositoryHandler extends DerbyRepositoryHandler {
protected long registerHdfsConnector(Connection conn) {
try {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/4c964a97/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
index 595b1c8..0752923 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
@@ -76,7 +76,7 @@ public class TestJobHandling extends DerbyTestCase {
MJob firstJob = handler.findJob(1, derbyConnection);
assertNotNull(firstJob);
assertEquals(1, firstJob.getPersistenceId());
- assertEquals("JA", firstJob.getName());
+ assertEquals("JA0", firstJob.getName());
List<MConfig> configs;
@@ -114,13 +114,13 @@ public class TestJobHandling extends DerbyTestCase {
list = handler.findJobs(derbyConnection);
assertEquals(4, list.size());
- assertEquals("JA", list.get(0).getName());
+ assertEquals("JA0", list.get(0).getName());
- assertEquals("JB", list.get(1).getName());
+ assertEquals("JB0", list.get(1).getName());
- assertEquals("JC", list.get(2).getName());
+ assertEquals("JC0", list.get(2).getName());
- assertEquals("JD", list.get(3).getName());
+ assertEquals("JD0", list.get(3).getName());
}
@Test
@@ -193,6 +193,19 @@ public class TestJobHandling extends DerbyTestCase {
assertCountForTable("SQOOP.SQ_JOB_INPUT", 12);
}
+ @Test(expected=SqoopException.class)
+ public void testCreateDuplicateJob() throws Exception {
+ // Duplicate jobs
+ MJob job = getJob();
+ fillJob(job);
+ job.setName("test");
+ handler.createJob(job, getDerbyDatabaseConnection());
+ assertEquals(1, job.getPersistenceId());
+
+ job.setPersistenceId(MJob.PERSISTANCE_ID_DEFAULT);
+ handler.createJob(job, getDerbyDatabaseConnection());
+ }
+
@Test
public void testUpdateJob() throws Exception {
loadJobs();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/4c964a97/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
index 38e632a..bbfe5bb 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
@@ -147,6 +147,18 @@ public class TestLinkHandling extends DerbyTestCase {
assertCountForTable("SQOOP.SQ_LINK_INPUT", 4);
}
+ @Test(expected=SqoopException.class)
+ public void testCreateDuplicateLink() throws Exception {
+ MLink link = getLink();
+ fillLink(link);
+ link.setName("test");
+ handler.createLink(link, getDerbyDatabaseConnection());
+ assertEquals(1, link.getPersistenceId());
+
+ link.setPersistenceId(MLink.PERSISTANCE_ID_DEFAULT);
+ handler.createLink(link, getDerbyDatabaseConnection());
+ }
+
@Test
public void testInUseLink() throws Exception {
loadLinks();
[21/52] [abbrv] SQOOP-1496: Sqoop2: Revisit/Refactor the
SubmissionEngine/ExecutionEngine APIs
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
index b05954b..ef7ff4e 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
@@ -17,21 +17,20 @@
*/
package org.apache.sqoop.execution.mapreduce;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.io.NullWritable;
import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.framework.ExecutionEngine;
-import org.apache.sqoop.framework.SubmissionRequest;
-import org.apache.sqoop.framework.configuration.JobConfiguration;
+import org.apache.sqoop.framework.JobRequest;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.etl.From;
import org.apache.sqoop.job.etl.To;
-import org.apache.sqoop.job.io.Data;
import org.apache.sqoop.job.io.SqoopWritable;
import org.apache.sqoop.job.mr.SqoopInputFormat;
import org.apache.sqoop.job.mr.SqoopMapper;
import org.apache.sqoop.job.mr.SqoopNullOutputFormat;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
/**
*
*/
@@ -41,44 +40,40 @@ public class MapreduceExecutionEngine extends ExecutionEngine {
* {@inheritDoc}
*/
@Override
- public SubmissionRequest createSubmissionRequest() {
- return new MRSubmissionRequest();
+ public JobRequest createJobRequest() {
+ return new MRJobRequest();
}
- public void prepareSubmission(SubmissionRequest gRequest) {
- MRSubmissionRequest request = (MRSubmissionRequest)gRequest;
+ public void prepareJob(JobRequest jobRequest) {
+ MRJobRequest mrJobRequest = (MRJobRequest)jobRequest;
// Add jar dependencies
- addDependencies(request);
+ addDependencies(mrJobRequest);
// Configure map-reduce classes for import
- request.setInputFormatClass(SqoopInputFormat.class);
+ mrJobRequest.setInputFormatClass(SqoopInputFormat.class);
- request.setMapperClass(SqoopMapper.class);
- request.setMapOutputKeyClass(SqoopWritable.class);
- request.setMapOutputValueClass(NullWritable.class);
+ mrJobRequest.setMapperClass(SqoopMapper.class);
+ mrJobRequest.setMapOutputKeyClass(SqoopWritable.class);
+ mrJobRequest.setMapOutputValueClass(NullWritable.class);
- request.setOutputFormatClass(SqoopNullOutputFormat.class);
- request.setOutputKeyClass(SqoopWritable.class);
- request.setOutputValueClass(NullWritable.class);
+ mrJobRequest.setOutputFormatClass(SqoopNullOutputFormat.class);
+ mrJobRequest.setOutputKeyClass(SqoopWritable.class);
+ mrJobRequest.setOutputValueClass(NullWritable.class);
- // Set up framework context
- From from = (From)request.getFromCallback();
- To to = (To)request.getToCallback();
- MutableMapContext context = request.getFrameworkContext();
+ From from = (From) mrJobRequest.getFrom();
+ To to = (To) mrJobRequest.getTo();
+
+ MutableMapContext context = mrJobRequest.getFrameworkContext();
context.setString(JobConstants.JOB_ETL_PARTITIONER, from.getPartitioner().getName());
context.setString(JobConstants.JOB_ETL_EXTRACTOR, from.getExtractor().getName());
context.setString(JobConstants.JOB_ETL_LOADER, to.getLoader().getName());
context.setString(JobConstants.JOB_ETL_DESTROYER, from.getDestroyer().getName());
context.setString(JobConstants.INTERMEDIATE_DATA_FORMAT,
- request.getIntermediateDataFormat().getName());
-
- if(request.getExtractors() != null) {
- context.setInteger(JobConstants.JOB_ETL_EXTRACTOR_NUM, request.getExtractors());
- }
+ mrJobRequest.getIntermediateDataFormat().getName());
- if(request.getExtractors() != null) {
- context.setInteger(JobConstants.JOB_ETL_EXTRACTOR_NUM, request.getExtractors());
+ if(mrJobRequest.getExtractors() != null) {
+ context.setInteger(JobConstants.JOB_ETL_EXTRACTOR_NUM, mrJobRequest.getExtractors());
}
}
@@ -91,7 +86,7 @@ public class MapreduceExecutionEngine extends ExecutionEngine {
*
* @param request Active request object.
*/
- protected void addDependencies(MRSubmissionRequest request) {
+ protected void addDependencies(MRJobRequest request) {
// Guava
request.addJarForClass(ThreadFactoryBuilder.class);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
index 92414d8..2ed06a8 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
@@ -90,9 +90,6 @@ public final class ConfigurationUtils {
private static final Text SCHEMA_TO_CONNECTOR_KEY = new Text(SCHEMA_TO_CONNECTOR);
- private static final String SCHEMA_HIO = JobConstants.PREFIX_JOB_CONFIG + "schema.hio";
-
- private static final Text SCHEMA_HIO_KEY = new Text(SCHEMA_HIO);
/**
* Persist Connector configuration object for connection.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ProgressRunnable.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ProgressRunnable.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ProgressRunnable.java
index b73b151..4c2e206 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ProgressRunnable.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ProgressRunnable.java
@@ -31,9 +31,9 @@ public class ProgressRunnable implements Runnable {
/**
* Context class that we should use for reporting progress.
*/
- private final TaskInputOutputContext context;
+ private final TaskInputOutputContext<?,?,?,?> context;
- public ProgressRunnable(final TaskInputOutputContext ctxt) {
+ public ProgressRunnable(final TaskInputOutputContext<?,?,?,?> ctxt) {
this.context = ctxt;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
index 59431f4..e3af6e1 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
@@ -32,8 +32,7 @@ import org.apache.sqoop.utils.ClassUtils;
*/
public class SqoopDestroyerExecutor {
- public static final Logger LOG =
- Logger.getLogger(SqoopDestroyerExecutor.class);
+ public static final Logger LOG = Logger.getLogger(SqoopDestroyerExecutor.class);
/**
* Execute destroyer.
@@ -56,10 +55,8 @@ public class SqoopDestroyerExecutor {
Object fromConfigConnection = ConfigurationUtils.getConnectorConnectionConfig(Direction.FROM, configuration);
Object fromConfigJob = ConfigurationUtils.getConnectorJobConfig(Direction.FROM, configuration);
- // Propagate connector schema in every case for now
- // TODO: Change to coditional choosing between Connector schemas.
+ // TODO(Abe/Gwen): Change to conditional choosing between schemas.
Schema schema = ConfigurationUtils.getConnectorSchema(Direction.FROM, configuration);
-
DestroyerContext destroyerContext = new DestroyerContext(subContext, success, schema);
LOG.info("Executing destroyer class " + destroyer.getClass());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java
index bbf7342..3e2b1c5 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java
@@ -64,9 +64,7 @@ public class SqoopFileOutputFormat
conf.set(JobConstants.JOB_MR_OUTPUT_CODEC, codecname);
}
- SqoopOutputFormatLoadExecutor executor =
- new SqoopOutputFormatLoadExecutor(context);
- return executor.getRecordWriter();
+ return new SqoopOutputFormatLoadExecutor(context).getRecordWriter();
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
index 3065680..6680f60 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
@@ -54,7 +54,7 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
* Service for reporting progress to mapreduce.
*/
private final ScheduledExecutorService progressService = Executors.newSingleThreadScheduledExecutor();
- private IntermediateDataFormat data = null;
+ private IntermediateDataFormat<String> dataFormat = null;
private SqoopWritable dataOut = null;
@Override
@@ -64,44 +64,36 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
String extractorName = conf.get(JobConstants.JOB_ETL_EXTRACTOR);
Extractor extractor = (Extractor) ClassUtils.instantiate(extractorName);
- // Propagate connector schema in every case for now
- // TODO: Change to coditional choosing between Connector schemas.
-
+ // TODO(Abe/Gwen): Change to conditional choosing between Connector schemas.
Schema schema = ConfigurationUtils.getConnectorSchema(Direction.FROM, conf);
- if (schema==null) {
+ if (schema == null) {
schema = ConfigurationUtils.getConnectorSchema(Direction.TO, conf);
}
- if (schema==null) {
+ if (schema == null) {
LOG.info("setting an empty schema");
}
-
- String intermediateDataFormatName = conf.get(JobConstants
- .INTERMEDIATE_DATA_FORMAT);
- data = (IntermediateDataFormat) ClassUtils.instantiate(intermediateDataFormatName);
- data.setSchema(schema);
+ String intermediateDataFormatName = conf.get(JobConstants.INTERMEDIATE_DATA_FORMAT);
+ dataFormat = (IntermediateDataFormat<String>) ClassUtils
+ .instantiate(intermediateDataFormatName);
+ dataFormat.setSchema(schema);
dataOut = new SqoopWritable();
- // Objects that should be pass to the Executor execution
- PrefixContext subContext = null;
- Object configConnection = null;
- Object configJob = null;
-
- // Get configs for extractor
- subContext = new PrefixContext(conf, JobConstants.PREFIX_CONNECTOR_FROM_CONTEXT);
- configConnection = ConfigurationUtils.getConnectorConnectionConfig(Direction.FROM, conf);
- configJob = ConfigurationUtils.getConnectorJobConfig(Direction.FROM, conf);
+ // Objects that should be passed to the Executor execution
+ PrefixContext subContext = new PrefixContext(conf, JobConstants.PREFIX_CONNECTOR_FROM_CONTEXT);
+ Object fromConfig = ConfigurationUtils.getConnectorConnectionConfig(Direction.FROM, conf);
+ Object fromJob = ConfigurationUtils.getConnectorJobConfig(Direction.FROM, conf);
SqoopSplit split = context.getCurrentKey();
- ExtractorContext extractorContext = new ExtractorContext(subContext, new MapDataWriter(context), schema);
+ ExtractorContext extractorContext = new ExtractorContext(subContext, new SqoopMapDataWriter(context), schema);
try {
LOG.info("Starting progress service");
progressService.scheduleAtFixedRate(new ProgressRunnable(context), 0, 2, TimeUnit.MINUTES);
LOG.info("Running extractor class " + extractorName);
- extractor.extract(extractorContext, configConnection, configJob, split.getPartition());
+ extractor.extract(extractorContext, fromConfig, fromJob, split.getPartition());
LOG.info("Extractor has finished");
context.getCounter(SqoopCounters.ROWS_READ)
.increment(extractor.getRowsRead());
@@ -117,37 +109,37 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
}
}
- private class MapDataWriter extends DataWriter {
+ private class SqoopMapDataWriter extends DataWriter {
private Context context;
- public MapDataWriter(Context context) {
+ public SqoopMapDataWriter(Context context) {
this.context = context;
}
@Override
public void writeArrayRecord(Object[] array) {
- data.setObjectData(array);
+ dataFormat.setObjectData(array);
writeContent();
}
@Override
public void writeStringRecord(String text) {
- data.setTextData(text);
+ dataFormat.setTextData(text);
writeContent();
}
@Override
public void writeRecord(Object obj) {
- data.setData(obj.toString());
+ dataFormat.setData(obj.toString());
writeContent();
}
private void writeContent() {
try {
if (LOG.isDebugEnabled()) {
- LOG.debug("Extracted data: " + data.getTextData());
+ LOG.debug("Extracted data: " + dataFormat.getTextData());
}
- dataOut.setString(data.getTextData());
+ dataOut.setString(dataFormat.getTextData());
context.write(dataOut, NullWritable.get());
} catch (Exception e) {
throw new SqoopException(MapreduceExecutionError.MAPRED_EXEC_0013, e);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
index e457cff..2996275 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
@@ -51,9 +51,9 @@ public class SqoopOutputFormatLoadExecutor {
private volatile boolean readerFinished = false;
private volatile boolean writerFinished = false;
- private volatile IntermediateDataFormat data;
+ private volatile IntermediateDataFormat<String> dataFormat;
private JobContext context;
- private SqoopRecordWriter producer;
+ private SqoopRecordWriter writer;
private Future<?> consumerFuture;
private Semaphore filled = new Semaphore(0, true);
private Semaphore free = new Semaphore(1, true);
@@ -63,14 +63,14 @@ public class SqoopOutputFormatLoadExecutor {
SqoopOutputFormatLoadExecutor(boolean isTest, String loaderName){
this.isTest = isTest;
this.loaderName = loaderName;
- data = new CSVIntermediateDataFormat();
- producer = new SqoopRecordWriter();
+ dataFormat = new CSVIntermediateDataFormat();
+ writer = new SqoopRecordWriter();
}
public SqoopOutputFormatLoadExecutor(JobContext jobctx) {
context = jobctx;
- producer = new SqoopRecordWriter();
- data = (IntermediateDataFormat) ClassUtils.instantiate(context
+ writer = new SqoopRecordWriter();
+ dataFormat = (IntermediateDataFormat<String>) ClassUtils.instantiate(context
.getConfiguration().get(JobConstants.INTERMEDIATE_DATA_FORMAT));
Schema schema = ConfigurationUtils.getConnectorSchema(Direction.FROM, context.getConfiguration());
@@ -78,14 +78,14 @@ public class SqoopOutputFormatLoadExecutor {
schema = ConfigurationUtils.getConnectorSchema(Direction.TO, context.getConfiguration());
}
- data.setSchema(schema);
+ dataFormat.setSchema(schema);
}
public RecordWriter<SqoopWritable, NullWritable> getRecordWriter() {
consumerFuture = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat
("OutputFormatLoader-consumer").build()).submit(
new ConsumerThread());
- return producer;
+ return writer;
}
/*
@@ -98,7 +98,7 @@ public class SqoopOutputFormatLoadExecutor {
public void write(SqoopWritable key, NullWritable value) throws InterruptedException {
free.acquire();
checkIfConsumerThrew();
- data.setTextData(key.getString());
+ dataFormat.setTextData(key.getString());
filled.release();
}
@@ -144,7 +144,7 @@ public class SqoopOutputFormatLoadExecutor {
}
}
- private class OutputFormatDataReader extends DataReader {
+ private class SqoopOutputFormatDataReader extends DataReader {
@Override
public Object[] readArrayRecord() throws InterruptedException {
@@ -154,7 +154,7 @@ public class SqoopOutputFormatLoadExecutor {
return null;
}
try {
- return data.getObjectData();
+ return dataFormat.getObjectData();
} finally {
releaseSema();
}
@@ -168,7 +168,7 @@ public class SqoopOutputFormatLoadExecutor {
return null;
}
try {
- return data.getTextData();
+ return dataFormat.getTextData();
} finally {
releaseSema();
}
@@ -181,7 +181,7 @@ public class SqoopOutputFormatLoadExecutor {
return null;
}
try {
- return data.getData();
+ return dataFormat.getData();
} catch (Throwable t) {
readerFinished = true;
LOG.error("Caught exception e while getting content ", t);
@@ -215,7 +215,7 @@ public class SqoopOutputFormatLoadExecutor {
public void run() {
LOG.info("SqoopOutputFormatLoadExecutor consumer thread is starting");
try {
- DataReader reader = new OutputFormatDataReader();
+ DataReader reader = new SqoopOutputFormatDataReader();
Configuration conf = null;
if (!isTest) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
index 6dbd870..51f778b 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
@@ -68,14 +68,14 @@ public final class SubmissionDisplayer {
}
}
- if(isVerbose() && submission.getConnectorSchema() != null) {
+ if(isVerbose() && submission.getFromSchema() != null) {
print(resourceString(Constants.RES_CONNECTOR_SCHEMA)+": ");
- println(submission.getConnectorSchema());
+ println(submission.getFromSchema());
}
- if(isVerbose() && submission.getHioSchema() != null) {
+ if(isVerbose() && submission.getToSchema() != null) {
print(resourceString(Constants.RES_HIO_SCHEMA)+": ");
- println(submission.getHioSchema());
+ println(submission.getToSchema());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/spi/src/main/java/org/apache/sqoop/job/etl/CallbackBase.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/CallbackBase.java b/spi/src/main/java/org/apache/sqoop/job/etl/CallbackBase.java
deleted file mode 100644
index 59a9457..0000000
--- a/spi/src/main/java/org/apache/sqoop/job/etl/CallbackBase.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.job.etl;
-
-/**
- * Set of default callbacks that must be implement by each job type.
- */
-public abstract class CallbackBase {
-
- private Class<? extends Initializer> initializer;
- private Class<? extends Destroyer> destroyer;
-
- public CallbackBase(
- Class<? extends Initializer> initializer,
- Class<? extends Destroyer> destroyer
- ) {
- this.initializer = initializer;
- this.destroyer = destroyer;
- }
-
- public Class<? extends Destroyer> getDestroyer() {
- return destroyer;
- }
-
- public Class<? extends Initializer> getInitializer() {
- return initializer;
- }
-
- @Override
- public String toString() {
- return "initializer=" + initializer.getName() +
- ", destroyer=" + destroyer.getName();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/spi/src/main/java/org/apache/sqoop/job/etl/From.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/From.java b/spi/src/main/java/org/apache/sqoop/job/etl/From.java
index 9b8d76f..80f4f29 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/From.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/From.java
@@ -26,7 +26,7 @@ package org.apache.sqoop.job.etl;
* -> (framework-defined steps)
* -> Destroyer
*/
-public class From extends CallbackBase {
+public class From extends Transferable {
private Class<? extends Partitioner> partitioner;
private Class<? extends Extractor> extractor;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/spi/src/main/java/org/apache/sqoop/job/etl/To.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/To.java b/spi/src/main/java/org/apache/sqoop/job/etl/To.java
index a791945..b8717ae 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/To.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/To.java
@@ -25,7 +25,7 @@ package org.apache.sqoop.job.etl;
* -> Loader
* -> Destroyer
*/
-public class To extends CallbackBase {
+public class To extends Transferable {
private Class<? extends Loader> loader;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/spi/src/main/java/org/apache/sqoop/job/etl/Transferable.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Transferable.java b/spi/src/main/java/org/apache/sqoop/job/etl/Transferable.java
new file mode 100644
index 0000000..dfe1d5e
--- /dev/null
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Transferable.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.job.etl;
+
+/**
+ * This entity encapsulates the workflow for data transfer via the
+ * {@link SqoopConnector}.It basically acts as an adapter between the data-source
+ * imported from or exported to.
+ */
+public abstract class Transferable {
+
+ private Class<? extends Initializer> initializer;
+ private Class<? extends Destroyer> destroyer;
+
+ public Transferable(
+ Class<? extends Initializer> initializer,
+ Class<? extends Destroyer> destroyer
+ ) {
+ this.initializer = initializer;
+ this.destroyer = destroyer;
+ }
+
+ public Class<? extends Destroyer> getDestroyer() {
+ return destroyer;
+ }
+
+ public Class<? extends Initializer> getInitializer() {
+ return initializer;
+ }
+
+ @Override
+ public String toString() {
+ return "initializer=" + initializer.getName() +
+ ", destroyer=" + destroyer.getName();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/spi/src/main/java/org/apache/sqoop/validation/Validator.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/validation/Validator.java b/spi/src/main/java/org/apache/sqoop/validation/Validator.java
index 9b791f8..f31adb5 100644
--- a/spi/src/main/java/org/apache/sqoop/validation/Validator.java
+++ b/spi/src/main/java/org/apache/sqoop/validation/Validator.java
@@ -17,7 +17,6 @@
*/
package org.apache.sqoop.validation;
-import org.apache.sqoop.model.MJob;
/**
* Connection and job metadata validator.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
index bfa6958..93741e6 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
@@ -29,9 +29,9 @@ import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.MapContext;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.execution.mapreduce.MRSubmissionRequest;
+import org.apache.sqoop.execution.mapreduce.MRJobRequest;
import org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine;
-import org.apache.sqoop.framework.SubmissionRequest;
+import org.apache.sqoop.framework.JobRequest;
import org.apache.sqoop.framework.SubmissionEngine;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.mr.ConfigurationUtils;
@@ -72,6 +72,7 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
*/
@Override
public void initialize(MapContext context, String prefix) {
+ super.initialize(context, prefix);
LOG.info("Initializing Map-reduce Submission Engine");
// Build global configuration, start with empty configuration object
@@ -125,6 +126,7 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
*/
@Override
public void destroy() {
+ super.destroy();
LOG.info("Destroying Mapreduce Submission Engine");
// Closing job client
@@ -147,9 +149,9 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
* {@inheritDoc}
*/
@Override
- public boolean submit(SubmissionRequest generalRequest) {
+ public boolean submit(JobRequest mrJobRequest) {
// We're supporting only map reduce jobs
- MRSubmissionRequest request = (MRSubmissionRequest) generalRequest;
+ MRJobRequest request = (MRJobRequest) mrJobRequest;
// Clone global configuration
Configuration configuration = new Configuration(globalConfiguration);
@@ -208,7 +210,7 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
ConfigurationUtils.setFrameworkConnectionConfig(Direction.TO, job, request.getFrameworkConnectionConfig(Direction.TO));
ConfigurationUtils.setFrameworkJobConfig(job, request.getConfigFrameworkJob());
// @TODO(Abe): Persist TO schema.
- ConfigurationUtils.setConnectorSchema(Direction.FROM, job, request.getSummary().getConnectorSchema());
+ ConfigurationUtils.setConnectorSchema(Direction.FROM, job, request.getSummary().getFromSchema());
if(request.getJobName() != null) {
job.setJobName("Sqoop: " + request.getJobName());
[37/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java
new file mode 100644
index 0000000..dea271a
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigDisplayer.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import static org.apache.sqoop.shell.ShellEnvironment.print;
+import static org.apache.sqoop.shell.ShellEnvironment.println;
+import static org.apache.sqoop.shell.ShellEnvironment.resourceString;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ResourceBundle;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.model.MAccountableEntity;
+import org.apache.sqoop.model.MBooleanInput;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MEnumInput;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MInputType;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MMapInput;
+import org.apache.sqoop.model.MStringInput;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.validation.Message;
+import org.apache.sqoop.validation.Status;
+
+/**
+ * Convenience static methods for displaying config related information
+ */
+public final class ConfigDisplayer {
+
+ public static void displayDriverConfigDetails(MDriverConfig driverConfig, ResourceBundle bundle) {
+ displayConfig(driverConfig.getConfigs(),
+ resourceString(Constants.RES_CONFIG_DISPLAYER_JOB), bundle);
+ }
+
+ public static void displayConnectorConfigDetails(MConnector connector, ResourceBundle bundle) {
+ displayConfig(
+ connector.getLinkConfig().getConfigs(),
+ resourceString(Constants.RES_CONFIG_DISPLAYER_LINK),
+ bundle);
+
+ displayConfig(
+ connector.getConfig(Direction.FROM).getConfigs(),
+ Direction.FROM.toString() + " " + resourceString(Constants.RES_CONFIG_DISPLAYER_JOB),
+ bundle);
+
+ displayConfig(
+ connector.getConfig(Direction.TO).getConfigs(),
+ Direction.TO.toString() + " " + resourceString(Constants.RES_CONFIG_DISPLAYER_JOB),
+ bundle);
+ }
+
+ private static void displayConfig(List<MConfig> configs,
+ String type,
+ ResourceBundle bundle) {
+ Iterator<MConfig> iterator = configs.iterator();
+ int findx = 1;
+ while (iterator.hasNext()) {
+ print(" ");
+ print(type);
+ print(" %s ", resourceString(Constants.RES_CONFIG_DISPLAYER_CONFIG));
+ print(findx++);
+ println(":");
+
+ MConfig config = iterator.next();
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_NAME));
+ println(config.getName());
+
+ // Label
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_LABEL));
+ println(bundle.getString(config.getLabelKey()));
+
+ // Help text
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_HELP));
+ println(bundle.getString(config.getHelpKey()));
+
+ List<MInput<?>> inputs = config.getInputs();
+ Iterator<MInput<?>> iiter = inputs.iterator();
+ int iindx = 1;
+ while (iiter.hasNext()) {
+ print(" %s ", resourceString(Constants.RES_CONFIG_DISPLAYER_INPUT));
+ print(iindx++);
+ println(":");
+
+ MInput<?> input = iiter.next();
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_NAME));
+ println(input.getName());
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_LABEL));
+ println(bundle.getString(input.getLabelKey()));
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_HELP));
+ println(bundle.getString(input.getHelpKey()));
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_TYPE));
+ println(input.getType());
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_SENSITIVE));
+ println(input.isSensitive());
+ if (input.getType() == MInputType.STRING) {
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_SIZE));
+ println(((MStringInput)input).getMaxLength());
+ } else if(input.getType() == MInputType.ENUM) {
+ print(" %s: ", resourceString(Constants.RES_CONFIG_DISPLAYER_POSSIBLE_VALUES));
+ println(StringUtils.join(((MEnumInput)input).getValues(), ","));
+ }
+ }
+ }
+ }
+
+ public static void displayConfig(List<MConfig> configs, ResourceBundle bundle) {
+ for(MConfig config : configs) {
+ displayConfig(config, bundle);
+ }
+ }
+
+ /**
+ * Method prints the warning message of ACCEPTABLE status
+ * @param entity - link or job instance
+ */
+ public static void displayConfigWarning(MAccountableEntity entity) {
+ List<MConfig> configList = new ArrayList<MConfig>();
+ boolean showMessage = true;
+ if (entity instanceof MLink) {
+ MLink link = (MLink) entity;
+ configList.addAll(link.getConnectorLinkConfig().getConfigs());
+ } else if(entity instanceof MJob) {
+ MJob job = (MJob) entity;
+ configList.addAll(job.getJobConfig(Direction.FROM).getConfigs());
+ configList.addAll(job.getDriverConfig().getConfigs());
+ configList.addAll(job.getJobConfig(Direction.TO).getConfigs());
+ }
+ for(MConfig config : configList) {
+ if(config.getValidationStatus() == Status.ACCEPTABLE) {
+ if(showMessage) {
+ print("\n@|yellow %s|@\n", resourceString(Constants.RES_CONFIG_DISPLAYER_FORM_WARNING));
+ showMessage = false;
+ }
+ for(Message message : config.getValidationMessages()) {
+ ConfigFiller.warningMessage(message.getMessage());
+ }
+ }
+ }
+ }
+
+ private static void displayConfig(MConfig config, ResourceBundle bundle) {
+ print(" ");
+ println(bundle.getString(config.getLabelKey()));
+
+ for (MInput<?> input : config.getInputs()) {
+ print(" ");
+ print(bundle.getString(input.getLabelKey()));
+ print(": ");
+ if(!input.isEmpty()) {
+ if (input.isSensitive()) {
+ print("(%s)", resourceString(Constants.RES_CONFIG_DISPLAYER_INPUT_SENSITIVE));
+ } else {
+ // Based on the input type, let's perconfig specific load
+ switch (input.getType()) {
+ case STRING:
+ displayInputString((MStringInput) input);
+ break;
+ case INTEGER:
+ displayInputInteger((MIntegerInput) input);
+ break;
+ case BOOLEAN:
+ displayInputBoolean((MBooleanInput) input);
+ break;
+ case MAP:
+ displayInputMap((MMapInput) input);
+ break;
+ case ENUM:
+ displayInputEnum((MEnumInput) input);
+ break;
+ default:
+ print("\n%s " + input.getType(), resourceString(Constants.RES_CONFIG_DISPLAYER_UNSUPPORTED_DATATYPE));
+ return;
+ }
+ }
+ }
+ println("");
+ }
+ }
+
+ /**
+ * Display content of String input.
+ *
+ * @param input String input
+ */
+ private static void displayInputString(MStringInput input) {
+ print(input.getValue());
+ }
+
+ /**
+ * Display content of Integer input.
+ *
+ * @param input Integer input
+ */
+ private static void displayInputInteger(MIntegerInput input) {
+ print(input.getValue());
+ }
+
+ /**
+ * Display content of Boolean input.
+ *
+ * @param input Boolean input
+ */
+ private static void displayInputBoolean(MBooleanInput input) {
+ print(input.getValue());
+ }
+
+ /**
+ * Display content of Map input
+ *
+ * @param input Map input
+ */
+ private static void displayInputMap(MMapInput input) {
+ for(Map.Entry<String, String> entry : input.getValue().entrySet()) {
+ println();
+ print(" ");
+ print(entry.getKey());
+ print(" = ");
+ print(entry.getValue());
+ }
+ }
+
+ /**
+ * Display content of Enum input
+ *
+ * @param input Enum input
+ */
+ private static void displayInputEnum(MEnumInput input) {
+ print(input.getValue());
+ }
+
+ private ConfigDisplayer() {
+ // Do not instantiate
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java
new file mode 100644
index 0000000..c61d33b
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigFiller.java
@@ -0,0 +1,911 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import jline.ConsoleReader;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang.StringUtils;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.model.MBooleanInput;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MEnumInput;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MMapInput;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MNamedElement;
+import org.apache.sqoop.model.MStringInput;
+import org.apache.sqoop.model.MValidatedElement;
+import org.apache.sqoop.validation.Message;
+import org.apache.sqoop.validation.Status;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.ResourceBundle;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+
+/**
+ * Convenient methods for retrieving user input and CLI options.
+ */
+public final class ConfigFiller {
+
+ /**
+ * Internal input that will be reused for loading names for link and
+ * job objects.
+ */
+ private static MStringInput nameInput = new MStringInput("object-name", false, (short)25);
+
+ /**
+ * Fill job object based on CLI options.
+ *
+ * @param line Associated console reader object
+ * @param job Job that user is suppose to fill in
+ * @return True if we filled all inputs, false if user has stopped processing
+ * @throws IOException
+ */
+ public static boolean fillJob(CommandLine line,
+ MJob job)
+ throws IOException {
+
+ job.setName(line.getOptionValue("name"));
+ return fillJobConfig(line,
+ job.getJobConfig(Direction.FROM).getConfigs(),
+ job.getJobConfig(Direction.TO).getConfigs(),
+ job.getDriverConfig().getConfigs());
+ }
+
+ /**
+ * Fill link object based on CLI options.
+ *
+ * @param line Associated command line options
+ * @param link Link that user is suppose to fill in
+ * @return True if we filled all inputs, false if user has stopped processing
+ * @throws IOException
+ */
+ public static boolean fillLink(CommandLine line, MLink link) throws IOException {
+
+ link.setName(line.getOptionValue("name"));
+ return fillLinkConfig(line, link.getConnectorLinkConfig().getConfigs());
+ }
+
+ /**
+ * Load CLI options for link configs
+ *
+ * @param line CLI options container
+ * @param linkConfig from config to read or edit
+ * @return
+ * @throws IOException
+ */
+ public static boolean fillLinkConfig(CommandLine line,
+ List<MConfig> linkConfig)
+ throws IOException {
+ return fillConfigs("link", linkConfig, line);
+ }
+
+ /**
+ * Load CLI options for job configs
+ *
+ * @param line CLI options container
+ * @param fromConfig from config to read or edit
+ * @param toConfig to config to read or edit
+ * @param driverConfig driver config to read or edit
+ * @return
+ * @throws IOException
+ */
+ public static boolean fillJobConfig(CommandLine line,
+ List<MConfig> fromConfig,
+ List<MConfig> toConfig,
+ List<MConfig> driverConfig)
+ throws IOException {
+ return fillConfigs("from", fromConfig, line)
+ && fillConfigs("to", toConfig, line)
+ && fillConfigs("driver", driverConfig, line);
+ }
+
+ /**
+ * Load all CLI options for a list of configs.
+ *
+ * @param prefix placed at the beginning of the CLI option key
+ * @param configs Forms to read or edit
+ * @param line CLI options container
+ * @return
+ * @throws IOException
+ */
+ static boolean fillConfigs(String prefix, List<MConfig> configs, CommandLine line)
+ throws IOException {
+ for (MConfig config : configs) {
+ if (!fillConfig(prefix, config, line)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Load all CLI options for a particular config.
+ *
+ * @param prefix placed at the beginning of the CLI option key
+ * @param config Config to read or edit
+ * @param line CLI options container
+ * @return
+ * @throws IOException
+ */
+ @SuppressWarnings("rawtypes")
+ static boolean fillConfig(String prefix, MConfig config, CommandLine line) throws IOException {
+ for (MInput input : config.getInputs()) {
+ if (!fillInput(prefix, input, line)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Load CLI option.
+ * Chooses the appropriate 'fill' method to use based on input type.
+ *
+ * Keys for CLI options are automatically created from the 'prefix' argument
+ * and 'input' argument: <prefix>-<config name>-<input name>
+ *
+ * @param prefix placed at the beginning of the CLI option key
+ * @param input Input that we should read or edit
+ * @param line CLI options container
+ * @return
+ * @throws IOException
+ */
+ @SuppressWarnings("rawtypes")
+ public static boolean fillInput(String prefix, MInput input, CommandLine line) throws IOException {
+ // Based on the input type, let's perconfig specific load
+ switch (input.getType()) {
+ case STRING:
+ return fillInputString(prefix, (MStringInput) input, line);
+ case INTEGER:
+ return fillInputInteger(prefix, (MIntegerInput) input, line);
+ case BOOLEAN:
+ return fillInputBoolean(prefix, (MBooleanInput) input, line);
+ case MAP:
+ return fillInputMap(prefix, (MMapInput) input, line);
+ case ENUM:
+ return fillInputEnum(prefix, (MEnumInput) input, line);
+ default:
+ println("Unsupported data type " + input.getType());
+ return true;
+ }
+ }
+
+ /**
+ * Load CLI option for enum type.
+ *
+ * Currently only supports numeric values.
+ *
+ * @param prefix placed at the beginning of the CLI option key
+ * @param input Input that we should read or edit
+ * @param line CLI options container
+ * @return
+ * @throws IOException
+ */
+ private static boolean fillInputEnum(String prefix,
+ MEnumInput input,
+ CommandLine line)
+ throws IOException {
+ String opt = ConfigOptions.getOptionKey(prefix, input);
+ if (line.hasOption(opt)) {
+ String value = line.getOptionValue(opt);
+ int index = java.util.Arrays.asList(input.getValues()).indexOf(value);
+
+ if(index < 0) {
+ errorMessage(input, String.format("Invalid option %s. Please use one of %s.", value, StringUtils.join(input.getValues(), ", ")));
+ return false;
+ }
+
+ input.setValue(value);
+ } else {
+ input.setEmpty();
+ }
+ return true;
+ }
+
+ /**
+ * Load CLI options for map type.
+ *
+ * Parses Key-Value pairs that take the config "<key>=<value>&<key>=<value>&...".
+ *
+ * @param prefix placed at the beginning of the CLI option key
+ * @param input Input that we should read or edit
+ * @param line CLI options container
+ * @return
+ * @throws IOException
+ */
+ private static boolean fillInputMap(String prefix,
+ MMapInput input,
+ CommandLine line)
+ throws IOException {
+ String opt = ConfigOptions.getOptionKey(prefix, input);
+ if (line.hasOption(opt)) {
+ String value = line.getOptionValue(opt);
+ Map<String, String> values = new HashMap<String, String>();
+ String[] entries = value.split("&");
+ for (String entry : entries) {
+ if (entry.contains("=")) {
+ String[] keyValue = entry.split("=");
+ values.put(keyValue[0], keyValue[1]);
+ } else {
+ errorMessage(input, "Don't know what to do with " + entry);
+ return false;
+ }
+ }
+ input.setValue(values);
+ } else {
+ input.setEmpty();
+ }
+ return true;
+ }
+
+ /**
+ * Load integer input from CLI option.
+ *
+ * @param prefix placed at the beginning of the CLI option key
+ * @param input Input that we should read or edit
+ * @param line CLI options container
+ * @return
+ * @throws IOException
+ */
+ private static boolean fillInputInteger(String prefix,
+ MIntegerInput input,
+ CommandLine line)
+ throws IOException {
+ String opt = ConfigOptions.getOptionKey(prefix, input);
+ if (line.hasOption(opt)) {
+ try {
+ input.setValue(Integer.valueOf(line.getOptionValue(ConfigOptions.getOptionKey(prefix, input))));
+ } catch (NumberFormatException ex) {
+ errorMessage(input, "Input is not valid integer number");
+ return false;
+ }
+ } else {
+ input.setEmpty();
+ }
+ return true;
+ }
+
+ /**
+ * Load string input from CLI option.
+ *
+ * @param prefix placed at the beginning of the CLI option key
+ * @param input Input that we should read or edit
+ * @param line CLI options container
+ * @return
+ * @throws IOException
+ */
+ public static boolean fillInputString(String prefix,
+ MStringInput input,
+ CommandLine line)
+ throws IOException {
+ String opt = ConfigOptions.getOptionKey(prefix, input);
+ if (line.hasOption(opt)) {
+ String value = line.getOptionValue(ConfigOptions.getOptionKey(prefix, input));
+ if(value.length() > input.getMaxLength()) {
+ errorMessage(input, "Size of input exceeds allowance for this input"
+ + " field. Maximal allowed size is " + input.getMaxLength());
+ }
+ input.setValue(value);
+ } else {
+ input.setEmpty();
+ }
+ return true;
+ }
+
+ /**
+ * Load boolean input from CLI option.
+ *
+ * @param prefix placed at the beginning of the CLI option key
+ * @param input Input that we should read or edit
+ * @param line CLI options container
+ * @return
+ * @throws IOException
+ */
+ public static boolean fillInputBoolean(String prefix,
+ MBooleanInput input,
+ CommandLine line)
+ throws IOException {
+ String opt = ConfigOptions.getOptionKey(prefix, input);
+ if (line.hasOption(opt)) {
+ input.setValue(Boolean.valueOf(line.getOptionValue(ConfigOptions.getOptionKey(prefix, input))));
+ } else {
+ input.setEmpty();
+ }
+ return true;
+ }
+
+ /**
+ * Fill link object based on user input.
+ *
+ * @param reader Associated console reader object
+ * @param link Link that user is suppose to fill in
+ * @param linkConfigBundle Connector resource bundle
+ * @return True if we filled all inputs, false if user has stopped processing
+ * @throws IOException
+ */
+ public static boolean fillLinkWithBundle(ConsoleReader reader, MLink link, ResourceBundle linkConfigBundle)
+ throws IOException {
+
+ link.setName(getName(reader, link.getName()));
+ return fillLinkConfigWithBundle(reader, link.getConnectorLinkConfig().getConfigs(), linkConfigBundle);
+ }
+
+ /**
+ * Fill job object based on user input.
+ *
+ * @param reader Associated console reader object
+ * @param job Job that user is suppose to fill in
+ * @param fromConfigBundle Connector resource bundle
+ * @param driverConfigBundle Driver config resource bundle
+ * @return True if we filled all inputs, false if user has stopped processing
+ * @throws IOException
+ */
+ public static boolean fillJobWithBundle(ConsoleReader reader,
+ MJob job,
+ ResourceBundle fromConfigBundle,
+ ResourceBundle toConfigBundle,
+ ResourceBundle driverConfigBundle)
+ throws IOException {
+
+ job.setName(getName(reader, job.getName()));
+
+ return fillJobConfigWithBundle(reader,
+ job.getJobConfig(Direction.FROM).getConfigs(),
+ fromConfigBundle,
+ job.getJobConfig(Direction.TO).getConfigs(),
+ toConfigBundle,
+ job.getDriverConfig().getConfigs(),
+ driverConfigBundle);
+ }
+ public static boolean fillLinkConfigWithBundle(ConsoleReader reader,
+ List<MConfig> linkConfig,
+ ResourceBundle linkConfigBundle) throws IOException {
+
+
+ if(!fillConfigsWithBundle(linkConfig, reader, linkConfigBundle)) {
+ return false;
+ }
+ return true;
+ }
+
+ public static boolean fillJobConfigWithBundle(ConsoleReader reader,
+ List<MConfig> fromConfig,
+ ResourceBundle fromConfigBundle,
+ List<MConfig> toConfig,
+ ResourceBundle toConfigBundle,
+ List<MConfig> driverConfig,
+ ResourceBundle driverConfigBundle) throws IOException {
+
+
+ // Job From config
+ if(!fillConfigsWithBundle(fromConfig, reader, fromConfigBundle)) {
+ return false;
+ }
+ // Job To config
+ if(!fillConfigsWithBundle(toConfig, reader, toConfigBundle)) {
+ return false;
+ }
+ // Job Driver config
+ if(!fillConfigsWithBundle(driverConfig, reader, driverConfigBundle)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ public static boolean fillConfigsWithBundle(List<MConfig> configs, ConsoleReader reader,
+ ResourceBundle configBundle) throws IOException {
+ for (MConfig config : configs) {
+ if (!fillConfigWithBundle(config, reader, configBundle)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static boolean fillConfigWithBundle(MConfig config, ConsoleReader reader, ResourceBundle bundle)
+ throws IOException {
+ println("");
+ println(bundle.getString(config.getLabelKey()));
+
+ // Print out config validation
+ printValidationMessage(config, false);
+ println("");
+
+ for (MInput input : config.getInputs()) {
+ if(!fillInputWithBundle(input, reader, bundle)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @SuppressWarnings("rawtypes")
+ static boolean fillInputWithBundle(MInput input, ConsoleReader reader, ResourceBundle bundle)
+ throws IOException {
+ // Print out validation
+ printValidationMessage(input, false);
+
+ // Based on the input type, let's perconfig specific load
+ switch (input.getType()) {
+ case STRING:
+ return fillInputStringWithBundle((MStringInput) input, reader, bundle);
+ case INTEGER:
+ return fillInputInteger((MIntegerInput) input, reader, bundle);
+ case BOOLEAN:
+ return fillInputBooleanWithBundle((MBooleanInput) input, reader, bundle);
+ case MAP:
+ return fillInputMapWithBundle((MMapInput) input, reader, bundle);
+ case ENUM:
+ return fillInputEnumWithBundle((MEnumInput) input, reader, bundle);
+ default:
+ println("Unsupported data type " + input.getType());
+ return true;
+ }
+ }
+
+ /**
+ * Load user input for enum type.
+ *
+ * Print out numbered list of all available options and let user choose one
+ * item from that.
+ *
+ * @param input Input that we should read or edit
+ * @param reader Associated console reader
+ * @param bundle Resource bundle
+ * @return True if user with to continue with loading addtional inputs
+ * @throws IOException
+ */
+ private static boolean fillInputEnumWithBundle(MEnumInput input,
+ ConsoleReader reader,
+ ResourceBundle bundle)
+ throws IOException {
+ // Prompt in enum case
+ println(bundle.getString(input.getLabelKey()) + ": ");
+
+ // Indexes
+ int i = -1;
+ int lastChoice = -1;
+
+ // Print out all values as a numbered list
+ for(String value : input.getValues()) {
+ i++;
+
+ println(" " + i + " : " + value);
+
+ // Only show last choice if not sensitive
+ if(!input.isEmpty() && value.equals(input.getValue()) && !input.isSensitive()) {
+ lastChoice = i;
+ }
+ }
+
+ // Prompt
+ reader.printString("Choose: ");
+
+ // Fill previously filled index when available
+ if(lastChoice != -1) {
+ reader.putString(Integer.toString(lastChoice));
+ }
+
+ reader.flushConsole();
+ String userTyped;
+ if(input.isSensitive()) {
+ userTyped = reader.readLine('*');
+ } else {
+ userTyped = reader.readLine();
+ }
+
+ if (userTyped == null) {
+ return false;
+ } else if (userTyped.isEmpty()) {
+ input.setEmpty();
+ } else {
+ Integer index;
+ try {
+ index = Integer.valueOf(userTyped);
+
+ if(index < 0 || index >= input.getValues().length) {
+ errorMessage("Invalid index");
+ return fillInputEnumWithBundle(input, reader, bundle);
+ }
+
+ input.setValue(input.getValues()[index]);
+ } catch (NumberFormatException ex) {
+ errorMessage("Input is not valid integer number");
+ return fillInputEnumWithBundle(input, reader, bundle);
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Load user input for map type.
+ *
+ * This implementation will load one map entry at the time. Current flows is
+ * as follows: if user did not enter anything (empty input) finish loading
+ * and return from function. If user specified input with equal sign (=),
+ * lets add new key value pair. Otherwise consider entire input as a key name
+ * and try to remove it from the map.
+ *
+ * Please note that following code do not supports equal sign in property
+ * name. It's however perfectly fine to have equal sign in value.
+ *
+ * @param input Input that we should read or edit
+ * @param reader Associated console reader
+ * @param bundle Resource bundle
+ * @return True if user wish to continue with loading additional inputs
+ * @throws IOException
+ */
+ private static boolean fillInputMapWithBundle(MMapInput input,
+ ConsoleReader reader,
+ ResourceBundle bundle)
+ throws IOException {
+ // Special prompt in Map case
+ println(bundle.getString(input.getLabelKey()) + ": ");
+
+ // Internal loading map
+ Map<String, String> values = input.getValue();
+ if(values == null) {
+ values = new HashMap<String, String>();
+ }
+
+ String userTyped;
+
+ while(true) {
+ // Print all current items in each iteration
+ // However do not printout if this input contains sensitive information.
+ println("There are currently " + values.size() + " values in the map:");
+ if (!input.isSensitive()) {
+ for(Map.Entry<String, String> entry : values.entrySet()) {
+ println(entry.getKey() + " = " + entry.getValue());
+ }
+ }
+
+ // Special prompt for Map entry
+ reader.printString("entry# ");
+ reader.flushConsole();
+
+ if(input.isSensitive()) {
+ userTyped = reader.readLine('*');
+ } else {
+ userTyped = reader.readLine();
+ }
+
+ if(userTyped == null) {
+ // Finish loading and return back to Sqoop shell
+ return false;
+ } else if(userTyped.isEmpty()) {
+ // User has finished loading data to Map input, either set input empty
+ // if there are no entries or propagate entries to the input
+ if(values.size() == 0) {
+ input.setEmpty();
+ } else {
+ input.setValue(values);
+ }
+ return true;
+ } else {
+ // User has specified regular input, let's check if it contains equals
+ // sign. Save new entry (or update existing one) if it does. Otherwise
+ // try to remove entry that user specified.
+ if(userTyped.contains("=")) {
+ String []keyValue = userTyped.split("=", 2);
+ values.put(handleUserInput(keyValue[0]), handleUserInput(keyValue[1]));
+ } else {
+ String key = handleUserInput(userTyped);
+ if(values.containsKey(key)) {
+ values.remove(key);
+ } else {
+ errorMessage("Don't know what to do with " + userTyped);
+ }
+ }
+ }
+
+ }
+ }
+
+ /**
+ * Handle special cases in user input.
+ *
+ * Preserve null and empty values, remove whitespace characters before and
+ * after loaded string and de-quote the string if it's quoted (to preserve
+ * spaces for example).
+ *
+ * @param input String loaded from user
+ * @return Unquoted transconfiged string
+ */
+ private static String handleUserInput(String input) {
+ // Preserve null and empty values
+ if(input == null) {
+ return null;
+ }
+ if(input.isEmpty()) {
+ return input;
+ }
+
+ // Removes empty characters at the begging and end of loaded string
+ input = input.trim();
+
+ int lastIndex = input.length() - 1;
+ char first = input.charAt(0);
+ char last = input.charAt(lastIndex);
+
+ // Remove quoting if present
+ if(first == '\'' && last == '\'') {
+ input = input.substring(1, lastIndex);
+ } else if(first == '"' && last == '"') {
+ input = input.substring(1, lastIndex);
+ }
+
+ // Return final string
+ return input;
+ }
+
+ private static boolean fillInputInteger(MIntegerInput input,
+ ConsoleReader reader,
+ ResourceBundle bundle)
+ throws IOException {
+ generatePrompt(reader, bundle, input);
+
+ // Fill already filled data when available
+ // However do not printout if this input contains sensitive information.
+ if(!input.isEmpty() && !input.isSensitive()) {
+ reader.putString(input.getValue().toString());
+ }
+
+ // Get the data
+ String userTyped;
+ if(input.isSensitive()) {
+ userTyped = reader.readLine('*');
+ } else {
+ userTyped = reader.readLine();
+ }
+
+ if (userTyped == null) {
+ return false;
+ } else if (userTyped.isEmpty()) {
+ input.setEmpty();
+ } else {
+ Integer value;
+ try {
+ value = Integer.valueOf(userTyped);
+ input.setValue(value);
+ } catch (NumberFormatException ex) {
+ errorMessage("Input is not valid integer number");
+ return fillInputInteger(input, reader, bundle);
+ }
+
+ input.setValue(Integer.valueOf(userTyped));
+ }
+
+ return true;
+ }
+
+ /**
+ * Load string input from the user.
+ *
+ * @param input Input that we should load in
+ * @param reader Associated console reader
+ * @param bundle Resource bundle for this input
+ * @return
+ * @throws IOException
+ */
+ static boolean fillInputStringWithBundle(MStringInput input,
+ ConsoleReader reader,
+ ResourceBundle bundle)
+ throws IOException {
+ generatePrompt(reader, bundle, input);
+
+ // Fill already filled data when available
+ // However do not printout if this input contains sensitive information.
+ if(!input.isEmpty() && !input.isSensitive()) {
+ reader.putString(input.getValue());
+ }
+
+ // Get the data
+ String userTyped;
+ if(input.isSensitive()) {
+ userTyped = reader.readLine('*');
+ } else {
+ userTyped = reader.readLine();
+ }
+
+ if (userTyped == null) {
+ // Propagate end of loading process
+ return false;
+ } else if (userTyped.isEmpty()) {
+ // Empty input in case that nothing was given
+ input.setEmpty();
+ } else {
+ // Set value that user has entered
+ input.setValue(userTyped);
+
+ // Check that it did not exceeds maximal allowance for given input
+ if(userTyped.length() > input.getMaxLength()) {
+ errorMessage("Size of input exceeds allowance for this input"
+ + " field. Maximal allowed size is " + input.getMaxLength());
+ return fillInputStringWithBundle(input, reader, bundle);
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Load boolean input from the user.
+ *
+ * @param input Input that we should load in
+ * @param reader Associated console reader
+ * @param bundle Resource bundle for this input
+ * @return
+ * @throws IOException
+ */
+ static boolean fillInputBooleanWithBundle(MBooleanInput input,
+ ConsoleReader reader,
+ ResourceBundle bundle)
+ throws IOException {
+ generatePrompt(reader, bundle, input);
+
+ // Fill already filled data when available
+ // However do not printout if this input contains sensitive information.
+ if(!input.isEmpty() && !input.isSensitive()) {
+ reader.putString(input.getValue().toString());
+ }
+
+ // Get the data
+ String userTyped;
+ if(input.isSensitive()) {
+ userTyped = reader.readLine('*');
+ } else {
+ userTyped = reader.readLine();
+ }
+
+ if (userTyped == null) {
+ // Propagate end of loading process
+ return false;
+ } else if (userTyped.isEmpty()) {
+ // Empty input in case that nothing was given
+ input.setEmpty();
+ } else {
+ // Set value that user has entered
+ input.setValue(Boolean.valueOf(userTyped));
+ }
+
+ return true;
+ }
+
+ @SuppressWarnings("rawtypes")
+ static void generatePrompt(ConsoleReader reader, ResourceBundle bundle, MInput input)
+ throws IOException {
+ reader.printString(bundle.getString(input.getLabelKey()) + ": ");
+ reader.flushConsole();
+ }
+
+ static String getName(ConsoleReader reader, String name) throws IOException {
+ if (name == null) {
+ nameInput.setEmpty();
+ } else {
+ nameInput.setValue(name);
+ }
+
+ fillInputStringWithBundle(nameInput, reader, getResourceBundle());
+
+ return nameInput.getValue();
+ }
+
+ /**
+ * Print validation message in cases that it's not in state "FINE"
+ *
+ * @param element Validated element
+ */
+ static void printValidationMessage(MValidatedElement element, boolean includeInputPrefix) {
+ if(element.getValidationStatus() == Status.getDefault()) {
+ return;
+ }
+
+ for(Message message : element.getValidationMessages())
+ switch (message.getStatus()) {
+ case UNACCEPTABLE:
+ if (includeInputPrefix) {
+ errorMessage(element, message.getMessage());
+ } else {
+ errorMessage(message.getMessage());
+ }
+ break;
+ case ACCEPTABLE:
+ if (includeInputPrefix) {
+ warningMessage(element, message.getMessage());
+ } else {
+ warningMessage(message.getMessage());
+ }
+ break;
+ default:
+ // Simply ignore all other states for the moment
+ break;
+ }
+ }
+
+ static void errorMessage(String message) {
+ println("Error message: @|red " + message + " |@");
+ }
+
+ static void errorMessage(MNamedElement input, String message) {
+ print(input.getName());
+ print(": ");
+ errorMessage(message);
+ }
+
+ static void warningMessage(String message) {
+ println("Warning message: @|yellow " + message + " |@");
+ }
+
+ static void warningMessage(MNamedElement input, String message) {
+ print(input.getName());
+ print(": ");
+ warningMessage(message);
+ }
+
+ public static void errorIntroduction() {
+ println("\n @|red There are issues with entered data, please revise your input:|@");
+ }
+
+ // link object has the connector link config
+ public static void printLinkValidationMessages(MLink link) {
+ for (MConfig config : link.getConnectorLinkConfig().getConfigs()) {
+ for (MInput<?> input : config.getInputs()) {
+ printValidationMessage(input, true);
+ }
+ }
+ }
+
+ // job has the from/to and the driver config
+ public static void printJobValidationMessages(MJob job) {
+ for (MConfig config : job.getJobConfig(Direction.FROM).getConfigs()) {
+ for (MInput<?> input : config.getInputs()) {
+ printValidationMessage(input, true);
+ }
+ }
+
+ for (MConfig config : job.getJobConfig(Direction.TO).getConfigs()) {
+ for (MInput<?> input : config.getInputs()) {
+ printValidationMessage(input, true);
+ }
+ }
+
+ for (MConfig config : job.getDriverConfig().getConfigs()) {
+ for (MInput<?> input : config.getInputs()) {
+ printValidationMessage(input, true);
+ }
+ }
+ }
+
+ private ConfigFiller() {
+ // Do not instantiate
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigOptions.java
new file mode 100644
index 0000000..97b6e3b
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/ConfigOptions.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MInputType;
+import org.apache.sqoop.shell.core.ShellError;
+
+/**
+ * Utilities for automatically creating org.apache.commons.cli.Option objects.
+ */
+public class ConfigOptions {
+ /**
+ * This method is used to automatically generate keys
+ * for a particular input.
+ *
+ * @param prefix Prefix to prepend to CLI option keys
+ * @param input
+ * @return
+ */
+ @SuppressWarnings("rawtypes")
+ public static String getOptionKey(String prefix, MInput input) {
+ return prefix + "-" + input.getName().replace('.', '-');
+ }
+
+ /**
+ * This method is used to automatically generate CLI options
+ * for a list of configs.
+ *
+ * @param prefix Prefix to prepend to CLI option keys
+ * @param configs Forms to get options for
+ * @return
+ */
+ public static List<Option> getConfigsOptions(String prefix, List<MConfig> configs) {
+ List<Option> options = new LinkedList<Option>();
+ for (MConfig config : configs) {
+ List<Option> configOptions = getConfigOptions(prefix, config);
+ options.addAll(configOptions);
+ }
+ return options;
+ }
+
+ /**
+ * This method is used to automatically generate CLI options
+ * for a particular config.
+ *
+ * @param prefix Prefix to prepend to CLI option keys
+ * @param config Config to get options for
+ * @return List<Option>
+ */
+ @SuppressWarnings({ "rawtypes", "static-access" })
+ public static List<Option> getConfigOptions(String prefix, MConfig config) {
+ List<Option> options = new LinkedList<Option>();
+ for (MInput input : config.getInputs()) {
+ if (input.getType().equals(MInputType.BOOLEAN)) {
+ options.add(OptionBuilder
+ .withLongOpt(getOptionKey(prefix, input))
+ .create());
+ } else {
+ options.add(OptionBuilder
+ .withLongOpt(getOptionKey(prefix, input))
+ .hasArg()
+ .create());
+ }
+ }
+ return options;
+ }
+
+ /**
+ * Parses command line options.
+ *
+ * @param options parse arglist against these.
+ * @param start beginning index in arglist.
+ * @param arglist arguments to parse.
+ * @param stopAtNonOption stop parsing when nonoption found in arglist.
+ * @return CommandLine object
+ */
+ public static CommandLine parseOptions(Options options, int start, List<String> arglist, boolean stopAtNonOption) {
+ String[] args = arglist.subList(start, arglist.size()).toArray(new String[arglist.size() - start]);
+
+ CommandLineParser parser = new GnuParser();
+ CommandLine line;
+ try {
+ line = parser.parse(options, args, stopAtNonOption);
+ } catch (ParseException e) {
+ throw new SqoopException(ShellError.SHELL_0003, e.getMessage(), e);
+ }
+ return line;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/DynamicConfigOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/DynamicConfigOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/DynamicConfigOptions.java
new file mode 100644
index 0000000..d12bd2f
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/DynamicConfigOptions.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import org.apache.commons.cli.Options;
+
+/**
+ * Automatically create options for different components.
+ */
+@SuppressWarnings("serial")
+public abstract class DynamicConfigOptions<M> extends Options {
+
+ /**
+ * Create dynamic options.
+ *
+ * @param model generate options from this
+ * @return this
+ */
+ public abstract void prepareOptions(M model);
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/DynamicFormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/DynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/DynamicFormOptions.java
deleted file mode 100644
index cc63610..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/DynamicFormOptions.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import org.apache.commons.cli.Options;
-
-/**
- * Automatically create options for different components.
- */
-@SuppressWarnings("serial")
-public abstract class DynamicFormOptions<M> extends Options {
-
- /**
- * Create dynamic options.
- *
- * @param model generate options from this
- * @return this
- */
- public abstract void prepareOptions(M model);
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
deleted file mode 100644
index dcbccef..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MAccountableEntity;
-import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MEnumInput;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MInputType;
-import org.apache.sqoop.model.MIntegerInput;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MMapInput;
-import org.apache.sqoop.model.MStringInput;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.validation.Message;
-import org.apache.sqoop.validation.Status;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-
-/**
- * Convenience static methods for displaying form related information
- */
-public final class FormDisplayer {
-
- public static void displayFormMetadataDetails(MDriverConfig driverConfig,
- ResourceBundle bundle) {
- displayFormsMetadata(
- driverConfig.getConnectionForms().getForms(),
- resourceString(Constants.RES_FORMDISPLAYER_CONNECTION),
- bundle);
-
- displayFormsMetadata(
- driverConfig.getJobForms().getForms(),
- resourceString(Constants.RES_FORMDISPLAYER_JOB),
- bundle);
- }
-
- public static void displayFormMetadataDetails(MConnector connector,
- ResourceBundle bundle) {
- displayFormsMetadata(
- connector.getConnectionForms().getForms(),
- resourceString(Constants.RES_FORMDISPLAYER_CONNECTION),
- bundle);
-
- // @TODO(Abe): Validate From/To output is correct.
- displayFormsMetadata(
- connector.getJobForms(Direction.FROM).getForms(),
- Direction.FROM.toString() + " " + resourceString(Constants.RES_FORMDISPLAYER_JOB),
- bundle);
-
- displayFormsMetadata(
- connector.getJobForms(Direction.TO).getForms(),
- Direction.TO.toString() + " " + resourceString(Constants.RES_FORMDISPLAYER_JOB),
- bundle);
- }
-
- public static void displayFormsMetadata(List<MForm> forms,
- String type,
- ResourceBundle bundle) {
- Iterator<MForm> fiter = forms.iterator();
- int findx = 1;
- while (fiter.hasNext()) {
- print(" ");
- print(type);
- print(" %s ", resourceString(Constants.RES_FORMDISPLAYER_FORM));
- print(findx++);
- println(":");
-
- MForm form = fiter.next();
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_NAME));
- println(form.getName());
-
- // Label
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_LABEL));
- println(bundle.getString(form.getLabelKey()));
-
- // Help text
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_HELP));
- println(bundle.getString(form.getHelpKey()));
-
- List<MInput<?>> inputs = form.getInputs();
- Iterator<MInput<?>> iiter = inputs.iterator();
- int iindx = 1;
- while (iiter.hasNext()) {
- print(" %s ", resourceString(Constants.RES_FORMDISPLAYER_INPUT));
- print(iindx++);
- println(":");
-
- MInput<?> input = iiter.next();
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_NAME));
- println(input.getName());
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_LABEL));
- println(bundle.getString(input.getLabelKey()));
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_HELP));
- println(bundle.getString(input.getHelpKey()));
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_TYPE));
- println(input.getType());
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_SENSITIVE));
- println(input.isSensitive());
- if (input.getType() == MInputType.STRING) {
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_SIZE));
- println(((MStringInput)input).getMaxLength());
- } else if(input.getType() == MInputType.ENUM) {
- print(" %s: ", resourceString(Constants.RES_FORMDISPLAYER_POSSIBLE_VALUES));
- println(StringUtils.join(((MEnumInput)input).getValues(), ","));
- }
- }
- }
- }
-
- public static void displayForms(List<MForm> forms, ResourceBundle bundle) {
- for(MForm form : forms) {
- displayForm(form, bundle);
- }
- }
-
- /**
- * Method prints the warning message of ACCEPTABLE status
- * @param entity - link or job instance
- */
- public static void displayFormWarning(MAccountableEntity entity) {
- List<MForm> formList = new ArrayList<MForm>();
- boolean showMessage = true;
- if (entity instanceof MLink) {
- MLink link = (MLink) entity;
- formList.addAll(link.getConnectorPart().getForms());
- formList.addAll(link.getFrameworkPart().getForms());
- } else if(entity instanceof MJob) {
- MJob job = (MJob) entity;
- formList.addAll(job.getConnectorPart(Direction.FROM).getForms());
- formList.addAll(job.getFrameworkPart().getForms());
- formList.addAll(job.getConnectorPart(Direction.TO).getForms());
- }
- for(MForm form : formList) {
- if(form.getValidationStatus() == Status.ACCEPTABLE) {
- if(showMessage) {
- print("\n@|yellow %s|@\n", resourceString(Constants.RES_FORMDISPLAYER_FORM_WARNING));
- showMessage = false;
- }
- for(Message message : form.getValidationMessages()) {
- FormFiller.warningMessage(message.getMessage());
- }
- }
- }
- }
-
- private static void displayForm(MForm form, ResourceBundle bundle) {
- print(" ");
- println(bundle.getString(form.getLabelKey()));
-
- for (MInput<?> input : form.getInputs()) {
- print(" ");
- print(bundle.getString(input.getLabelKey()));
- print(": ");
- if(!input.isEmpty()) {
- if (input.isSensitive()) {
- print("(%s)", resourceString(Constants.RES_FORMDISPLAYER_INPUT_SENSITIVE));
- } else {
- // Based on the input type, let's perform specific load
- switch (input.getType()) {
- case STRING:
- displayInputString((MStringInput) input);
- break;
- case INTEGER:
- displayInputInteger((MIntegerInput) input);
- break;
- case BOOLEAN:
- displayInputBoolean((MBooleanInput) input);
- break;
- case MAP:
- displayInputMap((MMapInput) input);
- break;
- case ENUM:
- displayInputEnum((MEnumInput) input);
- break;
- default:
- print("\n%s " + input.getType(), resourceString(Constants.RES_FORMDISPLAYER_UNSUPPORTED_DATATYPE));
- return;
- }
- }
- }
- println("");
- }
- }
-
- /**
- * Display content of String input.
- *
- * @param input String input
- */
- private static void displayInputString(MStringInput input) {
- print(input.getValue());
- }
-
- /**
- * Display content of Integer input.
- *
- * @param input Integer input
- */
- private static void displayInputInteger(MIntegerInput input) {
- print(input.getValue());
- }
-
- /**
- * Display content of Boolean input.
- *
- * @param input Boolean input
- */
- private static void displayInputBoolean(MBooleanInput input) {
- print(input.getValue());
- }
-
- /**
- * Display content of Map input
- *
- * @param input Map input
- */
- private static void displayInputMap(MMapInput input) {
- for(Map.Entry<String, String> entry : input.getValue().entrySet()) {
- println();
- print(" ");
- print(entry.getKey());
- print(" = ");
- print(entry.getValue());
- }
- }
-
- /**
- * Display content of Enum input
- *
- * @param input Enum input
- */
- private static void displayInputEnum(MEnumInput input) {
- print(input.getValue());
- }
-
- private FormDisplayer() {
- // Do not instantiate
- }
-}
[25/52] [abbrv] git commit: SQOOP-1488: Sqoop2: From/To: Run both
destroyers
Posted by ab...@apache.org.
SQOOP-1488: Sqoop2: From/To: Run both destroyers
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b04e796f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b04e796f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b04e796f
Branch: refs/heads/SQOOP-1367
Commit: b04e796f01cb659efc55314029fa18cfd80bb16d
Parents: 27fb31d
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Sun Sep 21 13:00:46 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:18 2014 -0700
----------------------------------------------------------------------
.../mapreduce/MapreduceExecutionEngine.java | 3 +-
.../java/org/apache/sqoop/job/JobConstants.java | 6 ++--
.../sqoop/job/mr/SqoopDestroyerExecutor.java | 33 ++++++++++++++------
.../sqoop/job/mr/SqoopFileOutputFormat.java | 7 +++--
.../sqoop/job/mr/SqoopNullOutputFormat.java | 7 +++--
.../org/apache/sqoop/job/TestMapReduce.java | 29 +++++++++++++++++
.../jdbc/generic/TableStagedRDBMSTest.java | 15 ++++-----
7 files changed, 75 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b04e796f/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
index ef7ff4e..049d183 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
@@ -68,7 +68,8 @@ public class MapreduceExecutionEngine extends ExecutionEngine {
context.setString(JobConstants.JOB_ETL_PARTITIONER, from.getPartitioner().getName());
context.setString(JobConstants.JOB_ETL_EXTRACTOR, from.getExtractor().getName());
context.setString(JobConstants.JOB_ETL_LOADER, to.getLoader().getName());
- context.setString(JobConstants.JOB_ETL_DESTROYER, from.getDestroyer().getName());
+ context.setString(JobConstants.JOB_ETL_FROM_DESTROYER, from.getDestroyer().getName());
+ context.setString(JobConstants.JOB_ETL_TO_DESTROYER, to.getDestroyer().getName());
context.setString(JobConstants.INTERMEDIATE_DATA_FORMAT,
mrJobRequest.getIntermediateDataFormat().getName());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b04e796f/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
index 4cdb002..542607a 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/JobConstants.java
@@ -37,9 +37,11 @@ public final class JobConstants extends Constants {
public static final String JOB_ETL_LOADER = PREFIX_JOB_CONFIG
+ "etl.loader";
- public static final String JOB_ETL_DESTROYER = PREFIX_JOB_CONFIG
- + "etl.destroyer";
+ public static final String JOB_ETL_FROM_DESTROYER = PREFIX_JOB_CONFIG
+ + "etl.from.destroyer";
+ public static final String JOB_ETL_TO_DESTROYER = PREFIX_JOB_CONFIG
+ + "etl.to.destroyer";
public static final String JOB_MR_OUTPUT_FILE = PREFIX_JOB_CONFIG
+ "mr.output.file";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b04e796f/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
index e3af6e1..aecde40 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
@@ -40,10 +40,24 @@ public class SqoopDestroyerExecutor {
* @param success True if the job execution was successfull
* @param configuration Configuration object to get destroyer class with context
* and configuration objects.
- * @param propertyName Name of property that holds destroyer class.
+ * @param direction The direction of the Destroyer to execute.
*/
- public static void executeDestroyer(boolean success, Configuration configuration, String propertyName) {
- Destroyer destroyer = (Destroyer) ClassUtils.instantiate(configuration.get(propertyName));
+ public static void executeDestroyer(boolean success, Configuration configuration, Direction direction) {
+ String destroyerPropertyName, prefixPropertyName;
+ switch (direction) {
+ default:
+ case FROM:
+ destroyerPropertyName = JobConstants.JOB_ETL_FROM_DESTROYER;
+ prefixPropertyName = JobConstants.PREFIX_CONNECTOR_FROM_CONTEXT;
+ break;
+
+ case TO:
+ destroyerPropertyName = JobConstants.JOB_ETL_TO_DESTROYER;
+ prefixPropertyName = JobConstants.PREFIX_CONNECTOR_TO_CONTEXT;
+ break;
+ }
+
+ Destroyer destroyer = (Destroyer) ClassUtils.instantiate(configuration.get(destroyerPropertyName));
if(destroyer == null) {
LOG.info("Skipping running destroyer as non was defined.");
@@ -51,16 +65,17 @@ public class SqoopDestroyerExecutor {
}
// Objects that should be pass to the Destroyer execution
- PrefixContext subContext = new PrefixContext(configuration, JobConstants.PREFIX_CONNECTOR_FROM_CONTEXT);
- Object fromConfigConnection = ConfigurationUtils.getConnectorConnectionConfig(Direction.FROM, configuration);
- Object fromConfigJob = ConfigurationUtils.getConnectorJobConfig(Direction.FROM, configuration);
+ PrefixContext subContext = new PrefixContext(configuration, prefixPropertyName);
+ Object configConnection = ConfigurationUtils.getConnectorConnectionConfig(direction, configuration);
+ Object configJob = ConfigurationUtils.getConnectorJobConfig(direction, configuration);
+
+ // Propagate connector schema in every case for now
+ Schema schema = ConfigurationUtils.getConnectorSchema(direction, configuration);
- // TODO(Abe/Gwen): Change to conditional choosing between schemas.
- Schema schema = ConfigurationUtils.getConnectorSchema(Direction.FROM, configuration);
DestroyerContext destroyerContext = new DestroyerContext(subContext, success, schema);
LOG.info("Executing destroyer class " + destroyer.getClass());
- destroyer.destroy(destroyerContext, fromConfigConnection, fromConfigJob);
+ destroyer.destroy(destroyerContext, configConnection, configJob);
}
private SqoopDestroyerExecutor() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b04e796f/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java
index 3e2b1c5..ca77e16 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopFileOutputFormat.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.log4j.Logger;
+import org.apache.sqoop.common.Direction;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.io.SqoopWritable;
@@ -84,7 +85,8 @@ public class SqoopFileOutputFormat
super.commitJob(context);
Configuration config = context.getConfiguration();
- SqoopDestroyerExecutor.executeDestroyer(true, config, JobConstants.JOB_ETL_DESTROYER);
+ SqoopDestroyerExecutor.executeDestroyer(true, config, Direction.FROM);
+ SqoopDestroyerExecutor.executeDestroyer(true, config, Direction.TO);
}
@Override
@@ -92,7 +94,8 @@ public class SqoopFileOutputFormat
super.abortJob(context, state);
Configuration config = context.getConfiguration();
- SqoopDestroyerExecutor.executeDestroyer(false, config, JobConstants.JOB_ETL_DESTROYER);
+ SqoopDestroyerExecutor.executeDestroyer(false, config, Direction.FROM);
+ SqoopDestroyerExecutor.executeDestroyer(false, config, Direction.TO);
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b04e796f/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
index b3461bb..594b5e9 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.log4j.Logger;
+import org.apache.sqoop.common.Direction;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.io.SqoopWritable;
@@ -67,7 +68,8 @@ public class SqoopNullOutputFormat extends OutputFormat<SqoopWritable, NullWrita
super.commitJob(jobContext);
Configuration config = jobContext.getConfiguration();
- SqoopDestroyerExecutor.executeDestroyer(true, config, JobConstants.JOB_ETL_DESTROYER);
+ SqoopDestroyerExecutor.executeDestroyer(true, config, Direction.FROM);
+ SqoopDestroyerExecutor.executeDestroyer(true, config, Direction.TO);
}
@Override
@@ -75,7 +77,8 @@ public class SqoopNullOutputFormat extends OutputFormat<SqoopWritable, NullWrita
super.abortJob(jobContext, state);
Configuration config = jobContext.getConfiguration();
- SqoopDestroyerExecutor.executeDestroyer(false, config, JobConstants.JOB_ETL_DESTROYER);
+ SqoopDestroyerExecutor.executeDestroyer(false, config, Direction.FROM);
+ SqoopDestroyerExecutor.executeDestroyer(false, config, Direction.TO);
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b04e796f/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
index 2dfc487..869c727 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
+import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
@@ -36,6 +37,8 @@ import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.connector.idf.CSVIntermediateDataFormat;
+import org.apache.sqoop.job.etl.Destroyer;
+import org.apache.sqoop.job.etl.DestroyerContext;
import org.apache.sqoop.job.etl.Extractor;
import org.apache.sqoop.job.etl.ExtractorContext;
import org.apache.sqoop.job.etl.Loader;
@@ -100,6 +103,8 @@ public class TestMapReduce extends TestCase {
conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
conf.set(JobConstants.JOB_ETL_EXTRACTOR, DummyExtractor.class.getName());
conf.set(JobConstants.JOB_ETL_LOADER, DummyLoader.class.getName());
+ conf.set(JobConstants.JOB_ETL_FROM_DESTROYER, DummyFromDestroyer.class.getName());
+ conf.set(JobConstants.JOB_ETL_TO_DESTROYER, DummyToDestroyer.class.getName());
conf.set(JobConstants.INTERMEDIATE_DATA_FORMAT,
CSVIntermediateDataFormat.class.getName());
Schema schema = new Schema("Test");
@@ -110,6 +115,10 @@ public class TestMapReduce extends TestCase {
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, schema);
JobUtils.runJob(job.getConfiguration(), SqoopInputFormat.class, SqoopMapper.class,
SqoopNullOutputFormat.class);
+
+ // Make sure both destroyers get called.
+ Assert.assertEquals(1, DummyFromDestroyer.count);
+ Assert.assertEquals(1, DummyToDestroyer.count);
}
public static class DummyPartition extends Partition {
@@ -251,4 +260,24 @@ public class TestMapReduce extends TestCase {
}
}
}
+
+ public static class DummyFromDestroyer extends Destroyer {
+
+ public static int count = 0;
+
+ @Override
+ public void destroy(DestroyerContext context, Object o, Object o2) {
+ count++;
+ }
+ }
+
+ public static class DummyToDestroyer extends Destroyer {
+
+ public static int count = 0;
+
+ @Override
+ public void destroy(DestroyerContext context, Object o, Object o2) {
+ count++;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b04e796f/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
index cb782c7..1af0cdc 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
@@ -68,15 +68,12 @@ public class TableStagedRDBMSTest extends ConnectorTestCase {
runJob(job);
- // @TODO(Abe): Change back after SQOOP-1488
-// assertEquals(0L, provider.rowCount(stageTableName));
-// assertEquals(4L, rowCount());
-// assertRowInCities(1, "USA", "San Francisco");
-// assertRowInCities(2, "USA", "Sunnyvale");
-// assertRowInCities(3, "Czech Republic", "Brno");
-// assertRowInCities(4, "USA", "Palo Alto");
- assertEquals(4L, provider.rowCount(stageTableName));
- assertEquals(0L, rowCount());
+ assertEquals(0L, provider.rowCount(stageTableName));
+ assertEquals(4L, rowCount());
+ assertRowInCities(1, "USA", "San Francisco");
+ assertRowInCities(2, "USA", "Sunnyvale");
+ assertRowInCities(3, "Czech Republic", "Brno");
+ assertRowInCities(4, "USA", "Palo Alto");
// Clean up testing table
provider.dropTable(stageTableName);
[35/52] [abbrv] git commit: SQOOP-1481: SQOOP2: Document the public
apis and end-end design for the SQ2 Connector
Posted by ab...@apache.org.
SQOOP-1481: SQOOP2: Document the public apis and end-end design for the SQ2 Connector
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b6000365
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b6000365
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b6000365
Branch: refs/heads/SQOOP-1367
Commit: b6000365736d7b03124417f6b8731d8760011432
Parents: 4c964a9
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Oct 9 08:16:52 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 18:22:51 2014 -0700
----------------------------------------------------------------------
docs/src/site/sphinx/CommandLineClient.rst | 6 +-
docs/src/site/sphinx/ConnectorDevelopment.rst | 154 ++++++++++---------
docs/src/site/sphinx/RESTAPI.rst | 54 +------
.../sqoop/connector/spi/SqoopConnector.java | 2 +-
4 files changed, 95 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b6000365/docs/src/site/sphinx/CommandLineClient.rst
----------------------------------------------------------------------
diff --git a/docs/src/site/sphinx/CommandLineClient.rst b/docs/src/site/sphinx/CommandLineClient.rst
index fb7f780..7cc6449 100644
--- a/docs/src/site/sphinx/CommandLineClient.rst
+++ b/docs/src/site/sphinx/CommandLineClient.rst
@@ -329,14 +329,14 @@ Create new job object.
+------------------------+------------------------------------------------------------------+
| Argument | Description |
+========================+==================================================================+
-| ``-x``, ``--xid <x>`` | Create new job object for connection with id ``<x>`` |
+| ``-f``, ``--from <x>`` | Create new job object with a FROM connection with id ``<x>`` |
+------------------------+------------------------------------------------------------------+
-| ``-t``, ``--type <t>`` | Create new job object with type ``<t>`` (``import``, ``export``) |
+| ``-t``, ``--to <t>`` | Create new job object with a TO connection with id ``<x>`` |
+------------------------+------------------------------------------------------------------+
Example: ::
- create job --xid 1
+ create job --from 1 --to 2
Update Command
--------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b6000365/docs/src/site/sphinx/ConnectorDevelopment.rst
----------------------------------------------------------------------
diff --git a/docs/src/site/sphinx/ConnectorDevelopment.rst b/docs/src/site/sphinx/ConnectorDevelopment.rst
index ae4f721..d700e4c 100644
--- a/docs/src/site/sphinx/ConnectorDevelopment.rst
+++ b/docs/src/site/sphinx/ConnectorDevelopment.rst
@@ -26,17 +26,15 @@ using the code of built-in connector ( ``GenericJdbcConnector`` ) as example.
What is Connector?
++++++++++++++++++
-Connector provides interaction with external databases.
-Connector reads data from databases for import,
-and write data to databases for export.
-Interaction with Hadoop is taken cared by common modules of Sqoop 2 framework.
+The connector provides the facilities to interact with external data sources.
+The connector can read from, or write to, a data source.
When do we add a new connector?
===============================
You add a new connector when you need to extract data from a new data source, or load
data to a new target.
In addition to the connector API, Sqoop 2 also has an engine interface.
-At the moment the only engine is MapReduce,but we may support additional engines in the future.
+At the moment the only engine is MapReduce, but we may support additional engines in the future.
Since many parallel execution engines are capable of reading/writing data
there may be a question of whether support for specific data stores should be done
through a new connector or new engine.
@@ -51,57 +49,73 @@ Connector Implementation
The ``SqoopConnector`` class defines functionality
which must be provided by Connectors.
-Each Connector must extends ``SqoopConnector`` and overrides methods shown below.
+Each Connector must extend ``SqoopConnector`` and override the methods shown below.
::
public abstract String getVersion();
public abstract ResourceBundle getBundle(Locale locale);
- public abstract Class getConnectionConfigurationClass();
- public abstract Class getJobConfigurationClass(MJob.Type jobType);
- public abstract Importer getImporter();
- public abstract Exporter getExporter();
+ public abstract Class getLinkConfigurationClass();
+ public abstract Class getJobConfigurationClass(Direction direction);
+ public abstract From getFrom();
+ public abstract To getTo();
public abstract Validator getValidator();
public abstract MetadataUpgrader getMetadataUpgrader();
-The ``getImporter`` method returns Importer_ instance
-which is a placeholder for the modules needed for import.
+Connectors can optionally override the following methods:
+::
+
+ public List<Direction> getSupportedDirections();
+
-The ``getExporter`` method returns Exporter_ instance
-which is a placeholder for the modules needed for export.
+The ``getFrom`` method returns From_ instance
+which is a placeholder for the modules needed to read from a data source.
+
+The ``getTo`` method returns Exporter_ instance
+which is a placeholder for the modules needed to write to a data source.
Methods such as ``getBundle`` , ``getConnectionConfigurationClass`` ,
``getJobConfigurationClass`` and ``getValidator``
are concerned to `Connector configurations`_ .
+The ``getSupportedDirections`` method returns a list of directions
+that a connector supports. This should be some subset of:
+::
+
+ public List<Direction> getSupportedDirections() {
+ return Arrays.asList(new Direction[]{
+ Direction.FROM,
+ Direction.TO
+ });
+ }
+
-Importer
-========
+From
+====
-Connector's ``getImporter`` method returns ``Importer`` instance
-which is a placeholder for the modules needed for import
-such as Partitioner_ and Extractor_ .
-Built-in ``GenericJdbcConnector`` defines ``Importer`` like this.
+The connector's ``getFrom`` method returns ``From`` instance
+which is a placeholder for the modules needed for reading
+from a data source. Modules such as Partitioner_ and Extractor_ .
+The built-in ``GenericJdbcConnector`` defines ``From`` like this.
::
- private static final Importer IMPORTER = new Importer(
- GenericJdbcImportInitializer.class,
- GenericJdbcImportPartitioner.class,
- GenericJdbcImportExtractor.class,
- GenericJdbcImportDestroyer.class);
+ private static final From FROM = new From(
+ GenericJdbcFromInitializer.class,
+ GenericJdbcPartitioner.class,
+ GenericJdbcExtractor.class,
+ GenericJdbcFromDestroyer.class);
...
@Override
- public Importer getImporter() {
- return IMPORTER;
+ public From getFrom() {
+ return FROM;
}
Extractor
---------
-Extractor (E for ETL) extracts data from external database and
-writes it to Sqoop framework for import.
+Extractor (E for ETL) extracts data from external database.
Extractor must overrides ``extract`` method.
::
@@ -114,7 +128,13 @@ Extractor must overrides ``extract`` method.
The ``extract`` method extracts data from database in some way and
writes it to ``DataWriter`` (provided by context) as `Intermediate representation`_ .
-Extractor must iterates in the ``extract`` method until the data from database exhausts.
+Extractors use Writer's provided by the ExtractorContext to send a record through the
+framework.
+::
+
+ context.getDataWriter().writeArrayRecord(array);
+
+The extractor must iterate through the entire dataset in the ``extract`` method.
::
while (resultSet.next()) {
@@ -127,9 +147,9 @@ Extractor must iterates in the ``extract`` method until the data from database e
Partitioner
-----------
-Partitioner creates ``Partition`` instances based on configurations.
+The Partitioner creates ``Partition`` instances based on configurations.
The number of ``Partition`` instances is decided
-based on the value users specified as the numbers of ectractors
+based on the value users specified as the numbers of extractors
in job configuration.
``Partition`` instances are passed to Extractor_ as the argument of ``extract`` method.
@@ -157,35 +177,35 @@ for doing preparation such as adding dependent jar files.
Destroyer is instantiated after MapReduce job is finished for clean up.
-Exporter
-========
+To
+==
-Connector's ``getExporter`` method returns ``Exporter`` instance
-which is a placeholder for the modules needed for export
-such as Loader_ .
-Built-in ``GenericJdbcConnector`` defines ``Exporter`` like this.
+The Connector's ``getTo`` method returns a ``To`` instance
+which is a placeholder for the modules needed for writing
+to a data source such as Loader_ .
+The built-in ``GenericJdbcConnector`` defines ``To`` like this.
::
- private static final Exporter EXPORTER = new Exporter(
- GenericJdbcExportInitializer.class,
- GenericJdbcExportLoader.class,
- GenericJdbcExportDestroyer.class);
+ private static final To TO = new To(
+ GenericJdbcToInitializer.class,
+ GenericJdbcLoader.class,
+ GenericJdbcToDestroyer.class);
...
@Override
- public Exporter getExporter() {
- return EXPORTER;
+ public To getTo() {
+ return TO;
}
Loader
------
-Loader (L for ETL) receives data from Sqoop framework and
-loads it to external database.
+A loader (L for ETL) receives data from the Sqoop framework and
+loads it to an external database.
-Loader must overrides ``load`` method.
+Loaders must overrides ``load`` method.
::
public abstract void load(LoaderContext context,
@@ -195,7 +215,7 @@ Loader must overrides ``load`` method.
The ``load`` method reads data from ``DataReader`` (provided by context)
in `Intermediate representation`_ and loads it to database in some way.
-Loader must iterates in the ``load`` method until the data from ``DataReader`` exhausts.
+Loader must iterate in the ``load`` method until the data from ``DataReader`` is exhausted.
::
while ((array = context.getDataReader().readArrayRecord()) != null) {
@@ -218,7 +238,7 @@ Connector Configurations
Connector specifications
========================
-Framework of the Sqoop loads definitions of connectors
+Sqoop loads definitions of connectors
from the file named ``sqoopconnector.properties``
which each connector implementation provides.
::
@@ -231,7 +251,7 @@ which each connector implementation provides.
Configurations
==============
-Implementation of ``SqoopConnector`` overrides methods such as
+Implementations of ``SqoopConnector`` overrides methods such as
``getConnectionConfigurationClass`` and ``getJobConfigurationClass``
returning configuration class.
::
@@ -242,12 +262,12 @@ returning configuration class.
}
@Override
- public Class getJobConfigurationClass(MJob.Type jobType) {
- switch (jobType) {
- case IMPORT:
- return ImportJobConfiguration.class;
- case EXPORT:
- return ExportJobConfiguration.class;
+ public Class getJobConfigurationClass(Direction direction) {
+ switch (direction) {
+ case FROM:
+ return FromJobConfiguration.class;
+ case TO:
+ return ToJobConfiguration.class;
default:
return null;
}
@@ -260,7 +280,7 @@ Annotations such as
are provided for defining configurations of each connectors
using these models.
-``ConfigurationClass`` is place holder for ``FormClasses`` .
+``ConfigurationClass`` is a place holder for ``FormClasses`` .
::
@ConfigurationClass
@@ -323,16 +343,12 @@ Validator validates configurations set by users.
Internal of Sqoop2 MapReduce Job
++++++++++++++++++++++++++++++++
-Sqoop 2 provides common MapReduce modules such as ``SqoopMapper`` and ``SqoopReducer``
-for the both of import and export.
-
-- For import, ``Extractor`` provided by connector extracts data from databases,
- and ``Loader`` provided by Sqoop2 loads data into Hadoop.
+Sqoop 2 provides common MapReduce modules such as ``SqoopMapper`` and ``SqoopReducer``.
-- For export, ``Extractor`` provided by Sqoop2 exracts data from Hadoop,
- and ``Loader`` provided by connector loads data into databases.
+When reading from a data source, the ``Extractor`` provided by the FROM connector extracts data from a database,
+and the ``Loader``, provided by the TO connector, loads data into another database.
-The diagram below describes the initialization phase of IMPORT job.
+The diagram below describes the initialization phase of a job.
``SqoopInputFormat`` create splits using ``Partitioner`` .
::
@@ -351,8 +367,8 @@ The diagram below describes the initialization phase of IMPORT job.
|-------------------------------------------------->|SqoopSplit|
| | | `----+-----'
-The diagram below describes the map phase of IMPORT job.
-``SqoopMapper`` invokes extractor's ``extract`` method.
+The diagram below describes the map phase of a job.
+``SqoopMapper`` invokes FROM connector's extractor's ``extract`` method.
::
,-----------.
@@ -378,8 +394,8 @@ The diagram below describes the map phase of IMPORT job.
| | | context.write
| | |-------------------------->
-The diagram below decribes the reduce phase of EXPORT job.
-``OutputFormat`` invokes loader's ``load`` method (via ``SqoopOutputFormatLoadExecutor`` ).
+The diagram below decribes the reduce phase of a job.
+``OutputFormat`` invokes TO connector's loader's ``load`` method (via ``SqoopOutputFormatLoadExecutor`` ).
::
,-------. ,---------------------.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b6000365/docs/src/site/sphinx/RESTAPI.rst
----------------------------------------------------------------------
diff --git a/docs/src/site/sphinx/RESTAPI.rst b/docs/src/site/sphinx/RESTAPI.rst
index 04908e8..0117cb6 100644
--- a/docs/src/site/sphinx/RESTAPI.rst
+++ b/docs/src/site/sphinx/RESTAPI.rst
@@ -418,7 +418,7 @@ So far, the resource contains only explanations for fields of forms. For example
"name":"generic-jdbc-connector",
"class":"org.apache.sqoop.connector.jdbc.GenericJdbcConnector",
"job-forms":{
- "IMPORT":[
+ "FROM":[
{
"id":2,
"inputs":[
@@ -475,7 +475,7 @@ So far, the resource contains only explanations for fields of forms. For example
"type":"CONNECTION"
}
],
- "EXPORT":[
+ "TO":[
{
"id":3,
"inputs":[
@@ -624,35 +624,7 @@ the id of the form to track these parameter inputs.
},
"framework-version":"1",
"job-forms":{
- "IMPORT":[
- {
- "id":5,
- "inputs":[
- {
- "id":18,
- "values":"HDFS",
- "name":"output.storageType",
- "type":"ENUM",
- "sensitive":false
- },
- {
- "id":19,
- "values":"TEXT_FILE,SEQUENCE_FILE",
- "name":"output.outputFormat",
- "type":"ENUM",
- "sensitive":false
- },
- {
- "id":20,
- "name":"output.outputDirectory",
- "type":"STRING",
- "size":255,
- "sensitive":false
- }
- ],
- "name":"output",
- "type":"CONNECTION"
- },
+ "FROM":[
{
"id":6,
"inputs":[
@@ -673,23 +645,9 @@ the id of the form to track these parameter inputs.
"type":"CONNECTION"
}
],
- "EXPORT":[
+ "TO":[
{
- "id":7,
- "inputs":[
- {
- "id":23,
- "name":"input.inputDirectory",
- "type":"STRING",
- "size":255,
- "sensitive":false
- }
- ],
- "name":"input",
- "type":"CONNECTION"
- },
- {
- "id":8,
+ "id":1,
"inputs":[
{
"id":24,
@@ -1272,7 +1230,7 @@ Provide the id of the job in the url [jid] part. If you provide ``all`` in the [
}
],
"connector-id":1,
- "type":"IMPORT",
+ "type":"FROM",
"framework":[
{
"id":5,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b6000365/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
index 849f091..4272470 100644
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
@@ -65,7 +65,7 @@ public abstract class SqoopConnector {
/**
* @return Get job configuration group class per direction type or null if not supported
*/
- public abstract Class getJobConfigurationClass(Direction jobType);
+ public abstract Class getJobConfigurationClass(Direction direction);
/**
* @return an <tt>From</tt> that provides classes for performing import.
[06/52] [abbrv] git commit: SQOOP-1465: Sqoop2: Validations: Add
support for multiple validation messages to Model classes
Posted by ab...@apache.org.
SQOOP-1465: Sqoop2: Validations: Add support for multiple validation messages to Model classes
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f89368e2
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f89368e2
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f89368e2
Branch: refs/heads/SQOOP-1367
Commit: f89368e2a0cc6236b2c50dfe544134f9d55208b9
Parents: e71dd75
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Mon Sep 8 23:16:19 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:17 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/sqoop/model/FormUtils.java | 10 +-
.../apache/sqoop/model/MValidatedElement.java | 107 +++++++------------
.../org/apache/sqoop/validation/Status.java | 16 ++-
.../org/apache/sqoop/model/TestFormUtils.java | 31 ------
.../sqoop/model/TestMValidatedElement.java | 36 +++----
.../apache/sqoop/shell/utils/FormDisplayer.java | 5 +-
.../apache/sqoop/shell/utils/FormFiller.java | 17 ++-
7 files changed, 88 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89368e2/common/src/main/java/org/apache/sqoop/model/FormUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/FormUtils.java b/common/src/main/java/org/apache/sqoop/model/FormUtils.java
index bfd97d5..586e0fe 100644
--- a/common/src/main/java/org/apache/sqoop/model/FormUtils.java
+++ b/common/src/main/java/org/apache/sqoop/model/FormUtils.java
@@ -314,9 +314,9 @@ public class FormUtils {
if (messages.containsKey(name)) {
Validation.Message message = messages.get(name);
- element.setValidationMessage(message.getStatus(), message.getMessage());
+ element.addValidationMessage(new Message(message.getStatus(), message.getMessage()));
} else {
- element.setValidationMessage(Status.getDefault(), null);
+ element.addValidationMessage(new Message(Status.getDefault(), null));
}
}
@@ -349,11 +349,9 @@ public class FormUtils {
List<Message> messages = result.getMessages().get(element.getName());
if(messages != null) {
- // TODO(SQOOP-1465) Add support for multiple messages (showing only the first one for now)
- Message message = messages.get(0);
- element.setValidationMessage(message.getStatus(), message.getMessage());
+ element.setValidationMessages(messages);
} else {
- element.setValidationMessage(Status.getDefault(), null);
+ element.resetValidationMessages();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89368e2/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java b/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java
index ab5473c..a50c815 100644
--- a/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java
+++ b/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java
@@ -17,122 +17,87 @@
*/
package org.apache.sqoop.model;
+import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
+import java.util.LinkedList;
+import java.util.List;
+
/**
- * Element that can be validated for correctness.
- *
- * Two severity levels are supported at the moment - warning and error.
- *
- * Warning:
- * Warning is something suspicious, potentially wrong but something that
- * can be ignored. For example in case of JDBC URL element, warning would
- * be if specified host is not responding - it's warning because specified
- * URL might be wrong. However at the same time URL might be right as only
- * target host might be down.
- *
- * Error:
- * Error represents unacceptable element content. For example in case of JDBC
- * URL path, error would be empty element or element containing invalid URL.
+ * Element that can have associated validation messages (0..N).
*/
public abstract class MValidatedElement extends MNamedElement {
/**
- * Validation message.
- *
- * One element can have only one message regardless of the type.
+ * Validation messages.
*/
- private String validationMessage;
+ private List<Message> validationMessages;
/**
- * Severity of the message.
+ * The worst status of all validation messages.
*/
private Status validationStatus;
public MValidatedElement(String name) {
super(name);
- // Everything is fine by default
- this.validationStatus = Status.getDefault();
+ resetValidationMessages();
}
public MValidatedElement(MValidatedElement other) {
super(other);
- this.validationMessage = other.validationMessage;
this.validationStatus = other.validationStatus;
+ this.validationMessages.addAll(other.validationMessages);
}
/**
- * Set validation message and given severity.
- *
- * @param status Message validation status
- * @param msg Message itself
- */
- public void setValidationMessage(Status status, String msg) {
- this.validationMessage = msg;
- this.validationStatus = status;
- }
-
- /**
- * Return validation message for given severity.
+ * Reset this validated element back to default state.
*
- * Return either associated message for given severity or null in case
- * that there is no message with given severity.
- *
- * @param status Message validation status
+ * Will remove all associated messages and validation status.
*/
- public String getValidationMessage(Status status) {
- return (validationStatus.equals(status)) ? validationMessage : null;
+ public void resetValidationMessages() {
+ this.validationStatus = Status.getDefault();
+ this.validationMessages = new LinkedList<Message>();
}
/**
- * Return validation message.
+ * Set validation messages (override anything that has been set before).
*
- * Return current validation message.
- */
- public String getValidationMessage() {
- return validationMessage;
- }
-
- /**
- * Return message validation status.
+ * @param msg Message itself
*/
- public Status getValidationStatus() {
- return validationStatus;
+ public void addValidationMessage(Message msg) {
+ this.validationMessages.add(msg);
+ this.validationStatus = Status.getWorstStatus(this.validationStatus, msg.getStatus());
}
/**
- * Set error message for this element.
+ * Override all previously existing validation messages.
*
- * @param errMsg Error message
+ * @param messages
*/
- public void setErrorMessage(String errMsg) {
- setValidationMessage(Status.UNACCEPTABLE, errMsg);
- }
+ public void setValidationMessages(List<Message> messages) {
+ this.validationMessages = messages;
+ this.validationStatus = Status.getDefault();
- /**
- * Return error message associated with this element.
- *
- * @return Error message
- */
- public String getErrorMessage() {
- return getValidationMessage(Status.UNACCEPTABLE);
+ for(Message message : messages ) {
+ this.validationStatus = Status.getWorstStatus(this.validationStatus, message.getStatus());
+ }
}
/**
- * Set warning message for this element.
+ * Return validation message for given severity.
*
- * @param warnMsg Warning message
+ * Return either associated message for given severity or null in case
+ * that there is no message with given severity.
*/
- public void setWarningMessage(String warnMsg) {
- setValidationMessage(Status.ACCEPTABLE, warnMsg);
+ public List<Message> getValidationMessages() {
+ return this.validationMessages;
}
/**
- * Retrieve warning message associated with this element.
- * @return
+ * Return message validation status.
*/
- public String getWarningMessage() {
- return getValidationMessage(Status.ACCEPTABLE);
+ public Status getValidationStatus() {
+ return validationStatus;
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89368e2/common/src/main/java/org/apache/sqoop/validation/Status.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/Status.java b/common/src/main/java/org/apache/sqoop/validation/Status.java
index ae89e59..5a546bf 100644
--- a/common/src/main/java/org/apache/sqoop/validation/Status.java
+++ b/common/src/main/java/org/apache/sqoop/validation/Status.java
@@ -18,7 +18,21 @@
package org.apache.sqoop.validation;
/**
- * Status modes of a validation process.
+ * Severity of validation message.
+ *
+ * Fine:
+ * Everything is correct (default state).
+ *
+ * Warning:
+ * Warning is something suspicious, potentially wrong but something that
+ * can be ignored. For example in case of JDBC URL element, warning would
+ * be if specified host is not responding - it's warning because specified
+ * URL might be wrong. However at the same time URL might be right as only
+ * target host might be down.
+ *
+ * Error:
+ * Error represents unacceptable element content. For example in case of JDBC
+ * URL path, error would be empty element or element containing invalid URL.
*
* TODO: This should really be renamed to "severity"
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89368e2/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java b/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
index 41dddb7..9fa2b6b 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
@@ -129,23 +129,6 @@ public class TestFormUtils extends TestCase {
assertNull(config.bForm.b2);
}
- public void testApplyValidation() {
- Validation validation = getValidation();
- List<MForm> forms = getForms();
-
- FormUtils.applyValidation(forms, validation);
-
- assertEquals(Status.ACCEPTABLE,
- forms.get(0).getInputs().get(0).getValidationStatus());
- assertEquals("e1",
- forms.get(0).getInputs().get(0).getValidationMessage());
-
- assertEquals(Status.UNACCEPTABLE,
- forms.get(0).getInputs().get(1).getValidationStatus());
- assertEquals("e2",
- forms.get(0).getInputs().get(1).getValidationMessage());
- }
-
public void testJson() {
Config config = new Config();
config.aForm.a1 = "A";
@@ -179,20 +162,6 @@ public class TestFormUtils extends TestCase {
assertEquals(Enumeration.X, targetConfig.cForm.enumeration);
}
- protected Validation getValidation() {
- Map<Validation.FormInput, Validation.Message> messages
- = new HashMap<Validation.FormInput, Validation.Message>();
-
- messages.put(
- new Validation.FormInput("aForm", "a1"),
- new Validation.Message(Status.ACCEPTABLE, "e1"));
- messages.put(
- new Validation.FormInput("aForm", "a2"),
- new Validation.Message(Status.UNACCEPTABLE, "e2"));
-
- return new Validation(Status.UNACCEPTABLE, messages);
- }
-
/**
* Form structure that corresponds to Config class declared below
* @return Form structure
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89368e2/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java b/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java
index 3fd5a95..cf34983 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMValidatedElement.java
@@ -19,6 +19,7 @@ package org.apache.sqoop.model;
import static org.junit.Assert.*;
+import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
import org.junit.Test;
@@ -41,30 +42,27 @@ public class TestMValidatedElement {
* Test for validation message and status
*/
@Test
- public void testValidationMessageStatus() {
+ public void testVarious() {
MValidatedElement input = new MIntegerInput("input", false);
+
// Default status
assertEquals(Status.FINE, input.getValidationStatus());
- // Set status and user message
- input.setValidationMessage(Status.ACCEPTABLE, "MY_MESSAGE");
+
+ // Add a message
+ input.addValidationMessage(new Message(Status.ACCEPTABLE, "MY_MESSAGE"));
assertEquals(Status.ACCEPTABLE, input.getValidationStatus());
- assertEquals("MY_MESSAGE", input.getValidationMessage());
- // Check for null if status does not equal
- assertNull(input.getValidationMessage(Status.FINE));
- assertNull(input.getErrorMessage());
- assertNotNull(input.getWarningMessage());
+ assertEquals(1, input.getValidationMessages().size());
+ assertEquals("MY_MESSAGE", input.getValidationMessages().get(0).getMessage());
+
+ // Reset
+ input.resetValidationMessages();
+ assertEquals(Status.FINE, input.getValidationStatus());
+ assertEquals(0, input.getValidationMessages().size());
+
// Set unacceptable status
- input.setValidationMessage(Status.UNACCEPTABLE, "MY_MESSAGE");
- assertNotNull(input.getErrorMessage());
- assertEquals("MY_MESSAGE", input.getErrorMessage());
- assertNull(input.getWarningMessage());
- // Set warning
- input.setWarningMessage("WARN");
- assertEquals(Status.ACCEPTABLE, input.getValidationStatus());
- assertEquals("WARN", input.getValidationMessage());
- // Unacceptable method
- input.setErrorMessage("ERROR");
+ input.addValidationMessage(new Message(Status.UNACCEPTABLE, "MY_MESSAGE"));
assertEquals(Status.UNACCEPTABLE, input.getValidationStatus());
- assertEquals("ERROR", input.getValidationMessage());
+ assertEquals(1, input.getValidationMessages().size());
+ assertEquals("MY_MESSAGE", input.getValidationMessages().get(0).getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89368e2/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
index e04d7b5..bbad183 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/FormDisplayer.java
@@ -33,6 +33,7 @@ import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
import java.util.ArrayList;
@@ -163,7 +164,9 @@ public final class FormDisplayer {
print("\n@|yellow %s|@\n", resourceString(Constants.RES_FORMDISPLAYER_FORM_WARNING));
showMessage = false;
}
- FormFiller.warningMessage(form.getValidationMessage());
+ for(Message message : form.getValidationMessages()) {
+ FormFiller.warningMessage(message.getMessage());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89368e2/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
index 9d63c6e..bbf7af7 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
@@ -33,6 +33,8 @@ import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MNamedElement;
import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.model.MValidatedElement;
+import org.apache.sqoop.validation.Message;
+import org.apache.sqoop.validation.Status;
import java.io.IOException;
import java.util.List;
@@ -849,19 +851,24 @@ public final class FormFiller {
* @param element Validated element
*/
public static void printValidationMessage(MValidatedElement element, boolean includeInputPrefix) {
- switch (element.getValidationStatus()) {
+ if(element.getValidationStatus() == Status.getDefault()) {
+ return;
+ }
+
+ for(Message message : element.getValidationMessages())
+ switch (message.getStatus()) {
case UNACCEPTABLE:
if (includeInputPrefix) {
- errorMessage(element, element.getValidationMessage());
+ errorMessage(element, message.getMessage());
} else {
- errorMessage(element.getValidationMessage());
+ errorMessage(message.getMessage());
}
break;
case ACCEPTABLE:
if (includeInputPrefix) {
- warningMessage(element, element.getValidationMessage());
+ warningMessage(element, message.getMessage());
} else {
- warningMessage(element.getValidationMessage());
+ warningMessage(message.getMessage());
}
break;
default:
[39/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
index ad42901..951d9b4 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
@@ -48,17 +48,17 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* </pre>
* </p>
* <p>
- * <strong>SQ_FORM</strong>: Form details.
+ * <strong>SQ_CONFIG</strong>: Config details.
* <pre>
* +----------------------------------+
- * | SQ_FORM |
+ * | SQ_CONFIG |
* +----------------------------------+
- * | SQF_ID: BIGINT PK AUTO-GEN |
- * | SQF_CONNECTOR: BIGINT | FK SQ_CONNECTOR(SQC_ID),NULL for framework
- * | SQF_DIRECTION: VARCHAR(32) | "FROM"|"TO"|NULL
- * | SQF_NAME: VARCHAR(64) |
- * | SQF_TYPE: VARCHAR(32) | "CONNECTION"|"JOB"
- * | SQF_INDEX: SMALLINT |
+ * | SQ_CFG_ID: BIGINT PK AUTO-GEN |
+ * | SQ_CFG_OWNER: BIGINT | FK SQ_CFG_OWNER(SQC_ID),NULL for driver
+ * | SQ_CFG_DIRECTION: VARCHAR(32) | "FROM"|"TO"|NULL
+ * | SQ_CFG_NAME: VARCHAR(64) |
+ * | SQ_CFG_TYPE: VARCHAR(32) | "LINK"|"JOB"
+ * | SQ_CFG_INDEX: SMALLINT |
* +----------------------------------+
* </pre>
* </p>
@@ -70,7 +70,7 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* +----------------------------+
* | SQI_ID: BIGINT PK AUTO-GEN |
* | SQI_NAME: VARCHAR(64) |
- * | SQI_FORM: BIGINT | FK SQ_FORM(SQF_ID)
+ * | SQI_CONFIG: BIGINT | FK SQ_CONFIG(SQ_CFG_ID)
* | SQI_INDEX: SMALLINT |
* | SQI_TYPE: VARCHAR(32) | "STRING"|"MAP"
* | SQI_STRMASK: BOOLEAN |
@@ -80,19 +80,19 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* </pre>
* </p>
* <p>
- * <strong>SQ_CONNECTION</strong>: Stored connections
+ * <strong>SQ_LINK</strong>: Stored connections
* <pre>
* +--------------------------------+
- * | SQ_CONNECTION |
+ * | SQ_LINK |
* +--------------------------------+
- * | SQN_ID: BIGINT PK AUTO-GEN |
- * | SQN_NAME: VARCHAR(64) |
- * | SQN_CONNECTOR: BIGINT | FK SQ_CONNECTOR(SQC_ID)
- * | SQN_CREATION_USER: VARCHAR(32) |
- * | SQN_CREATION_DATE: TIMESTAMP |
- * | SQN_UPDATE_USER: VARCHAR(32) |
- * | SQN_UPDATE_DATE: TIMESTAMP |
- * | SQN_ENABLED: BOOLEAN |
+ * | SQ_LNK_ID: BIGINT PK AUTO-GEN |
+ * | SQ_LNK_NAME: VARCHAR(64) |
+ * | SQ_LNK_CONNECTOR: BIGINT | FK SQ_CONNECTOR(SQC_ID)
+ * | SQ_LNK_CREATION_USER: VARCHAR(32) |
+ * | SQ_LNK_CREATION_DATE: TIMESTAMP |
+ * | SQ_LNK_UPDATE_USER: VARCHAR(32) |
+ * | SQ_LNK_UPDATE_DATE: TIMESTAMP |
+ * | SQ_LNK_ENABLED: BOOLEAN |
* +--------------------------------+
* </pre>
* </p>
@@ -104,8 +104,8 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* +--------------------------------+
* | SQB_ID: BIGINT PK AUTO-GEN |
* | SQB_NAME: VARCHAR(64) |
- * | SQB_FROM_CONNECTION: BIGINT | FK SQ_CONNECTION(SQN_ID)
- * | SQB_TO_CONNECTION: BIGINT | FK SQ_CONNECTION(SQN_ID)
+ * | SQB_FROM_LINK: BIGINT | FK SQ_LINK(SQ_LNK_ID)
+ * | SQB_TO_LINK: BIGINT | FK SQ_LINK(SQ_LNK_ID)
* | SQB_CREATION_USER: VARCHAR(32) |
* | SQB_CREATION_DATE: TIMESTAMP |
* | SQB_UPDATE_USER: VARCHAR(32) |
@@ -115,14 +115,14 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* </pre>
* </p>
* <p>
- * <strong>SQ_CONNECTION_INPUT</strong>: N:M relationship connection and input
+ * <strong>SQ_LINK_INPUT</strong>: N:M relationship link and input
* <pre>
* +----------------------------+
- * | SQ_CONNECTION_INPUT |
+ * | SQ_LINK_INPUT |
* +----------------------------+
- * | SQNI_CONNECTION: BIGINT PK | FK SQ_CONNECTION(SQN_ID)
- * | SQNI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID)
- * | SQNI_VALUE: LONG VARCHAR |
+ * | SQ_LNKI_LINK: BIGINT PK | FK SQ_LINK(SQ_LNK_ID)
+ * | SQ_LNKI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID)
+ * | SQ_LNKI_VALUE: LONG VARCHAR |
* +----------------------------+
* </pre>
* </p>
@@ -221,17 +221,17 @@ public final class DerbySchemaQuery {
+ COLUMN_SQC_VERSION + " VARCHAR(64) "
+ ")";
- // DDL: Create table SQ_FORM
- public static final String QUERY_CREATE_TABLE_SQ_FORM =
- "CREATE TABLE " + TABLE_SQ_FORM + " ("
- + COLUMN_SQF_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
- + COLUMN_SQF_CONNECTOR + " BIGINT, "
- + COLUMN_SQF_OPERATION + " VARCHAR(32), "
- + COLUMN_SQF_NAME + " VARCHAR(64), "
- + COLUMN_SQF_TYPE + " VARCHAR(32), "
- + COLUMN_SQF_INDEX + " SMALLINT, "
- + "CONSTRAINT " + CONSTRAINT_SQF_SQC + " "
- + "FOREIGN KEY (" + COLUMN_SQF_CONNECTOR + ") "
+ // DDL: Create table SQ_CONFIG ( It stores the configs defined by every connector), if connector is null then it is driver config
+ public static final String QUERY_CREATE_TABLE_SQ_CONFIG =
+ "CREATE TABLE " + TABLE_SQ_CONFIG + " ("
+ + COLUMN_SQ_CFG_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
+ + COLUMN_SQ_CFG_OWNER + " BIGINT, "
+ + COLUMN_SQ_CFG_OPERATION + " VARCHAR(32), "
+ + COLUMN_SQ_CFG_NAME + " VARCHAR(64), "
+ + COLUMN_SQ_CFG_TYPE + " VARCHAR(32), "
+ + COLUMN_SQ_CFG_INDEX + " SMALLINT, "
+ + "CONSTRAINT " + CONSTRAINT_SQ_CFG_SQC + " "
+ + "FOREIGN KEY (" + COLUMN_SQ_CFG_OWNER + ") "
+ "REFERENCES " + TABLE_SQ_CONNECTOR + " (" + COLUMN_SQC_ID + ")"
+ ")";
@@ -240,60 +240,60 @@ public final class DerbySchemaQuery {
"CREATE TABLE " + TABLE_SQ_INPUT + " ("
+ COLUMN_SQI_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
+ COLUMN_SQI_NAME + " VARCHAR(64), "
- + COLUMN_SQI_FORM + " BIGINT, "
+ + COLUMN_SQI_CONFIG + " BIGINT, "
+ COLUMN_SQI_INDEX + " SMALLINT, "
+ COLUMN_SQI_TYPE + " VARCHAR(32), "
+ COLUMN_SQI_STRMASK + " BOOLEAN, "
+ COLUMN_SQI_STRLENGTH + " SMALLINT, "
+ COLUMN_SQI_ENUMVALS + " VARCHAR(100),"
- + "CONSTRAINT " + CONSTRAINT_SQI_SQF + " "
- + "FOREIGN KEY (" + COLUMN_SQI_FORM + ") "
- + "REFERENCES " + TABLE_SQ_FORM + " (" + COLUMN_SQF_ID + ")"
+ + "CONSTRAINT " + CONSTRAINT_SQI_SQ_CFG + " "
+ + "FOREIGN KEY (" + COLUMN_SQI_CONFIG + ") "
+ + "REFERENCES " + TABLE_SQ_CONFIG + " (" + COLUMN_SQ_CFG_ID + ")"
+ ")";
- // DDL: Create table SQ_CONNECTION
- public static final String QUERY_CREATE_TABLE_SQ_CONNECTION =
- "CREATE TABLE " + TABLE_SQ_CONNECTION + " ("
- + COLUMN_SQN_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
- + COLUMN_SQN_CONNECTOR + " BIGINT, "
- + COLUMN_SQN_NAME + " VARCHAR(32),"
- + COLUMN_SQN_CREATION_DATE + " TIMESTAMP,"
- + COLUMN_SQN_UPDATE_DATE + " TIMESTAMP,"
- + "CONSTRAINT " + CONSTRAINT_SQN_SQC + " "
- + "FOREIGN KEY(" + COLUMN_SQN_CONNECTOR + ") "
+ // DDL: Create table SQ_LINK
+ public static final String QUERY_CREATE_TABLE_SQ_LINK =
+ "CREATE TABLE " + TABLE_SQ_LINK + " ("
+ + COLUMN_SQ_LNK_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
+ + COLUMN_SQ_LNK_CONNECTOR + " BIGINT, "
+ + COLUMN_SQ_LNK_NAME + " VARCHAR(32),"
+ + COLUMN_SQ_LNK_CREATION_DATE + " TIMESTAMP,"
+ + COLUMN_SQ_LNK_UPDATE_DATE + " TIMESTAMP,"
+ + "CONSTRAINT " + CONSTRAINT_SQ_LNK_SQC + " "
+ + "FOREIGN KEY(" + COLUMN_SQ_LNK_CONNECTOR + ") "
+ " REFERENCES " + TABLE_SQ_CONNECTOR + " (" + COLUMN_SQC_ID + ")"
+ ")";
- // DDL: Add enabled column to table SQ_CONNECTION
- public static final String QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_ENABLED =
- "ALTER TABLE " + TABLE_SQ_CONNECTION + " ADD "
- + COLUMN_SQN_ENABLED + " BOOLEAN "
+ // DDL: Add enabled column to table SQ_LINK
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_ENABLED =
+ "ALTER TABLE " + TABLE_SQ_LINK + " ADD "
+ + COLUMN_SQ_LNK_ENABLED + " BOOLEAN "
+ "DEFAULT TRUE";
- // DDL: Add creation_user column to table SQ_CONNECTION
- public static final String QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_CREATION_USER =
- "ALTER TABLE " + TABLE_SQ_CONNECTION + " ADD "
- + COLUMN_SQN_CREATION_USER + " VARCHAR(32) "
+ // DDL: Add creation_user column to table SQ_LINK
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_CREATION_USER =
+ "ALTER TABLE " + TABLE_SQ_LINK + " ADD "
+ + COLUMN_SQ_LNK_CREATION_USER + " VARCHAR(32) "
+ "DEFAULT NULL";
- // DDL: Add update_user column to table SQ_CONNECTION
- public static final String QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_UPDATE_USER =
- "ALTER TABLE " + TABLE_SQ_CONNECTION + " ADD "
- + COLUMN_SQN_UPDATE_USER + " VARCHAR(32) "
+ // DDL: Add update_user column to table SQ_LINK
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_UPDATE_USER =
+ "ALTER TABLE " + TABLE_SQ_LINK + " ADD "
+ + COLUMN_SQ_LNK_UPDATE_USER + " VARCHAR(32) "
+ "DEFAULT NULL";
// DDL: Create table SQ_JOB
public static final String QUERY_CREATE_TABLE_SQ_JOB =
"CREATE TABLE " + TABLE_SQ_JOB + " ("
+ COLUMN_SQB_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
- + COLUMN_SQB_CONNECTION + " BIGINT, "
+ + COLUMN_SQB_LINK + " BIGINT, "
+ COLUMN_SQB_NAME + " VARCHAR(64), "
+ COLUMN_SQB_TYPE + " VARCHAR(64),"
+ COLUMN_SQB_CREATION_DATE + " TIMESTAMP,"
+ COLUMN_SQB_UPDATE_DATE + " TIMESTAMP,"
- + "CONSTRAINT " + CONSTRAINT_SQB_SQN + " "
- + "FOREIGN KEY(" + COLUMN_SQB_CONNECTION + ") "
- + "REFERENCES " + TABLE_SQ_CONNECTION + " (" + COLUMN_SQN_ID + ")"
+ + "CONSTRAINT " + CONSTRAINT_SQB_SQ_LNK + " "
+ + "FOREIGN KEY(" + COLUMN_SQB_LINK + ") "
+ + "REFERENCES " + TABLE_SQ_LINK + " (" + COLUMN_SQ_LNK_ID + ")"
+ ")";
// DDL: Add enabled column to table SQ_JOB
@@ -314,18 +314,18 @@ public final class DerbySchemaQuery {
+ COLUMN_SQB_UPDATE_USER + " VARCHAR(32) "
+ "DEFAULT NULL";
- // DDL: Create table SQ_CONNECTION_INPUT
- public static final String QUERY_CREATE_TABLE_SQ_CONNECTION_INPUT =
- "CREATE TABLE " + TABLE_SQ_CONNECTION_INPUT + " ("
- + COLUMN_SQNI_CONNECTION + " BIGINT, "
- + COLUMN_SQNI_INPUT + " BIGINT, "
- + COLUMN_SQNI_VALUE + " LONG VARCHAR,"
- + "PRIMARY KEY (" + COLUMN_SQNI_CONNECTION + ", " + COLUMN_SQNI_INPUT + "), "
- + "CONSTRAINT " + CONSTRAINT_SQNI_SQN + " "
- + "FOREIGN KEY (" + COLUMN_SQNI_CONNECTION + ") "
- + "REFERENCES " + TABLE_SQ_CONNECTION + " (" + COLUMN_SQN_ID + "),"
- + "CONSTRAINT " + CONSTRAINT_SQNI_SQI + " "
- + "FOREIGN KEY (" + COLUMN_SQNI_INPUT + ") "
+ // DDL: Create table SQ_LINK_INPUT
+ public static final String QUERY_CREATE_TABLE_SQ_LINK_INPUT =
+ "CREATE TABLE " + TABLE_SQ_LINK_INPUT + " ("
+ + COLUMN_SQ_LNKI_LINK + " BIGINT, "
+ + COLUMN_SQ_LNKI_INPUT + " BIGINT, "
+ + COLUMN_SQ_LNKI_VALUE + " LONG VARCHAR,"
+ + "PRIMARY KEY (" + COLUMN_SQ_LNKI_LINK + ", " + COLUMN_SQ_LNKI_INPUT + "), "
+ + "CONSTRAINT " + CONSTRAINT_SQ_LNKI_SQ_LNK + " "
+ + "FOREIGN KEY (" + COLUMN_SQ_LNKI_LINK + ") "
+ + "REFERENCES " + TABLE_SQ_LINK + " (" + COLUMN_SQ_LNK_ID + "),"
+ + "CONSTRAINT " + CONSTRAINT_SQ_LNKI_SQI + " "
+ + "FOREIGN KEY (" + COLUMN_SQ_LNKI_INPUT + ") "
+ "REFERENCES " + TABLE_SQ_INPUT + " (" + COLUMN_SQI_ID + ")"
+ ")";
@@ -454,38 +454,38 @@ public final class DerbySchemaQuery {
+ COLUMN_SQC_VERSION
+ " FROM " + TABLE_SQ_CONNECTOR;
- // DML: Fetch all forms for a given connector
- public static final String STMT_FETCH_FORM_CONNECTOR =
+ // DML: Fetch all configs for a given connector
+ public static final String STMT_FETCH_CONFIG_CONNECTOR =
"SELECT "
- + COLUMN_SQF_ID + ", "
- + COLUMN_SQF_CONNECTOR + ", "
- + COLUMN_SQF_DIRECTION + ", "
- + COLUMN_SQF_NAME + ", "
- + COLUMN_SQF_TYPE + ", "
- + COLUMN_SQF_INDEX
- + " FROM " + TABLE_SQ_FORM
- + " WHERE " + COLUMN_SQF_CONNECTOR + " = ? "
- + " ORDER BY " + COLUMN_SQF_INDEX;
-
- // DML: Fetch all framework forms
- public static final String STMT_FETCH_FORM_FRAMEWORK =
+ + COLUMN_SQ_CFG_ID + ", "
+ + COLUMN_SQ_CFG_OWNER + ", "
+ + COLUMN_SQ_CFG_DIRECTION + ", "
+ + COLUMN_SQ_CFG_NAME + ", "
+ + COLUMN_SQ_CFG_TYPE + ", "
+ + COLUMN_SQ_CFG_INDEX
+ + " FROM " + TABLE_SQ_CONFIG
+ + " WHERE " + COLUMN_SQ_CFG_OWNER + " = ? "
+ + " ORDER BY " + COLUMN_SQ_CFG_INDEX;
+
+ // DML: Fetch all driver configs
+ public static final String STMT_FETCH_CONFIG_DRIVER =
"SELECT "
- + COLUMN_SQF_ID + ", "
- + COLUMN_SQF_CONNECTOR + ", "
- + COLUMN_SQF_DIRECTION + ", "
- + COLUMN_SQF_NAME + ", "
- + COLUMN_SQF_TYPE + ", "
- + COLUMN_SQF_INDEX
- + " FROM " + TABLE_SQ_FORM
- + " WHERE " + COLUMN_SQF_CONNECTOR + " IS NULL "
- + " ORDER BY " + COLUMN_SQF_TYPE + ", " + COLUMN_SQF_DIRECTION + ", " + COLUMN_SQF_INDEX;
-
- // DML: Fetch inputs for a given form
+ + COLUMN_SQ_CFG_ID + ", "
+ + COLUMN_SQ_CFG_OWNER + ", "
+ + COLUMN_SQ_CFG_DIRECTION + ", "
+ + COLUMN_SQ_CFG_NAME + ", "
+ + COLUMN_SQ_CFG_TYPE + ", "
+ + COLUMN_SQ_CFG_INDEX
+ + " FROM " + TABLE_SQ_CONFIG
+ + " WHERE " + COLUMN_SQ_CFG_OWNER + " IS NULL "
+ + " ORDER BY " + COLUMN_SQ_CFG_TYPE + ", " + COLUMN_SQ_CFG_DIRECTION + ", " + COLUMN_SQ_CFG_INDEX;
+
+ // DML: Fetch inputs for a given config
public static final String STMT_FETCH_INPUT =
"SELECT "
+ COLUMN_SQI_ID + ", "
+ COLUMN_SQI_NAME + ", "
- + COLUMN_SQI_FORM + ", "
+ + COLUMN_SQI_CONFIG + ", "
+ COLUMN_SQI_INDEX + ", "
+ COLUMN_SQI_TYPE + ", "
+ COLUMN_SQI_STRMASK + ", "
@@ -493,27 +493,27 @@ public final class DerbySchemaQuery {
+ COLUMN_SQI_ENUMVALS + ", "
+ "cast(null as varchar(100))"
+ " FROM " + TABLE_SQ_INPUT
- + " WHERE " + COLUMN_SQI_FORM + " = ?"
+ + " WHERE " + COLUMN_SQI_CONFIG + " = ?"
+ " ORDER BY " + COLUMN_SQI_INDEX;
- // DML: Fetch inputs and values for a given connection
- public static final String STMT_FETCH_CONNECTION_INPUT =
+ // DML: Fetch inputs and values for a given link
+ public static final String STMT_FETCH_LINK_INPUT =
"SELECT "
+ COLUMN_SQI_ID + ", "
+ COLUMN_SQI_NAME + ", "
- + COLUMN_SQI_FORM + ", "
+ + COLUMN_SQI_CONFIG + ", "
+ COLUMN_SQI_INDEX + ", "
+ COLUMN_SQI_TYPE + ", "
+ COLUMN_SQI_STRMASK + ", "
+ COLUMN_SQI_STRLENGTH + ","
+ COLUMN_SQI_ENUMVALS + ", "
- + COLUMN_SQNI_VALUE
+ + COLUMN_SQ_LNKI_VALUE
+ " FROM " + TABLE_SQ_INPUT
- + " LEFT OUTER JOIN " + TABLE_SQ_CONNECTION_INPUT
- + " ON " + COLUMN_SQNI_INPUT + " = " + COLUMN_SQI_ID
- + " AND " + COLUMN_SQNI_CONNECTION + " = ?"
- + " WHERE " + COLUMN_SQI_FORM + " = ?"
- + " AND (" + COLUMN_SQNI_CONNECTION + " = ?" + " OR " + COLUMN_SQNI_CONNECTION + " IS NULL)"
+ + " LEFT OUTER JOIN " + TABLE_SQ_LINK_INPUT
+ + " ON " + COLUMN_SQ_LNKI_INPUT + " = " + COLUMN_SQI_ID
+ + " AND " + COLUMN_SQ_LNKI_LINK + " = ?"
+ + " WHERE " + COLUMN_SQI_CONFIG + " = ?"
+ + " AND (" + COLUMN_SQ_LNKI_LINK + " = ?" + " OR " + COLUMN_SQ_LNKI_LINK + " IS NULL)"
+ " ORDER BY " + COLUMN_SQI_INDEX;
// DML: Fetch inputs and values for a given job
@@ -521,7 +521,7 @@ public final class DerbySchemaQuery {
"SELECT "
+ COLUMN_SQI_ID + ", "
+ COLUMN_SQI_NAME + ", "
- + COLUMN_SQI_FORM + ", "
+ + COLUMN_SQI_CONFIG + ", "
+ COLUMN_SQI_INDEX + ", "
+ COLUMN_SQI_TYPE + ", "
+ COLUMN_SQI_STRMASK + ", "
@@ -532,7 +532,7 @@ public final class DerbySchemaQuery {
+ " LEFT OUTER JOIN " + TABLE_SQ_JOB_INPUT
+ " ON " + COLUMN_SQBI_INPUT + " = " + COLUMN_SQI_ID
+ " AND " + COLUMN_SQBI_JOB + " = ?"
- + " WHERE " + COLUMN_SQI_FORM + " = ?"
+ + " WHERE " + COLUMN_SQI_CONFIG + " = ?"
+ " AND (" + COLUMN_SQBI_JOB + " = ? OR " + COLUMN_SQBI_JOB + " IS NULL)"
+ " ORDER BY " + COLUMN_SQI_INDEX;
@@ -544,21 +544,21 @@ public final class DerbySchemaQuery {
+ COLUMN_SQC_VERSION
+ ") VALUES (?, ?, ?)";
- // DML: Insert form base
- public static final String STMT_INSERT_FORM_BASE =
- "INSERT INTO " + TABLE_SQ_FORM + " ("
- + COLUMN_SQF_CONNECTOR + ", "
- + COLUMN_SQF_DIRECTION + ", "
- + COLUMN_SQF_NAME + ", "
- + COLUMN_SQF_TYPE + ", "
- + COLUMN_SQF_INDEX
+ // DML: Insert config base
+ public static final String STMT_INSERT_CONFIG_BASE =
+ "INSERT INTO " + TABLE_SQ_CONFIG + " ("
+ + COLUMN_SQ_CFG_OWNER + ", "
+ + COLUMN_SQ_CFG_DIRECTION + ", "
+ + COLUMN_SQ_CFG_NAME + ", "
+ + COLUMN_SQ_CFG_TYPE + ", "
+ + COLUMN_SQ_CFG_INDEX
+ ") VALUES ( ?, ?, ?, ?, ?)";
- // DML: Insert form input
+ // DML: Insert config input
public static final String STMT_INSERT_INPUT_BASE =
"INSERT INTO " + TABLE_SQ_INPUT + " ("
+ COLUMN_SQI_NAME + ", "
- + COLUMN_SQI_FORM + ", "
+ + COLUMN_SQI_CONFIG + ", "
+ COLUMN_SQI_INDEX + ", "
+ COLUMN_SQI_TYPE + ", "
+ COLUMN_SQI_STRMASK + ", "
@@ -566,37 +566,37 @@ public final class DerbySchemaQuery {
+ COLUMN_SQI_ENUMVALS
+ ") VALUES (?, ?, ?, ?, ?, ?, ?)";
- // Delete all forms for a given connector
- public static final String STMT_DELETE_FORMS_FOR_CONNECTOR =
- "DELETE FROM " + TABLE_SQ_FORM
- + " WHERE " + COLUMN_SQF_CONNECTOR + " = ?";
+ // Delete all configs for a given connector
+ public static final String STMT_DELETE_CONFIGS_FOR_CONNECTOR =
+ "DELETE FROM " + TABLE_SQ_CONFIG
+ + " WHERE " + COLUMN_SQ_CFG_OWNER + " = ?";
// Delete all inputs for a given connector
public static final String STMT_DELETE_INPUTS_FOR_CONNECTOR =
"DELETE FROM " + TABLE_SQ_INPUT
+ " WHERE "
- + COLUMN_SQI_FORM
+ + COLUMN_SQI_CONFIG
+ " IN (SELECT "
- + COLUMN_SQF_ID
- + " FROM " + TABLE_SQ_FORM
+ + COLUMN_SQ_CFG_ID
+ + " FROM " + TABLE_SQ_CONFIG
+ " WHERE "
- + COLUMN_SQF_CONNECTOR + " = ?)";
+ + COLUMN_SQ_CFG_OWNER + " = ?)";
- // Delete all framework inputs
- public static final String STMT_DELETE_FRAMEWORK_INPUTS =
+ // Delete all driver inputs
+ public static final String STMT_DELETE_DRIVER_INPUTS =
"DELETE FROM " + TABLE_SQ_INPUT
+ " WHERE "
- + COLUMN_SQI_FORM
+ + COLUMN_SQI_CONFIG
+ " IN (SELECT "
- + COLUMN_SQF_ID
- + " FROM " + TABLE_SQ_FORM
+ + COLUMN_SQ_CFG_ID
+ + " FROM " + TABLE_SQ_CONFIG
+ " WHERE "
- + COLUMN_SQF_CONNECTOR + " IS NULL)";
+ + COLUMN_SQ_CFG_OWNER + " IS NULL)";
- // Delete all framework forms
- public static final String STMT_DELETE_FRAMEWORK_FORMS =
- "DELETE FROM " + TABLE_SQ_FORM
- + " WHERE " + COLUMN_SQF_CONNECTOR + " IS NULL";
+ // Delete all driver configs
+ public static final String STMT_DELETE_DRIVER_CONFIGS =
+ "DELETE FROM " + TABLE_SQ_CONFIG
+ + " WHERE " + COLUMN_SQ_CFG_OWNER + " IS NULL";
@@ -608,102 +608,102 @@ public final class DerbySchemaQuery {
+ COLUMN_SQC_VERSION + " = ? "
+ " WHERE " + COLUMN_SQC_ID + " = ?";
- // DML: Insert new connection
- public static final String STMT_INSERT_CONNECTION =
- "INSERT INTO " + TABLE_SQ_CONNECTION + " ("
- + COLUMN_SQN_NAME + ", "
- + COLUMN_SQN_CONNECTOR + ", "
- + COLUMN_SQN_ENABLED + ", "
- + COLUMN_SQN_CREATION_USER + ", "
- + COLUMN_SQN_CREATION_DATE + ", "
- + COLUMN_SQN_UPDATE_USER + ", "
- + COLUMN_SQN_UPDATE_DATE
+ // DML: Insert new link
+ public static final String STMT_INSERT_LINK =
+ "INSERT INTO " + TABLE_SQ_LINK + " ("
+ + COLUMN_SQ_LNK_NAME + ", "
+ + COLUMN_SQ_LNK_CONNECTOR + ", "
+ + COLUMN_SQ_LNK_ENABLED + ", "
+ + COLUMN_SQ_LNK_CREATION_USER + ", "
+ + COLUMN_SQ_LNK_CREATION_DATE + ", "
+ + COLUMN_SQ_LNK_UPDATE_USER + ", "
+ + COLUMN_SQ_LNK_UPDATE_DATE
+ ") VALUES (?, ?, ?, ?, ?, ?, ?)";
- // DML: Insert new connection inputs
- public static final String STMT_INSERT_CONNECTION_INPUT =
- "INSERT INTO " + TABLE_SQ_CONNECTION_INPUT + " ("
- + COLUMN_SQNI_CONNECTION + ", "
- + COLUMN_SQNI_INPUT + ", "
- + COLUMN_SQNI_VALUE
+ // DML: Insert new link inputs
+ public static final String STMT_INSERT_LINK_INPUT =
+ "INSERT INTO " + TABLE_SQ_LINK_INPUT + " ("
+ + COLUMN_SQ_LNKI_LINK + ", "
+ + COLUMN_SQ_LNKI_INPUT + ", "
+ + COLUMN_SQ_LNKI_VALUE
+ ") VALUES (?, ?, ?)";
- // DML: Update connection
- public static final String STMT_UPDATE_CONNECTION =
- "UPDATE " + TABLE_SQ_CONNECTION + " SET "
- + COLUMN_SQN_NAME + " = ?, "
- + COLUMN_SQN_UPDATE_USER + " = ?, "
- + COLUMN_SQN_UPDATE_DATE + " = ? "
- + " WHERE " + COLUMN_SQN_ID + " = ?";
-
- // DML: Enable or disable connection
- public static final String STMT_ENABLE_CONNECTION =
- "UPDATE " + TABLE_SQ_CONNECTION + " SET "
- + COLUMN_SQN_ENABLED + " = ? "
- + " WHERE " + COLUMN_SQN_ID + " = ?";
-
- // DML: Delete rows from connection input table
- public static final String STMT_DELETE_CONNECTION_INPUT =
- "DELETE FROM " + TABLE_SQ_CONNECTION_INPUT
- + " WHERE " + COLUMN_SQNI_CONNECTION + " = ?";
-
- // DML: Delete row from connection table
- public static final String STMT_DELETE_CONNECTION =
- "DELETE FROM " + TABLE_SQ_CONNECTION
- + " WHERE " + COLUMN_SQN_ID + " = ?";
-
- // DML: Select one specific connection
- public static final String STMT_SELECT_CONNECTION_SINGLE =
+ // DML: Update link
+ public static final String STMT_UPDATE_LINK =
+ "UPDATE " + TABLE_SQ_LINK + " SET "
+ + COLUMN_SQ_LNK_NAME + " = ?, "
+ + COLUMN_SQ_LNK_UPDATE_USER + " = ?, "
+ + COLUMN_SQ_LNK_UPDATE_DATE + " = ? "
+ + " WHERE " + COLUMN_SQ_LNK_ID + " = ?";
+
+ // DML: Enable or disable link
+ public static final String STMT_ENABLE_LINK =
+ "UPDATE " + TABLE_SQ_LINK + " SET "
+ + COLUMN_SQ_LNK_ENABLED + " = ? "
+ + " WHERE " + COLUMN_SQ_LNK_ID + " = ?";
+
+ // DML: Delete rows from link input table
+ public static final String STMT_DELETE_LINK_INPUT =
+ "DELETE FROM " + TABLE_SQ_LINK_INPUT
+ + " WHERE " + COLUMN_SQ_LNKI_LINK + " = ?";
+
+ // DML: Delete row from link table
+ public static final String STMT_DELETE_LINK =
+ "DELETE FROM " + TABLE_SQ_LINK
+ + " WHERE " + COLUMN_SQ_LNK_ID + " = ?";
+
+ // DML: Select one specific link
+ public static final String STMT_SELECT_LINK_SINGLE =
"SELECT "
- + COLUMN_SQN_ID + ", "
- + COLUMN_SQN_NAME + ", "
- + COLUMN_SQN_CONNECTOR + ", "
- + COLUMN_SQN_ENABLED + ", "
- + COLUMN_SQN_CREATION_USER + ", "
- + COLUMN_SQN_CREATION_DATE + ", "
- + COLUMN_SQN_UPDATE_USER + ", "
- + COLUMN_SQN_UPDATE_DATE
- + " FROM " + TABLE_SQ_CONNECTION
- + " WHERE " + COLUMN_SQN_ID + " = ?";
+ + COLUMN_SQ_LNK_ID + ", "
+ + COLUMN_SQ_LNK_NAME + ", "
+ + COLUMN_SQ_LNK_CONNECTOR + ", "
+ + COLUMN_SQ_LNK_ENABLED + ", "
+ + COLUMN_SQ_LNK_CREATION_USER + ", "
+ + COLUMN_SQ_LNK_CREATION_DATE + ", "
+ + COLUMN_SQ_LNK_UPDATE_USER + ", "
+ + COLUMN_SQ_LNK_UPDATE_DATE
+ + " FROM " + TABLE_SQ_LINK
+ + " WHERE " + COLUMN_SQ_LNK_ID + " = ?";
// DML: Select all connections
- public static final String STMT_SELECT_CONNECTION_ALL =
+ public static final String STMT_SELECT_LINK_ALL =
"SELECT "
- + COLUMN_SQN_ID + ", "
- + COLUMN_SQN_NAME + ", "
- + COLUMN_SQN_CONNECTOR + ", "
- + COLUMN_SQN_ENABLED + ", "
- + COLUMN_SQN_CREATION_USER + ", "
- + COLUMN_SQN_CREATION_DATE + ", "
- + COLUMN_SQN_UPDATE_USER + ", "
- + COLUMN_SQN_UPDATE_DATE
- + " FROM " + TABLE_SQ_CONNECTION;
+ + COLUMN_SQ_LNK_ID + ", "
+ + COLUMN_SQ_LNK_NAME + ", "
+ + COLUMN_SQ_LNK_CONNECTOR + ", "
+ + COLUMN_SQ_LNK_ENABLED + ", "
+ + COLUMN_SQ_LNK_CREATION_USER + ", "
+ + COLUMN_SQ_LNK_CREATION_DATE + ", "
+ + COLUMN_SQ_LNK_UPDATE_USER + ", "
+ + COLUMN_SQ_LNK_UPDATE_DATE
+ + " FROM " + TABLE_SQ_LINK;
// DML: Select all connections for a specific connector.
- public static final String STMT_SELECT_CONNECTION_FOR_CONNECTOR =
+ public static final String STMT_SELECT_LINK_FOR_CONNECTOR =
"SELECT "
- + COLUMN_SQN_ID + ", "
- + COLUMN_SQN_NAME + ", "
- + COLUMN_SQN_CONNECTOR + ", "
- + COLUMN_SQN_ENABLED + ", "
- + COLUMN_SQN_CREATION_USER + ", "
- + COLUMN_SQN_CREATION_DATE + ", "
- + COLUMN_SQN_UPDATE_USER + ", "
- + COLUMN_SQN_UPDATE_DATE
- + " FROM " + TABLE_SQ_CONNECTION
- + " WHERE " + COLUMN_SQN_CONNECTOR + " = ?";
-
- // DML: Check if given connection exists
- public static final String STMT_SELECT_CONNECTION_CHECK =
- "SELECT count(*) FROM " + TABLE_SQ_CONNECTION
- + " WHERE " + COLUMN_SQN_ID + " = ?";
+ + COLUMN_SQ_LNK_ID + ", "
+ + COLUMN_SQ_LNK_NAME + ", "
+ + COLUMN_SQ_LNK_CONNECTOR + ", "
+ + COLUMN_SQ_LNK_ENABLED + ", "
+ + COLUMN_SQ_LNK_CREATION_USER + ", "
+ + COLUMN_SQ_LNK_CREATION_DATE + ", "
+ + COLUMN_SQ_LNK_UPDATE_USER + ", "
+ + COLUMN_SQ_LNK_UPDATE_DATE
+ + " FROM " + TABLE_SQ_LINK
+ + " WHERE " + COLUMN_SQ_LNK_CONNECTOR + " = ?";
+
+ // DML: Check if given link exists
+ public static final String STMT_SELECT_LINK_CHECK =
+ "SELECT count(*) FROM " + TABLE_SQ_LINK
+ + " WHERE " + COLUMN_SQ_LNK_ID + " = ?";
// DML: Insert new job
public static final String STMT_INSERT_JOB =
"INSERT INTO " + TABLE_SQ_JOB + " ("
+ COLUMN_SQB_NAME + ", "
- + COLUMN_SQB_FROM_CONNECTION + ", "
- + COLUMN_SQB_TO_CONNECTION + ", "
+ + COLUMN_SQB_FROM_LINK + ", "
+ + COLUMN_SQB_TO_LINK + ", "
+ COLUMN_SQB_ENABLED + ", "
+ COLUMN_SQB_CREATION_USER + ", "
+ COLUMN_SQB_CREATION_DATE + ", "
@@ -747,59 +747,59 @@ public final class DerbySchemaQuery {
"SELECT count(*) FROM " + TABLE_SQ_JOB
+ " WHERE " + COLUMN_SQB_ID + " = ?";
- // DML: Check if there are jobs for given connection
- public static final String STMT_SELECT_JOBS_FOR_CONNECTION_CHECK =
+ // DML: Check if there are jobs for given link
+ public static final String STMT_SELECT_JOBS_FOR_LINK_CHECK =
"SELECT"
+ " count(*)"
+ " FROM " + TABLE_SQ_JOB
- + " JOIN " + TABLE_SQ_CONNECTION
- + " ON " + COLUMN_SQB_FROM_CONNECTION + " = " + COLUMN_SQN_ID
- + " WHERE " + COLUMN_SQN_ID + " = ? ";
+ + " JOIN " + TABLE_SQ_LINK
+ + " ON " + COLUMN_SQB_FROM_LINK + " = " + COLUMN_SQ_LNK_ID
+ + " WHERE " + COLUMN_SQ_LNK_ID + " = ? ";
// DML: Select one specific job
public static final String STMT_SELECT_JOB_SINGLE =
"SELECT "
- + "FROM_CONNECTOR." + COLUMN_SQN_CONNECTOR + ", "
- + "TO_CONNECTOR." + COLUMN_SQN_CONNECTOR + ", "
+ + "FROM_CONNECTOR." + COLUMN_SQ_LNK_CONNECTOR + ", "
+ + "TO_CONNECTOR." + COLUMN_SQ_LNK_CONNECTOR + ", "
+ "job." + COLUMN_SQB_ID + ", "
+ "job." + COLUMN_SQB_NAME + ", "
- + "job." + COLUMN_SQB_FROM_CONNECTION + ", "
- + "job." + COLUMN_SQB_TO_CONNECTION + ", "
+ + "job." + COLUMN_SQB_FROM_LINK + ", "
+ + "job." + COLUMN_SQB_TO_LINK + ", "
+ "job." + COLUMN_SQB_ENABLED + ", "
+ "job." + COLUMN_SQB_CREATION_USER + ", "
+ "job." + COLUMN_SQB_CREATION_DATE + ", "
+ "job." + COLUMN_SQB_UPDATE_USER + ", "
+ "job." + COLUMN_SQB_UPDATE_DATE
+ " FROM " + TABLE_SQ_JOB + " job"
- + " LEFT JOIN " + TABLE_SQ_CONNECTION
- + " FROM_CONNECTOR ON " + COLUMN_SQB_FROM_CONNECTION + " = FROM_CONNECTOR." + COLUMN_SQN_ID
- + " LEFT JOIN " + TABLE_SQ_CONNECTION
- + " TO_CONNECTOR ON " + COLUMN_SQB_TO_CONNECTION + " = TO_CONNECTOR." + COLUMN_SQN_ID
+ + " LEFT JOIN " + TABLE_SQ_LINK
+ + " FROM_CONNECTOR ON " + COLUMN_SQB_FROM_LINK + " = FROM_CONNECTOR." + COLUMN_SQ_LNK_ID
+ + " LEFT JOIN " + TABLE_SQ_LINK
+ + " TO_CONNECTOR ON " + COLUMN_SQB_TO_LINK + " = TO_CONNECTOR." + COLUMN_SQ_LNK_ID
+ " WHERE " + COLUMN_SQB_ID + " = ?";
// DML: Select all jobs
public static final String STMT_SELECT_JOB_ALL =
"SELECT "
- + "FROM_CONNECTION." + COLUMN_SQN_CONNECTOR + ", "
- + "TO_CONNECTION." + COLUMN_SQN_CONNECTOR + ", "
+ + "FROM_LINK." + COLUMN_SQ_LNK_CONNECTOR + ", "
+ + "TO_LINK." + COLUMN_SQ_LNK_CONNECTOR + ", "
+ "JOB." + COLUMN_SQB_ID + ", "
+ "JOB." + COLUMN_SQB_NAME + ", "
- + "JOB." + COLUMN_SQB_FROM_CONNECTION + ", "
- + "JOB." + COLUMN_SQB_TO_CONNECTION + ", "
+ + "JOB." + COLUMN_SQB_FROM_LINK + ", "
+ + "JOB." + COLUMN_SQB_TO_LINK + ", "
+ "JOB." + COLUMN_SQB_ENABLED + ", "
+ "JOB." + COLUMN_SQB_CREATION_USER + ", "
+ "JOB." + COLUMN_SQB_CREATION_DATE + ", "
+ "JOB." + COLUMN_SQB_UPDATE_USER + ", "
+ "JOB." + COLUMN_SQB_UPDATE_DATE
+ " FROM " + TABLE_SQ_JOB + " JOB"
- + " LEFT JOIN " + TABLE_SQ_CONNECTION + " FROM_CONNECTION"
- + " ON " + COLUMN_SQB_FROM_CONNECTION + " = FROM_CONNECTION." + COLUMN_SQN_ID
- + " LEFT JOIN " + TABLE_SQ_CONNECTION + " TO_CONNECTION"
- + " ON " + COLUMN_SQB_TO_CONNECTION + " = TO_CONNECTION." + COLUMN_SQN_ID;
+ + " LEFT JOIN " + TABLE_SQ_LINK + " FROM_LINK"
+ + " ON " + COLUMN_SQB_FROM_LINK + " = FROM_LINK." + COLUMN_SQ_LNK_ID
+ + " LEFT JOIN " + TABLE_SQ_LINK + " TO_LINK"
+ + " ON " + COLUMN_SQB_TO_LINK + " = TO_LINK." + COLUMN_SQ_LNK_ID;
// DML: Select all jobs for a Connector
public static final String STMT_SELECT_ALL_JOBS_FOR_CONNECTOR = STMT_SELECT_JOB_ALL
- + " WHERE FROM_CONNECTION." + COLUMN_SQN_CONNECTOR + " = ? OR TO_CONNECTION." + COLUMN_SQN_CONNECTOR + " = ?";
+ + " WHERE FROM_LINK." + COLUMN_SQ_LNK_CONNECTOR + " = ? OR TO_LINK." + COLUMN_SQ_LNK_CONNECTOR + " = ?";
// DML: Insert new submission
public static final String STMT_INSERT_SUBMISSION =
@@ -951,117 +951,117 @@ public final class DerbySchemaQuery {
+ COLUMN_SQC_VERSION + " SET DATA TYPE VARCHAR(64)";
// Version 4 Upgrade
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_CONNECTION_TO_SQB_FROM_CONNECTION =
- "RENAME COLUMN " + TABLE_SQ_JOB + "." + COLUMN_SQB_CONNECTION
- + " TO " + COLUMN_SQB_FROM_CONNECTION;
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_LINK_TO_SQB_FROM_LINK =
+ "RENAME COLUMN " + TABLE_SQ_JOB + "." + COLUMN_SQB_LINK
+ + " TO " + COLUMN_SQB_FROM_LINK;
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_CONNECTION =
- "ALTER TABLE " + TABLE_SQ_JOB + " ADD COLUMN " + COLUMN_SQB_TO_CONNECTION
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_LINK =
+ "ALTER TABLE " + TABLE_SQ_JOB + " ADD COLUMN " + COLUMN_SQB_TO_LINK
+ " BIGINT";
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQN =
- "ALTER TABLE " + TABLE_SQ_JOB + " DROP CONSTRAINT " + CONSTRAINT_SQB_SQN;
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQ_LNK =
+ "ALTER TABLE " + TABLE_SQ_JOB + " DROP CONSTRAINT " + CONSTRAINT_SQB_SQ_LNK;
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_FROM =
- "ALTER TABLE " + TABLE_SQ_JOB + " ADD CONSTRAINT " + CONSTRAINT_SQB_SQN_FROM
- + " FOREIGN KEY (" + COLUMN_SQB_FROM_CONNECTION + ") REFERENCES "
- + TABLE_SQ_CONNECTION + " (" + COLUMN_SQN_ID + ")";
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_FROM =
+ "ALTER TABLE " + TABLE_SQ_JOB + " ADD CONSTRAINT " + CONSTRAINT_SQB_SQ_LNK_FROM
+ + " FOREIGN KEY (" + COLUMN_SQB_FROM_LINK + ") REFERENCES "
+ + TABLE_SQ_LINK + " (" + COLUMN_SQ_LNK_ID + ")";
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_TO =
- "ALTER TABLE " + TABLE_SQ_JOB + " ADD CONSTRAINT " + CONSTRAINT_SQB_SQN_TO
- + " FOREIGN KEY (" + COLUMN_SQB_TO_CONNECTION + ") REFERENCES "
- + TABLE_SQ_CONNECTION + " (" + COLUMN_SQN_ID + ")";
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_TO =
+ "ALTER TABLE " + TABLE_SQ_JOB + " ADD CONSTRAINT " + CONSTRAINT_SQB_SQ_LNK_TO
+ + " FOREIGN KEY (" + COLUMN_SQB_TO_LINK + ") REFERENCES "
+ + TABLE_SQ_LINK + " (" + COLUMN_SQ_LNK_ID + ")";
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_RENAME_COLUMN_SQF_OPERATION_TO_SQF_DIRECTION =
- "RENAME COLUMN " + TABLE_SQ_FORM + "." + COLUMN_SQF_OPERATION
- + " TO " + COLUMN_SQF_DIRECTION;
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_RENAME_COLUMN_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION =
+ "RENAME COLUMN " + TABLE_SQ_CONFIG + "." + COLUMN_SQ_CFG_OPERATION
+ + " TO " + COLUMN_SQ_CFG_DIRECTION;
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_OPERATION_TO_SQF_DIRECTION =
- "UPDATE " + TABLE_SQ_FORM + " SET " + COLUMN_SQF_DIRECTION
- + "=? WHERE " + COLUMN_SQF_DIRECTION + "=?"
- + " AND " + COLUMN_SQF_CONNECTOR + " IS NOT NULL";
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION =
+ "UPDATE " + TABLE_SQ_CONFIG + " SET " + COLUMN_SQ_CFG_DIRECTION
+ + "=? WHERE " + COLUMN_SQ_CFG_DIRECTION + "=?"
+ + " AND " + COLUMN_SQ_CFG_OWNER + " IS NOT NULL";
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR =
- "UPDATE " + TABLE_SQ_FORM + " SET " + COLUMN_SQF_CONNECTOR + "= ?"
- + " WHERE " + COLUMN_SQF_CONNECTOR + " IS NULL AND "
- + COLUMN_SQF_NAME + " IN (?, ?)";
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_CONNECTOR =
+ "UPDATE " + TABLE_SQ_CONFIG + " SET " + COLUMN_SQ_CFG_OWNER + "= ?"
+ + " WHERE " + COLUMN_SQ_CFG_OWNER + " IS NULL AND "
+ + COLUMN_SQ_CFG_NAME + " IN (?, ?)";
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR_HDFS_FORM_DIRECTION =
- "UPDATE " + TABLE_SQ_FORM + " SET " + COLUMN_SQF_DIRECTION + "= ?"
- + " WHERE " + COLUMN_SQF_NAME + "= ?";
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_CONNECTOR_HDFS_CONFIG_DIRECTION =
+ "UPDATE " + TABLE_SQ_CONFIG + " SET " + COLUMN_SQ_CFG_DIRECTION + "= ?"
+ + " WHERE " + COLUMN_SQ_CFG_NAME + "= ?";
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION_COPY_SQB_FROM_CONNECTION =
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_LINK_COPY_SQB_FROM_LINK =
"UPDATE " + TABLE_SQ_JOB + " SET "
- + COLUMN_SQB_TO_CONNECTION + "=" + COLUMN_SQB_FROM_CONNECTION
+ + COLUMN_SQB_TO_LINK + "=" + COLUMN_SQB_FROM_LINK
+ " WHERE " + COLUMN_SQB_TYPE + "= ?";
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_FROM_CONNECTION =
- "UPDATE " + TABLE_SQ_JOB + " SET " + COLUMN_SQB_FROM_CONNECTION + "=?"
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_FROM_LINK =
+ "UPDATE " + TABLE_SQ_JOB + " SET " + COLUMN_SQB_FROM_LINK + "=?"
+ " WHERE " + COLUMN_SQB_TYPE + "= ?";
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION =
- "UPDATE " + TABLE_SQ_JOB + " SET " + COLUMN_SQB_TO_CONNECTION + "=?"
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_LINK =
+ "UPDATE " + TABLE_SQ_JOB + " SET " + COLUMN_SQB_TO_LINK + "=?"
+ " WHERE " + COLUMN_SQB_TYPE + "= ?";
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_NAME =
- "UPDATE " + TABLE_SQ_FORM + " SET "
- + COLUMN_SQF_NAME + "= ?"
- + " WHERE " + COLUMN_SQF_NAME + "= ?"
- + " AND " + COLUMN_SQF_DIRECTION + "= ?";
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_SQ_CFG_NAME =
+ "UPDATE " + TABLE_SQ_CONFIG + " SET "
+ + COLUMN_SQ_CFG_NAME + "= ?"
+ + " WHERE " + COLUMN_SQ_CFG_NAME + "= ?"
+ + " AND " + COLUMN_SQ_CFG_DIRECTION + "= ?";
/**
- * Intended to rename forms based on direction.
- * e.g. If SQ_FORM.SQF_NAME = 'table' and parameter 1 = 'from'
- * then SQ_FORM.SQF_NAME = 'fromJobConfig'.
+ * Intended to rename configs based on direction.
+ * e.g. If SQ_CONFIG.SQ_CFG_NAME = 'table' and parameter 1 = 'from'
+ * then SQ_CONFIG.SQ_CFG_NAME = 'fromJobConfig'.
*/
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES =
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_TABLE_INPUT_NAMES =
"UPDATE " + TABLE_SQ_INPUT + " SET "
+ COLUMN_SQI_NAME + "=("
+ "? || UPPER(SUBSTR(" + COLUMN_SQI_NAME + ",1,1))"
+ " || SUBSTR(" + COLUMN_SQI_NAME + ",2) )"
- + " WHERE " + COLUMN_SQI_FORM + " IN ("
- + " SELECT " + COLUMN_SQF_ID + " FROM " + TABLE_SQ_FORM + " WHERE " + COLUMN_SQF_NAME + "= ?"
- + " AND " + COLUMN_SQF_DIRECTION + "= ?)";
+ + " WHERE " + COLUMN_SQI_CONFIG + " IN ("
+ + " SELECT " + COLUMN_SQ_CFG_ID + " FROM " + TABLE_SQ_CONFIG + " WHERE " + COLUMN_SQ_CFG_NAME + "= ?"
+ + " AND " + COLUMN_SQ_CFG_DIRECTION + "= ?)";
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_DIRECTION_TO_NULL =
- "UPDATE " + TABLE_SQ_FORM + " SET "
- + COLUMN_SQF_DIRECTION + "= NULL"
- + " WHERE " + COLUMN_SQF_NAME + "= ?";
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_DIRECTION_TO_NULL =
+ "UPDATE " + TABLE_SQ_CONFIG + " SET "
+ + COLUMN_SQ_CFG_DIRECTION + "= NULL"
+ + " WHERE " + COLUMN_SQ_CFG_NAME + "= ?";
- public static final String QUERY_SELECT_THROTTLING_FORM_INPUT_IDS =
+ public static final String QUERY_SELECT_THROTTLING_CONFIG_INPUT_IDS =
"SELECT SQI." + COLUMN_SQI_ID + " FROM " + TABLE_SQ_INPUT + " SQI"
- + " INNER JOIN " + TABLE_SQ_FORM + " SQF ON SQI." + COLUMN_SQI_FORM + "=SQF." + COLUMN_SQF_ID
- + " WHERE SQF." + COLUMN_SQF_NAME + "='throttling' AND SQF." + COLUMN_SQF_DIRECTION + "=?";
+ + " INNER JOIN " + TABLE_SQ_CONFIG + " SQ_CFG ON SQI." + COLUMN_SQI_CONFIG + "=SQ_CFG." + COLUMN_SQ_CFG_ID
+ + " WHERE SQ_CFG." + COLUMN_SQ_CFG_NAME + "='throttling' AND SQ_CFG." + COLUMN_SQ_CFG_DIRECTION + "=?";
/**
* Intended to change SQ_JOB_INPUT.SQBI_INPUT from EXPORT
- * throttling form, to IMPORT throttling form.
+ * throttling config, to IMPORT throttling config.
*/
- public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_UPDATE_THROTTLING_FORM_INPUTS =
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_UPDATE_THROTTLING_CONFIG_INPUTS =
"UPDATE " + TABLE_SQ_JOB_INPUT + " SQBI SET"
- + " SQBI." + COLUMN_SQBI_INPUT + "=(" + QUERY_SELECT_THROTTLING_FORM_INPUT_IDS
+ + " SQBI." + COLUMN_SQBI_INPUT + "=(" + QUERY_SELECT_THROTTLING_CONFIG_INPUT_IDS
+ " AND SQI." + COLUMN_SQI_NAME + "=("
+ "SELECT SQI2." + COLUMN_SQI_NAME + " FROM " + TABLE_SQ_INPUT + " SQI2"
+ " WHERE SQI2." + COLUMN_SQI_ID + "=SQBI." + COLUMN_SQBI_INPUT + " FETCH FIRST 1 ROWS ONLY"
+ "))"
- + "WHERE SQBI." + COLUMN_SQBI_INPUT + " IN (" + QUERY_SELECT_THROTTLING_FORM_INPUT_IDS + ")";
+ + "WHERE SQBI." + COLUMN_SQBI_INPUT + " IN (" + QUERY_SELECT_THROTTLING_CONFIG_INPUT_IDS + ")";
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_REMOVE_EXTRA_FORM_INPUTS =
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_REMOVE_EXTRA_CONFIG_INPUTS =
"DELETE FROM " + TABLE_SQ_INPUT + " SQI"
- + " WHERE SQI." + COLUMN_SQI_FORM + " IN ("
- + "SELECT SQF." + COLUMN_SQF_ID + " FROM " + TABLE_SQ_FORM + " SQF "
- + " WHERE SQF." + COLUMN_SQF_NAME + "= ?"
- + " AND SQF." + COLUMN_SQF_DIRECTION + "= ?)";
-
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_REMOVE_EXTRA_FRAMEWORK_FORM =
- "DELETE FROM " + TABLE_SQ_FORM
- + " WHERE " + COLUMN_SQF_NAME + "= ?"
- + " AND " + COLUMN_SQF_DIRECTION + "= ?";
-
- public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_FRAMEWORK_INDEX =
- "UPDATE " + TABLE_SQ_FORM + " SET "
- + COLUMN_SQF_INDEX + "= ?"
- + " WHERE " + COLUMN_SQF_NAME + "= ?";
+ + " WHERE SQI." + COLUMN_SQI_CONFIG + " IN ("
+ + "SELECT SQ_CFG." + COLUMN_SQ_CFG_ID + " FROM " + TABLE_SQ_CONFIG + " SQ_CFG "
+ + " WHERE SQ_CFG." + COLUMN_SQ_CFG_NAME + "= ?"
+ + " AND SQ_CFG." + COLUMN_SQ_CFG_DIRECTION + "= ?)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_REMOVE_EXTRA_DRIVER_CONFIG =
+ "DELETE FROM " + TABLE_SQ_CONFIG
+ + " WHERE " + COLUMN_SQ_CFG_NAME + "= ?"
+ + " AND " + COLUMN_SQ_CFG_DIRECTION + "= ?";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_DRIVER_INDEX =
+ "UPDATE " + TABLE_SQ_CONFIG + " SET "
+ + COLUMN_SQ_CFG_INDEX + "= ?"
+ + " WHERE " + COLUMN_SQ_CFG_NAME + "= ?";
public static final String QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE =
"ALTER TABLE " + TABLE_SQ_JOB + " DROP COLUMN " + COLUMN_SQB_TYPE;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index 4b95687..bf72626 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -17,20 +17,35 @@
*/
package org.apache.sqoop.repository.derby;
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.driver.Driver;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
-import org.apache.sqoop.model.MMapInput;
-import org.apache.sqoop.model.MStringInput;
-import org.junit.After;
-import org.junit.Before;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_SCHEMA_SQOOP;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_CONFIG;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_CONNECTOR;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_COUNTER;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_COUNTER_GROUP;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_COUNTER_SUBMISSION;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_INPUT;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_JOB;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_JOB_INPUT;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_LINK;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_LINK_INPUT;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_SUBMISSION;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_SYSTEM;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_CONFIG_RENAME_COLUMN_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_CREATION_USER;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_ENABLED;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_LINK;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_UPDATE_USER;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_FROM;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_TO;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQ_LNK;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_LINK_TO_SQB_FROM_LINK;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_CREATION_USER;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_ENABLED;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_UPDATE_USER;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_CREATION_USER;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_UPDATE_USER;
+import static org.junit.Assert.assertEquals;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -42,8 +57,22 @@ import java.sql.Statement;
import java.util.LinkedList;
import java.util.List;
-import static org.apache.sqoop.repository.derby.DerbySchemaQuery.*;
-import static org.junit.Assert.assertEquals;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriver;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MMapInput;
+import org.apache.sqoop.model.MStringInput;
+import org.apache.sqoop.model.MToConfig;
+import org.junit.After;
+import org.junit.Before;
/**
* Abstract class with convenience methods for testing derby repository.
@@ -91,11 +120,11 @@ abstract public class DerbyTestCase {
if (version > 0) {
runQuery(QUERY_CREATE_SCHEMA_SQOOP);
runQuery(QUERY_CREATE_TABLE_SQ_CONNECTOR);
- runQuery(QUERY_CREATE_TABLE_SQ_FORM);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONFIG);
runQuery(QUERY_CREATE_TABLE_SQ_INPUT);
- runQuery(QUERY_CREATE_TABLE_SQ_CONNECTION);
+ runQuery(QUERY_CREATE_TABLE_SQ_LINK);
runQuery(QUERY_CREATE_TABLE_SQ_JOB);
- runQuery(QUERY_CREATE_TABLE_SQ_CONNECTION_INPUT);
+ runQuery(QUERY_CREATE_TABLE_SQ_LINK_INPUT);
runQuery(QUERY_CREATE_TABLE_SQ_JOB_INPUT);
runQuery(QUERY_CREATE_TABLE_SQ_SUBMISSION);
runQuery(QUERY_CREATE_TABLE_SQ_COUNTER_GROUP);
@@ -105,10 +134,10 @@ abstract public class DerbyTestCase {
if (version > 1) {
runQuery(QUERY_CREATE_TABLE_SQ_SYSTEM);
- runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_ENABLED);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_ENABLED);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_ENABLED);
- runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_CREATION_USER);
- runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_UPDATE_USER);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_CREATION_USER);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_UPDATE_USER);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_CREATION_USER);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_UPDATE_USER);
runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_CREATION_USER);
@@ -116,12 +145,12 @@ abstract public class DerbyTestCase {
}
if (version > 3) {
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_RENAME_COLUMN_SQF_OPERATION_TO_SQF_DIRECTION);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_CONNECTION_TO_SQB_FROM_CONNECTION);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_CONNECTION);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQN);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_FROM);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_TO);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_RENAME_COLUMN_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_LINK_TO_SQB_FROM_LINK);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_LINK);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQ_LNK);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_FROM);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_TO);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE);
}
@@ -180,65 +209,64 @@ abstract public class DerbyTestCase {
String connector = "1";
- // Connector form entries
+ // Connector config entries
for(String operation : new String[] {"null", "'IMPORT'", "'EXPORT'"}) {
String type;
if(operation.equals("null")) {
- type = "CONNECTION";
+ type = "LINK";
} else {
type = "JOB";
}
- runQuery("INSERT INTO SQOOP.SQ_FORM"
- + "(SQF_CONNECTOR, SQF_OPERATION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ runQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_OPERATION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ "VALUES("
+ connector + ", "
+ operation
- + ", 'F1', '"
+ + ", 'C1', '"
+ type
+ "', 0)");
- runQuery("INSERT INTO SQOOP.SQ_FORM"
- + "(SQF_CONNECTOR, SQF_OPERATION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ runQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_OPERATION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ "VALUES("
+ connector + ", "
+ operation
- + ", 'F2', '"
+ + ", 'C2', '"
+ type
+ "', 1)");
}
- // Framework form entries
- runQuery("INSERT INTO SQOOP.SQ_FORM"
- + "(SQF_CONNECTOR, SQF_OPERATION, SQF_NAME, SQF_TYPE, SQF_INDEX) VALUES"
+ // Driver config entries
+ runQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_OPERATION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) VALUES"
+ "(NULL, 'IMPORT', 'output', 'JOB', 0),"
+ "(NULL, 'IMPORT', 'throttling', 'JOB', 1),"
+ "(NULL, 'EXPORT', 'input', 'JOB', 0),"
+ "(NULL, 'EXPORT', 'throttling', 'JOB', 1),"
- + "(NULL, NULL, 'security', 'CONNECTION', 0)");
+ + "(NULL, NULL, 'security', 'LINK', 0)");
// Connector input entries
- int x = 0;
for(int i = 0; i < 3; i++) {
- // First form
+ // First config
runQuery("INSERT INTO SQOOP.SQ_INPUT"
- +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ +"(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ " VALUES('I1', " + (i * 2 + 1) + ", 0, 'STRING', false, 30)");
runQuery("INSERT INTO SQOOP.SQ_INPUT"
- +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ +"(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ " VALUES('I2', " + (i * 2 + 1) + ", 1, 'MAP', false, 30)");
- // Second form
+ // Second config
runQuery("INSERT INTO SQOOP.SQ_INPUT"
- +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ +"(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ " VALUES('I3', " + (i * 2 + 2) + ", 0, 'STRING', false, 30)");
runQuery("INSERT INTO SQOOP.SQ_INPUT"
- +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ +"(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ " VALUES('I4', " + (i * 2 + 2) + ", 1, 'MAP', false, 30)");
}
- // Framework input entries.
- runQuery("INSERT INTO SQOOP.SQ_INPUT (SQI_NAME, SQI_FORM, SQI_INDEX,"
+ // Driver input entries.
+ runQuery("INSERT INTO SQOOP.SQ_INPUT (SQI_NAME, SQI_CONFIG, SQI_INDEX,"
+ " SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_ENUMVALS)"
+" VALUES ('security.maxConnections',11,0,'INTEGER','false',NULL,NULL),"
+ "('input.inputDirectory',9,0,'STRING','false',255,NULL),"
@@ -257,74 +285,72 @@ abstract public class DerbyTestCase {
runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ "VALUES('A', 'org.apache.sqoop.test.A', '1.0-test')");
- // Connector part
for (String connector : new String[]{"1"}) {
- // Form entries
+ // connector configs
for (String direction : new String[]{"null", "'FROM'", "'TO'"}) {
String type;
if (direction.equals("null")) {
- type = "CONNECTION";
+ type = "LINK";
} else {
type = "JOB";
}
- runQuery("INSERT INTO SQOOP.SQ_FORM"
- + "(SQF_CONNECTOR, SQF_DIRECTION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ runQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_DIRECTION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ "VALUES("
+ connector + ", "
+ direction
- + ", 'F1', '"
+ + ", 'C1', '"
+ type
+ "', 0)");
- runQuery("INSERT INTO SQOOP.SQ_FORM"
- + "(SQF_CONNECTOR, SQF_DIRECTION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ runQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_DIRECTION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ "VALUES("
+ connector + ", "
+ direction
- + ", 'F2', '"
+ + ", 'C2', '"
+ type
+ "', 1)");
}
}
- // Framework part
- for (String type : new String[]{"CONNECTION", "JOB"}) {
- runQuery("INSERT INTO SQOOP.SQ_FORM"
- + "(SQF_CONNECTOR, SQF_DIRECTION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ // driver config
+ for (String type : new String[]{"JOB"}) {
+ runQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_DIRECTION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ "VALUES(NULL, NULL"
- + ", 'F1', '"
+ + ", 'C1', '"
+ type
+ "', 0)");
- runQuery("INSERT INTO SQOOP.SQ_FORM"
- + "(SQF_CONNECTOR, SQF_DIRECTION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ runQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_DIRECTION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ "VALUES(NULL, NULL"
- + ", 'F2', '"
+ + ", 'C2', '"
+ type
+ "', 1)");
}
// Input entries
- // Connector link parts: 0-3
- // Connector job (FROM) parts: 4-7
- // Connector job (TO) parts: 8-11
- // Framework link parts: 12-15
- // Framework job parts: 16-19
- for (int i = 0; i < 5; i++) {
- // First form
+ // Connector LINK config: 0-3
+ // Connector job (FROM) config: 4-7
+ // Connector job (TO) config: 8-11
+ // Driver JOB config: 12-15
+ for (int i = 0; i < 4; i++) {
+ // First config
runQuery("INSERT INTO SQOOP.SQ_INPUT"
- + "(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ " VALUES('I1', " + (i * 2 + 1) + ", 0, 'STRING', false, 30)");
runQuery("INSERT INTO SQOOP.SQ_INPUT"
- + "(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ " VALUES('I2', " + (i * 2 + 1) + ", 1, 'MAP', false, 30)");
- // Second form
+ // Second config
runQuery("INSERT INTO SQOOP.SQ_INPUT"
- + "(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ " VALUES('I3', " + (i * 2 + 2) + ", 0, 'STRING', false, 30)");
runQuery("INSERT INTO SQOOP.SQ_INPUT"
- + "(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + "(SQI_NAME, SQI_CONFIG, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ " VALUES('I4', " + (i * 2 + 2) + ", 1, 'MAP', false, 30)");
}
}
@@ -350,7 +376,7 @@ abstract public class DerbyTestCase {
}
}
- protected void loadConnectorAndDriverConfig() throws Exception {
+ protected void loadConnectorLinkConfig() throws Exception {
loadConnectorAndDriverConfig(LATEST_SYSTEM_VERSION);
}
@@ -364,15 +390,15 @@ abstract public class DerbyTestCase {
switch (version) {
case 2:
// Insert two links - CA and CB
- runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ runQuery("INSERT INTO SQOOP.SQ_LINK(SQ_LNK_NAME, SQ_LNK_CONNECTOR) "
+ "VALUES('CA', 1)");
- runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ runQuery("INSERT INTO SQOOP.SQ_LINK(SQ_LNK_NAME, SQ_LNK_CONNECTOR) "
+ "VALUES('CB', 1)");
for(String ci : new String[] {"1", "2"}) {
for(String i : new String[] {"1", "3", "13", "15"}) {
- runQuery("INSERT INTO SQOOP.SQ_CONNECTION_INPUT"
- + "(SQNI_CONNECTION, SQNI_INPUT, SQNI_VALUE) "
+ runQuery("INSERT INTO SQOOP.SQ_LINK_INPUT"
+ + "(SQ_LNKI_LINK, SQ_LNKI_INPUT, SQ_LNKI_VALUE) "
+ "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
}
}
@@ -380,15 +406,15 @@ abstract public class DerbyTestCase {
case 4:
// Insert two links - CA and CB
- runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ runQuery("INSERT INTO SQOOP.SQ_LINK(SQ_LNK_NAME, SQ_LNK_CONNECTOR) "
+ "VALUES('CA', 1)");
- runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ runQuery("INSERT INTO SQOOP.SQ_LINK(SQ_LNK_NAME, SQ_LNK_CONNECTOR) "
+ "VALUES('CB', 1)");
for (String ci : new String[]{"1", "2"}) {
for (String i : new String[]{"1", "3", "13", "15"}) {
- runQuery("INSERT INTO SQOOP.SQ_CONNECTION_INPUT"
- + "(SQNI_CONNECTION, SQNI_INPUT, SQNI_VALUE) "
+ runQuery("INSERT INTO SQOOP.SQ_LINK_INPUT"
+ + "(SQ_LNKI_LINK, SQ_LNKI_INPUT, SQ_LNKI_VALUE) "
+ "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
}
}
@@ -414,7 +440,7 @@ abstract public class DerbyTestCase {
case 2:
for(String type : new String[] {"IMPORT", "EXPORT"}) {
for(String name : new String[] {"JA", "JB"} ) {
- runQuery("INSERT INTO SQOOP.SQ_JOB(SQB_NAME, SQB_CONNECTION, SQB_TYPE)"
+ runQuery("INSERT INTO SQOOP.SQ_JOB(SQB_NAME, SQB_LINK, SQB_TYPE)"
+ " VALUES('" + name + "', 1, '" + type + "')");
}
}
@@ -441,19 +467,19 @@ abstract public class DerbyTestCase {
case 4:
for (String name : new String[]{"JA", "JB", "JC", "JD"}) {
- runQuery("INSERT INTO SQOOP.SQ_JOB(SQB_NAME, SQB_FROM_CONNECTION, SQB_TO_CONNECTION)"
+ runQuery("INSERT INTO SQOOP.SQ_JOB(SQB_NAME, SQB_FROM_LINK, SQB_TO_LINK)"
+ " VALUES('" + name + "', 1, 1)");
}
// Odd IDs inputs have values
for (String ci : new String[]{"1", "2", "3", "4"}) {
- for (String i : new String[]{"5", "9", "17"}) {
+ for (String i : new String[]{"5", "9", "13"}) {
runQuery("INSERT INTO SQOOP.SQ_JOB_INPUT"
+ "(SQBI_JOB, SQBI_INPUT, SQBI_VALUE) "
+ "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
}
- for (String i : new String[]{"7", "11", "19"}) {
+ for (String i : new String[]{"7", "11", "15"}) {
runQuery("INSERT INTO SQOOP.SQ_JOB_INPUT"
+ "(SQBI_JOB, SQBI_INPUT, SQBI_VALUE) "
+ "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
@@ -524,71 +550,70 @@ abstract public class DerbyTestCase {
protected MConnector getConnector() {
return new MConnector("A", "org.apache.sqoop.test.A", "1.0-test",
- getConnectionForms(), getJobForms(), getJobForms());
+ getLinkConfig(), getFromConfig(), getToConfig());
}
-
- protected MDriverConfig getDriverConfig() {
- return new MDriverConfig(getConnectionForms(), getJobForms(),
- Driver.CURRENT_DRIVER_VERSION);
+
+ protected MDriver getDriver() {
+ return new MDriver(getDriverConfig(), DriverBean.CURRENT_DRIVER_VERSION);
}
protected void fillLink(MLink link) {
- List<MForm> forms;
-
- forms = link.getConnectorPart().getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
-
- forms = link.getFrameworkPart().getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value13");
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value15");
+ List<MConfig> configs = link.getConnectorLinkConfig().getConfigs();
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("Value1");
+ ((MStringInput)configs.get(1).getInputs().get(0)).setValue("Value2");
}
protected void fillJob(MJob job) {
- List<MForm> forms;
+ List<MConfig> configs = job.getJobConfig(Direction.FROM).getConfigs();
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("Value1");
+ ((MStringInput)configs.get(1).getInputs().get(0)).setValue("Value2");
- forms = job.getConnectorPart(Direction.FROM).getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
+ configs = job.getJobConfig(Direction.TO).getConfigs();
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("Value1");
+ ((MStringInput)configs.get(1).getInputs().get(0)).setValue("Value2");
+
+ configs = job.getDriverConfig().getConfigs();
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("Value13");
+ ((MStringInput)configs.get(1).getInputs().get(0)).setValue("Value15");
+ }
- forms = job.getConnectorPart(Direction.TO).getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
+ protected MLinkConfig getLinkConfig() {
+ return new MLinkConfig(getConfigs());
+ }
- forms = job.getFrameworkPart().getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value13");
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value15");
+ protected MFromConfig getFromConfig() {
+ return new MFromConfig(getConfigs());
}
- protected MConnectionForms getConnectionForms() {
- return new MConnectionForms(getForms());
+ protected MToConfig getToConfig() {
+ return new MToConfig(getConfigs());
}
- protected MJobForms getJobForms() {
- return new MJobForms(getForms());
+ protected MDriverConfig getDriverConfig() {
+ return new MDriverConfig(getConfigs());
}
- protected List<MForm> getForms() {
- List<MForm> forms = new LinkedList<MForm>();
+ protected List<MConfig> getConfigs() {
+ List<MConfig> jobConfigs = new LinkedList<MConfig>();
- List<MInput<?>> inputs;
- MInput input;
-
- inputs = new LinkedList<MInput<?>>();
- input = new MStringInput("I1", false, (short)30);
+ List<MInput<?>> inputs = new LinkedList<MInput<?>>();
+ MInput input = new MStringInput("I1", false, (short)30);
inputs.add(input);
input = new MMapInput("I2", false);
inputs.add(input);
- forms.add(new MForm("F1", inputs));
+ // adding the from part of the job config
+ jobConfigs.add(new MConfig("C1", inputs));
+ // to
inputs = new LinkedList<MInput<?>>();
input = new MStringInput("I3", false, (short)30);
inputs.add(input);
input = new MMapInput("I4", false);
inputs.add(input);
- forms.add(new MForm("F2", inputs));
+ // adding the to part of the job config
+ jobConfigs.add(new MConfig("C2", inputs));
- return forms;
+ return jobConfigs;
}
/**
@@ -641,8 +666,8 @@ abstract public class DerbyTestCase {
* @throws Exception
*/
protected void generateDatabaseState() throws Exception {
- for(String tbl : new String[] {"SQ_CONNECTOR", "SQ_FORM", "SQ_INPUT",
- "SQ_CONNECTION", "SQ_CONNECTION_INPUT", "SQ_JOB", "SQ_JOB_INPUT"}) {
+ for(String tbl : new String[] {"SQ_CONNECTOR", "SQ_CONFIG", "SQ_INPUT",
+ "SQ_LINK", "SQ_LINK_INPUT", "SQ_JOB", "SQ_JOB_INPUT"}) {
generateTableState("SQOOP." + tbl);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
index 15306c2..a0e8b91 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
@@ -51,7 +51,7 @@ public class TestConnectorHandling extends DerbyTestCase {
assertNull(handler.findConnector("B", getDerbyDatabaseConnection()));
// Load connector into repository
- loadConnectorAndDriverConfig();
+ loadConnectorLinkConfig();
// Retrieve it
MConnector connector = handler.findConnector("A", getDerbyDatabaseConnection());
@@ -69,7 +69,7 @@ public class TestConnectorHandling extends DerbyTestCase {
// No connectors in an empty repository, we expect an empty list
assertEquals(handler.findConnectors(getDerbyDatabaseConnection()).size(),0);
- loadConnectorAndDriverConfig();
+ loadConnectorLinkConfig();
addConnector();
// Retrieve connectors
@@ -93,7 +93,7 @@ public class TestConnectorHandling extends DerbyTestCase {
// Now check content in corresponding tables
assertCountForTable("SQOOP.SQ_CONNECTOR", 1);
- assertCountForTable("SQOOP.SQ_FORM", 6);
+ assertCountForTable("SQOOP.SQ_CONFIG", 6);
assertCountForTable("SQOOP.SQ_INPUT", 12);
// Registered connector should be easily recovered back
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
deleted file mode 100644
index 9b18bd3..0000000
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.repository.derby;
-
-import org.apache.sqoop.driver.Driver;
-import org.apache.sqoop.model.MDriverConfig;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
-/**
- * Test driver config methods on Derby repository.
- */
-public class TestDriverConfigHandling extends DerbyTestCase {
-
- DerbyRepositoryHandler handler;
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
-
- handler = new DerbyRepositoryHandler();
-
- // We always needs schema for this test case
- createSchema();
- }
-
- @Test
- public void testFindDriverConfig() throws Exception {
- // On empty repository, no driverConfig should be there
- assertNull(handler.findDriverConfig(getDerbyDatabaseConnection()));
- // Load Connector and DriverConfig into repository
- loadConnectorAndDriverConfig();
- // Retrieve it
- MDriverConfig driverConfig = handler.findDriverConfig(getDerbyDatabaseConnection());
- assertNotNull(driverConfig);
-
- // Get original structure
- MDriverConfig originalDriverConfig = getDriverConfig();
-
- // And compare them
- assertEquals(originalDriverConfig, driverConfig);
- }
-
- @Test
- public void testRegisterConnector() throws Exception {
- MDriverConfig driverConfig = getDriverConfig();
- handler.registerDriverConfig(driverConfig, getDerbyDatabaseConnection());
-
- // Connector should get persistence ID
- assertEquals(1, driverConfig.getPersistenceId());
-
- // Now check content in corresponding tables
- assertCountForTable("SQOOP.SQ_CONNECTOR", 0);
- assertCountForTable("SQOOP.SQ_FORM", 4);
- assertCountForTable("SQOOP.SQ_INPUT", 8);
-
- // Registered framework should be easily recovered back
- MDriverConfig retrieved = handler.findDriverConfig(getDerbyDatabaseConnection());
- assertNotNull(retrieved);
- assertEquals(driverConfig, retrieved);
- assertEquals(driverConfig.getVersion(), retrieved.getVersion());
- }
-
- private String getDriverVersion() throws Exception {
- final String frameworkVersionQuery =
- "SELECT SQM_VALUE FROM SQOOP.SQ_SYSTEM WHERE SQM_KEY=?";
- String retVal = null;
- PreparedStatement preparedStmt = null;
- ResultSet resultSet = null;
- try {
- preparedStmt =
- getDerbyDatabaseConnection().prepareStatement(frameworkVersionQuery);
- preparedStmt.setString(1, DerbyRepoConstants.SYSKEY_DRIVER_VERSION);
- resultSet = preparedStmt.executeQuery();
- if(resultSet.next())
- retVal = resultSet.getString(1);
- return retVal;
- } finally {
- if(preparedStmt !=null) {
- try {
- preparedStmt.close();
- } catch(SQLException e) {
- }
- }
- if(resultSet != null) {
- try {
- resultSet.close();
- } catch(SQLException e) {
- }
- }
- }
- }
-
- @Test
- public void testDriverVersion() throws Exception {
- handler.registerDriverConfig(getDriverConfig(), getDerbyDatabaseConnection());
-
- final String lowerVersion = Integer.toString(
- Integer.parseInt(Driver.CURRENT_DRIVER_VERSION) - 1);
- assertEquals(Driver.CURRENT_DRIVER_VERSION, getDriverVersion());
- runQuery("UPDATE SQOOP.SQ_SYSTEM SET SQM_VALUE='" + lowerVersion +
- "' WHERE SQM_KEY = '" + DerbyRepoConstants.SYSKEY_DRIVER_VERSION + "'");
- assertEquals(lowerVersion, getDriverVersion());
-
- MDriverConfig framework = getDriverConfig();
- handler.updateDriverConfig(framework, getDerbyDatabaseConnection());
-
- assertEquals(Driver.CURRENT_DRIVER_VERSION, framework.getVersion());
-
- assertEquals(Driver.CURRENT_DRIVER_VERSION, getDriverVersion());
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java
new file mode 100644
index 0000000..d597bd8
--- /dev/null
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverHandling.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.repository.derby;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.MDriver;
+import org.apache.sqoop.model.MDriverConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test driver config methods on Derby repository.
+ */
+public class TestDriverHandling extends DerbyTestCase {
+
+ private static final Object CURRENT_DRIVER_VERSION = "1";
+ DerbyRepositoryHandler handler;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+ }
+
+ @Test
+ public void testFindDriverConfig() throws Exception {
+ // On empty repository, no driverConfig should be there
+ assertNull(handler.findDriver(getDerbyDatabaseConnection()));
+ // Load Connector and DriverConfig into repository
+ loadConnectorLinkConfig();
+ // Retrieve it
+ MDriver driver = handler.findDriver(getDerbyDatabaseConnection());
+ assertNotNull(driver);
+
+ // Get original structure
+ MDriverConfig originalDriverConfig = getDriverConfig();
+ // And compare them
+ assertEquals(originalDriverConfig, driver.getDriverConfig());
+ }
+
+ public void testRegisterDriverAndConnectorConfig() throws Exception {
+ MDriver driver = getDriver();
+ handler.registerDriver(driver, getDerbyDatabaseConnection());
+
+ // Connector should get persistence ID
+ assertEquals(1, driver.getPersistenceId());
+
+ // Now check content in corresponding tables
+ assertCountForTable("SQOOP.SQ_CONNECTOR", 0);
+ assertCountForTable("SQOOP.SQ_CONFIG", 2);
+ assertCountForTable("SQOOP.SQ_INPUT", 4);
+
+ // Registered driver config should be easily recovered back
+ MDriver retrieved = handler.findDriver(getDerbyDatabaseConnection());
+ assertNotNull(retrieved);
+ assertEquals(driver, retrieved);
+ assertEquals(driver.getVersion(), retrieved.getVersion());
+ }
+
+ private String getDriverVersion() throws Exception {
+ final String frameworkVersionQuery =
+ "SELECT SQM_VALUE FROM SQOOP.SQ_SYSTEM WHERE SQM_KEY=?";
+ String retVal = null;
+ PreparedStatement preparedStmt = null;
+ ResultSet resultSet = null;
+ try {
+ preparedStmt =
+ getDerbyDatabaseConnection().prepareStatement(frameworkVersionQuery);
+ preparedStmt.setString(1, DerbyRepoConstants.SYSKEY_DRIVER_VERSION);
+ resultSet = preparedStmt.executeQuery();
+ if(resultSet.next())
+ retVal = resultSet.getString(1);
+ return retVal;
+ } finally {
+ if(preparedStmt !=null) {
+ try {
+ preparedStmt.close();
+ } catch(SQLException e) {
+ }
+ }
+ if(resultSet != null) {
+ try {
+ resultSet.close();
+ } catch(SQLException e) {
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testDriverVersion() throws Exception {
+ MDriver driver = getDriver();
+ handler.registerDriver(driver, getDerbyDatabaseConnection());
+
+ final String lowerVersion = Integer.toString(Integer
+ .parseInt(DriverBean.CURRENT_DRIVER_VERSION) - 1);
+ assertEquals(CURRENT_DRIVER_VERSION, getDriverVersion());
+ runQuery("UPDATE SQOOP.SQ_SYSTEM SET SQM_VALUE='" + lowerVersion + "' WHERE SQM_KEY = '"
+ + DerbyRepoConstants.SYSKEY_DRIVER_VERSION + "'");
+ assertEquals(lowerVersion, getDriverVersion());
+
+ handler.updateDriver(driver, getDerbyDatabaseConnection());
+
+ assertEquals(CURRENT_DRIVER_VERSION, driver.getVersion());
+
+ assertEquals(CURRENT_DRIVER_VERSION, getDriverVersion());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
index d744693..260c2a9 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
@@ -19,11 +19,11 @@ package org.apache.sqoop.repository.derby;
import org.apache.sqoop.model.MBooleanInput;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MEnumInput;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MMapInput;
@@ -86,19 +86,19 @@ public class TestInputTypes extends DerbyTestCase {
@Test
public void testEntityDataSerialization() throws Exception {
MConnector connector = getConnector();
- MDriverConfig driverConfig = getDriverConfig();
+ MDriver driver = getDriver();
// Register objects for everything and our new connector
handler.registerConnector(connector, getDerbyDatabaseConnection());
- handler.registerDriverConfig(driverConfig, getDerbyDatabaseConnection());
+ handler.registerDriver(driver, getDerbyDatabaseConnection());
// Inserted values
Map<String, String> map = new HashMap<String, String>();
map.put("A", "B");
// Connection object with all various values
- MLink link = new MLink(connector.getPersistenceId(), connector.getConnectionForms(), driverConfig.getConnectionForms());
- MConnectionForms forms = link.getConnectorPart();
+ MLink link = new MLink(connector.getPersistenceId(), connector.getLinkConfig());
+ MLinkConfig forms = link.getConnectorLinkConfig();
forms.getStringInput("f.String").setValue("A");
forms.getMapInput("f.Map").setValue(map);
forms.getIntegerInput("f.Integer").setValue(1);
@@ -111,7 +111,7 @@ public class TestInputTypes extends DerbyTestCase {
// Retrieve created link
MLink retrieved = handler.findLink(link.getPersistenceId(), getDerbyDatabaseConnection());
- forms = retrieved.getConnectorPart();
+ forms = retrieved.getConnectorLinkConfig();
assertEquals("A", forms.getStringInput("f.String").getValue());
assertEquals(map, forms.getMapInput("f.Map").getValue());
assertEquals(1, (int)forms.getIntegerInput("f.Integer").getValue());
@@ -125,8 +125,8 @@ public class TestInputTypes extends DerbyTestCase {
* @return Forms with all data types
*/
@Override
- protected List<MForm> getForms() {
- List<MForm> forms = new LinkedList<MForm>();
+ protected List<MConfig> getConfigs() {
+ List<MConfig> forms = new LinkedList<MConfig>();
List<MInput<?>> inputs;
MInput input;
@@ -148,7 +148,7 @@ public class TestInputTypes extends DerbyTestCase {
input = new MEnumInput("f.Enum", false, new String[] {"YES", "NO"});
inputs.add(input);
- forms.add(new MForm("f", inputs));
+ forms.add(new MConfig("f", inputs));
return forms;
}
}
[43/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java
index 8b23144..12ceb21 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.connector.jdbc.GenericJdbcConnectorConstants;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
@@ -28,7 +28,7 @@ import org.apache.sqoop.validation.validators.NullOrContains;
/**
*
*/
-@FormClass( validators = {@Validator(FromJobConfig.FormValidator.class)})
+@ConfigClass( validators = {@Validator(FromJobConfig.ConfigValidator.class)})
public class FromJobConfig {
@Input(size = 50)
public String schemaName;
@@ -51,16 +51,16 @@ public class FromJobConfig {
@Input(size = 50)
public String boundaryQuery;
- public static class FormValidator extends AbstractValidator<FromJobConfig> {
+ public static class ConfigValidator extends AbstractValidator<FromJobConfig> {
@Override
- public void validate(FromJobConfig form) {
- if(form.tableName == null && form.sql == null) {
+ public void validate(FromJobConfig config) {
+ if(config.tableName == null && config.sql == null) {
addMessage(Status.UNACCEPTABLE, "Either table name or SQL must be specified");
}
- if(form.tableName != null && form.sql != null) {
+ if(config.tableName != null && config.sql != null) {
addMessage(Status.UNACCEPTABLE, "Both table name and SQL cannot be specified");
}
- if(form.schemaName != null && form.sql != null) {
+ if(config.schemaName != null && config.sql != null) {
addMessage(Status.UNACCEPTABLE, "Both schema name and SQL cannot be specified");
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
index b036421..39e8edd 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
@@ -18,14 +18,14 @@
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
+import org.apache.sqoop.model.Config;
/**
*
*/
@ConfigurationClass
public class FromJobConfiguration {
- @Form public FromJobConfig fromJobConfig;
+ @Config public FromJobConfig fromJobConfig;
public FromJobConfiguration() {
fromJobConfig = new FromJobConfig();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java
index 7b76eeb..be86855 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java
@@ -17,7 +17,7 @@
*/
package org.apache.sqoop.connector.jdbc.configuration;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
@@ -33,7 +33,7 @@ import java.util.Map;
/**
*
*/
-@FormClass(validators = {@Validator(LinkConfig.FormValidator.class)})
+@ConfigClass(validators = {@Validator(LinkConfig.ConfigValidator.class)})
public class LinkConfig {
@Input(size = 128, validators = {@Validator(NotEmpty.class), @Validator(ClassAvailable.class)} )
public String jdbcDriver;
@@ -50,7 +50,7 @@ public class LinkConfig {
@Input
public Map<String, String> jdbcProperties;
- public static class FormValidator extends AbstractValidator<LinkConfig> {
+ public static class ConfigValidator extends AbstractValidator<LinkConfig> {
@Override
public void validate(LinkConfig linkConfig) {
// See if we can connect to the database
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java
index 7d614f7..ed55bff 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
+import org.apache.sqoop.model.Config;
/**
*
@@ -26,9 +26,9 @@ import org.apache.sqoop.model.Form;
@ConfigurationClass
public class LinkConfiguration {
- @Form public LinkConfig link;
+ @Config public LinkConfig linkConfig;
public LinkConfiguration() {
- link = new LinkConfig();
+ linkConfig = new LinkConfig();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
index a42a6ec..2428601 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
@@ -17,7 +17,7 @@
*/
package org.apache.sqoop.connector.jdbc.configuration;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
@@ -26,7 +26,7 @@ import org.apache.sqoop.validation.validators.AbstractValidator;
/**
*
*/
-@FormClass(validators = {@Validator(ToJobConfig.FormValidator.class)})
+@ConfigClass(validators = {@Validator(ToJobConfig.ConfigValidator.class)})
public class ToJobConfig {
@Input(size = 50) public String schemaName;
@Input(size = 2000) public String tableName;
@@ -35,19 +35,19 @@ public class ToJobConfig {
@Input(size = 2000) public String stageTableName;
@Input public Boolean clearStageTable;
- public static class FormValidator extends AbstractValidator<ToJobConfig> {
+ public static class ConfigValidator extends AbstractValidator<ToJobConfig> {
@Override
- public void validate(ToJobConfig form) {
- if(form.tableName == null && form.sql == null) {
+ public void validate(ToJobConfig config) {
+ if(config.tableName == null && config.sql == null) {
addMessage(Status.UNACCEPTABLE, "Either table name or SQL must be specified");
}
- if(form.tableName != null && form.sql != null) {
+ if(config.tableName != null && config.sql != null) {
addMessage(Status.UNACCEPTABLE, "Both table name and SQL cannot be specified");
}
- if(form.tableName == null && form.stageTableName != null) {
+ if(config.tableName == null && config.stageTableName != null) {
addMessage(Status.UNACCEPTABLE, "Stage table name cannot be specified without specifying table name");
}
- if(form.stageTableName == null && form.clearStageTable != null) {
+ if(config.stageTableName == null && config.clearStageTable != null) {
addMessage(Status.UNACCEPTABLE, "Clear stage table cannot be specified without specifying name of the stage table.");
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
index ad68681..fd5d54b 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
@@ -18,14 +18,14 @@
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
+import org.apache.sqoop.model.Config;
/**
*
*/
@ConfigurationClass
public class ToJobConfiguration {
- @Form public ToJobConfig toJobConfig;
+ @Config public ToJobConfig toJobConfig;
public ToJobConfiguration() {
toJobConfig = new ToJobConfig();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties b/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties
index 921ac28..c535e9b 100644
--- a/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties
+++ b/connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties
@@ -18,33 +18,33 @@
############################
# Link Config
#
-link.label = Link configuration
-link.help = You must supply the information requested in order to \
+linkConfig.label = Link configuration
+linkConfig.help = You must supply the information requested in order to \
create a connection object.
# jdbc driver
-link.jdbcDriver.label = JDBC Driver Class
-link.jdbcDriver.help = Enter the fully qualified class name of the JDBC \
+linkConfig.jdbcDriver.label = JDBC Driver Class
+linkConfig.jdbcDriver.help = Enter the fully qualified class name of the JDBC \
driver that will be used for establishing this connection.
# connect string
-link.connectionString.label = JDBC Connection String
-link.connectionString.help = Enter the value of JDBC connection string to be \
+linkConfig.connectionString.label = JDBC Connection String
+linkConfig.connectionString.help = Enter the value of JDBC connection string to be \
used by this connector for creating connections.
# username string
-link.username.label = Username
-link.username.help = Enter the username to be used for connecting to the \
+linkConfig.username.label = Username
+linkConfig.username.help = Enter the username to be used for connecting to the \
database.
# password string
-link.password.label = Password
-link.password.help = Enter the password to be used for connecting to the \
+linkConfig.password.label = Password
+linkConfig.password.help = Enter the password to be used for connecting to the \
database.
# jdbc properties
-link.jdbcProperties.label = JDBC Connection Properties
-link.jdbcProperties.help = Enter any JDBC properties that should be \
+linkConfig.jdbcProperties.label = JDBC Connection Properties
+linkConfig.jdbcProperties.help = Enter any JDBC properties that should be \
supplied during the creation of connection.
# From Job Config
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
index 1022370..61846b7 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
@@ -60,7 +60,6 @@ public class GenericJdbcExecutorTest {
}
@Test
- @SuppressWarnings("unchecked")
public void testDeleteTableData() throws Exception {
executor.deleteTableData(table);
assertEquals("Table " + table + " is expected to be empty.",
@@ -68,7 +67,6 @@ public class GenericJdbcExecutorTest {
}
@Test
- @SuppressWarnings("unchecked")
public void testMigrateData() throws Exception {
assertEquals("Table " + emptyTable + " is expected to be empty.",
0, executor.getTableRowCount(emptyTable));
@@ -86,7 +84,6 @@ public class GenericJdbcExecutorTest {
}
@Test
- @SuppressWarnings("unchecked")
public void testGetTableRowCount() throws Exception {
assertEquals("Table " + table + " is expected to be empty.",
NUMBER_OF_ROWS, executor.getTableRowCount(table));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
index fbdf9c6..d1e6805 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
@@ -74,10 +74,10 @@ public class TestExtractor {
public void testQuery() throws Exception {
MutableContext context = new MutableMapContext();
- LinkConfiguration connectionConfig = new LinkConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
- connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
@@ -92,25 +92,25 @@ public class TestExtractor {
partition = new GenericJdbcPartition();
partition.setConditions("-50.0 <= DCOL AND DCOL < -16.6666666666666665");
- extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
+ extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("-16.6666666666666665 <= DCOL AND DCOL < 16.666666666666667");
- extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
+ extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("16.666666666666667 <= DCOL AND DCOL <= 50.0");
- extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
+ extractor.extract(extractorContext, linkConfig, jobConfig, partition);
}
@Test
public void testSubquery() throws Exception {
MutableContext context = new MutableMapContext();
- LinkConfiguration connectionConfig = new LinkConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
- connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
@@ -127,15 +127,15 @@ public class TestExtractor {
partition = new GenericJdbcPartition();
partition.setConditions("-50 <= ICOL AND ICOL < -16");
- extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
+ extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("-16 <= ICOL AND ICOL < 17");
- extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
+ extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("17 <= ICOL AND ICOL < 50");
- extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
+ extractor.extract(extractorContext, linkConfig, jobConfig, partition);
}
public class DummyWriter extends DataWriter {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
index 54e6acf..345fe9b 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
@@ -116,19 +116,19 @@ public class TestFromInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableName() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.tableName = schemalessTableName;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.tableName = schemalessTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT * FROM " + executor.delimitIdentifier(schemalessTableName)
@@ -143,20 +143,20 @@ public class TestFromInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumns() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.tableName = schemalessTableName;
- jobConf.fromJobConfig.columns = tableColumns;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.tableName = schemalessTableName;
+ jobConfig.fromJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT ICOL,VCOL FROM " + executor.delimitIdentifier(schemalessTableName)
@@ -171,20 +171,20 @@ public class TestFromInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableSql() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.sql = schemalessTableSql;
- jobConf.fromJobConfig.partitionColumn = "DCOL";
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.sql = schemalessTableSql;
+ jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT * FROM " + executor.delimitIdentifier(schemalessTableName)
@@ -199,21 +199,21 @@ public class TestFromInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableSqlWithTableColumns() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.sql = schemalessTableSql;
- jobConf.fromJobConfig.columns = tableColumns;
- jobConf.fromJobConfig.partitionColumn = "DCOL";
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.sql = schemalessTableSql;
+ jobConfig.fromJobConfig.columns = tableColumns;
+ jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM "
@@ -229,22 +229,22 @@ public class TestFromInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithSchema() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.schemaName = schemaName;
- jobConf.fromJobConfig.tableName = tableName;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.schemaName = schemaName;
+ jobConfig.fromJobConfig.tableName = tableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT * FROM " + fullTableName
@@ -259,23 +259,23 @@ public class TestFromInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumnsWithSchema() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.schemaName = schemaName;
- jobConf.fromJobConfig.tableName = tableName;
- jobConf.fromJobConfig.columns = tableColumns;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.schemaName = schemaName;
+ jobConfig.fromJobConfig.tableName = tableName;
+ jobConfig.fromJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT ICOL,VCOL FROM " + fullTableName
@@ -290,23 +290,23 @@ public class TestFromInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableSqlWithSchema() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.schemaName = schemaName;
- jobConf.fromJobConfig.sql = tableSql;
- jobConf.fromJobConfig.partitionColumn = "DCOL";
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.schemaName = schemaName;
+ jobConfig.fromJobConfig.sql = tableSql;
+ jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT * FROM " + fullTableName
@@ -321,68 +321,68 @@ public class TestFromInitializer {
@Test
@SuppressWarnings("unchecked")
public void testGetSchemaForTable() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.schemaName = schemaName;
- jobConf.fromJobConfig.tableName = tableName;
- jobConf.fromJobConfig.partitionColumn = "DCOL";
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.schemaName = schemaName;
+ jobConfig.fromJobConfig.tableName = tableName;
+ jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
- Schema schema = initializer.getSchema(initializerContext, connConf, jobConf);
- assertEquals(getSchema(jobConf.fromJobConfig.schemaName + "." + tableName), schema);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
+ Schema schema = initializer.getSchema(initializerContext, linkConfig, jobConfig);
+ assertEquals(getSchema(jobConfig.fromJobConfig.schemaName + "." + tableName), schema);
}
@Test
@SuppressWarnings("unchecked")
public void testGetSchemaForSql() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.schemaName = schemaName;
- jobConf.fromJobConfig.sql = tableSql;
- jobConf.fromJobConfig.partitionColumn = "DCOL";
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.schemaName = schemaName;
+ jobConfig.fromJobConfig.sql = tableSql;
+ jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
- Schema schema = initializer.getSchema(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
+ Schema schema = initializer.getSchema(initializerContext, linkConfig, jobConfig);
assertEquals(getSchema("Query"), schema);
}
@Test
@SuppressWarnings("unchecked")
public void testTableSqlWithTableColumnsWithSchema() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.fromJobConfig.schemaName = schemaName;
- jobConf.fromJobConfig.sql = tableSql;
- jobConf.fromJobConfig.columns = tableColumns;
- jobConf.fromJobConfig.partitionColumn = "DCOL";
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.fromJobConfig.schemaName = schemaName;
+ jobConfig.fromJobConfig.sql = tableSql;
+ jobConfig.fromJobConfig.columns = tableColumns;
+ jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM "
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
index 144b92a..538b033 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
@@ -82,10 +82,10 @@ public class TestLoader {
public void testInsert() throws Exception {
MutableContext context = new MutableMapContext();
- LinkConfiguration connectionConfig = new LinkConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
- connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
ToJobConfiguration jobConfig = new ToJobConfiguration();
@@ -95,7 +95,7 @@ public class TestLoader {
Loader loader = new GenericJdbcLoader();
DummyReader reader = new DummyReader();
LoaderContext loaderContext = new LoaderContext(context, reader, null);
- loader.load(loaderContext, connectionConfig, jobConfig);
+ loader.load(loaderContext, linkConfig, jobConfig);
int index = START;
ResultSet rs = executor.executeQuery("SELECT * FROM "
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
index ec75e1e..3ae64f0 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
@@ -58,12 +58,12 @@ public class TestPartitioner {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5 <= ICOL AND ICOL < -3",
@@ -90,12 +90,12 @@ public class TestPartitioner {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5 <= ICOL AND ICOL < -1",
@@ -120,12 +120,12 @@ public class TestPartitioner {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 13, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5 <= ICOL AND ICOL < -4",
@@ -157,12 +157,12 @@ public class TestPartitioner {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf((double)(START + NUMBER_OF_ROWS - 1)));
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5.0 <= DCOL AND DCOL < -3.0",
@@ -189,12 +189,12 @@ public class TestPartitioner {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf((double)(START + NUMBER_OF_ROWS - 1)));
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5.0 <= DCOL AND DCOL < -1.6666666666666665",
@@ -211,12 +211,12 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(START));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(START + NUMBER_OF_ROWS - 1));
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5 <= ICOL AND ICOL < -3",
@@ -235,12 +235,12 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(new BigDecimal(START)));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(new BigDecimal(START + NUMBER_OF_ROWS - 1)));
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"-5 <= DCOL AND DCOL < -2",
@@ -257,12 +257,12 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(new BigDecimal(START)));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(new BigDecimal(START)));
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"DCOL = -5",
@@ -282,12 +282,12 @@ public class TestPartitioner {
.toString());
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
@@ -311,12 +311,12 @@ public class TestPartitioner {
Time.valueOf("10:40:50").toString());
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"'01:01:01' <= TCOL AND TCOL < '04:14:17'",
@@ -337,12 +337,12 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
Timestamp.valueOf("2013-12-31 10:40:50.654").toString());
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 12:14:17.634'",
"'2013-05-02 12:14:17.634' <= TSCOL AND TSCOL < '2013-08-31 23:27:34.144'",
@@ -362,12 +362,12 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "1");
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"BCOL = TRUE",
"BCOL = FALSE",
@@ -386,12 +386,12 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "Z");
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 25, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"'A' <= VCCOL AND VCCOL < 'B'",
@@ -434,11 +434,11 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "Warty Warthog");
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
assertEquals(partitions.size(), 5);
// First partition needs to contain entire upper bound
assertTrue(partitions.get(0).toString().contains("Breezy Badger"));
@@ -458,13 +458,13 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "AAF");
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"'AAA' <= VCCOL AND VCCOL < 'AAB'",
@@ -488,14 +488,14 @@ public class TestPartitioner {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "AAE");
- LinkConfiguration connConf = new LinkConfiguration();
- FromJobConfiguration jobConf = new FromJobConfiguration();
- jobConf.fromJobConfig.partitionColumnNull = true;
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ FromJobConfiguration jobConfig = new FromJobConfiguration();
+ jobConfig.fromJobConfig.partitionColumnNull = true;
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
- List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
+ List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"VCCOL IS NULL",
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
index a87ce7a..243de01 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
@@ -17,6 +17,10 @@
*/
package org.apache.sqoop.connector.jdbc;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.common.SqoopException;
@@ -24,17 +28,13 @@ import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.job.etl.Initializer;
import org.apache.sqoop.job.etl.InitializerContext;
+import org.apache.sqoop.validation.ConfigValidationResult;
+import org.apache.sqoop.validation.ConfigValidationRunner;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.ValidationResult;
-import org.apache.sqoop.validation.ValidationRunner;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
public class TestToInitializer {
private final String schemaName;
private final String tableName;
@@ -82,21 +82,21 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableName() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemalessTableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.tableName = schemalessTableName;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.tableName = schemalessTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullTableName + " VALUES (?,?,?)");
}
@@ -104,22 +104,22 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumns() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemalessTableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.tableName = schemalessTableName;
- jobConf.toJobConfig.columns = tableColumns;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.tableName = schemalessTableName;
+ jobConfig.toJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullTableName + " (" + tableColumns + ") VALUES (?,?)");
}
@@ -127,19 +127,19 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableSql() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.sql = schemalessTableSql;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.sql = schemalessTableSql;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + executor.delimitIdentifier(schemalessTableName) + " VALUES (?,?,?)");
}
@@ -147,22 +147,22 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithSchema() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.schemaName = schemaName;
- jobConf.toJobConfig.tableName = tableName;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.schemaName = schemaName;
+ jobConfig.toJobConfig.tableName = tableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullTableName + " VALUES (?,?,?)");
}
@@ -170,23 +170,23 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumnsWithSchema() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.schemaName = schemaName;
- jobConf.toJobConfig.tableName = tableName;
- jobConf.toJobConfig.columns = tableColumns;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.schemaName = schemaName;
+ jobConfig.toJobConfig.tableName = tableName;
+ jobConfig.toJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullTableName + " (" + tableColumns + ") VALUES (?,?)");
}
@@ -194,20 +194,20 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testTableSqlWithSchema() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.schemaName = schemaName;
- jobConf.toJobConfig.sql = tableSql;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.schemaName = schemaName;
+ jobConfig.toJobConfig.sql = tableSql;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + executor.delimitIdentifier(tableName) + " VALUES (?,?,?)");
}
@@ -229,13 +229,13 @@ public class TestToInitializer {
@Test
public void testNonExistingStageTable() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.tableName = schemalessTableName;
- jobConf.toJobConfig.stageTableName = stageTableName;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.tableName = schemalessTableName;
+ jobConfig.toJobConfig.stageTableName = stageTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@@ -243,7 +243,7 @@ public class TestToInitializer {
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
try {
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
fail("Initialization should fail for non-existing stage table.");
} catch(SqoopException se) {
//expected
@@ -253,15 +253,15 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testNonEmptyStageTable() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.tableName = schemalessTableName;
- jobConf.toJobConfig.stageTableName = stageTableName;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.tableName = schemalessTableName;
+ jobConfig.toJobConfig.stageTableName = stageTableName;
createTable(fullStageTableName);
executor.executeUpdate("INSERT INTO " + fullStageTableName +
" VALUES(1, 1.1, 'one')");
@@ -271,7 +271,7 @@ public class TestToInitializer {
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
try {
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
fail("Initialization should fail for non-empty stage table.");
} catch(SqoopException se) {
//expected
@@ -280,17 +280,17 @@ public class TestToInitializer {
@Test
public void testClearStageTableValidation() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
//specifying clear stage table flag without specifying name of
// the stage table
- jobConf.toJobConfig.tableName = schemalessTableName;
- jobConf.toJobConfig.clearStageTable = false;
- ValidationRunner validationRunner = new ValidationRunner();
- ValidationResult result = validationRunner.validate(jobConf);
+ jobConfig.toJobConfig.tableName = schemalessTableName;
+ jobConfig.toJobConfig.clearStageTable = false;
+ ConfigValidationRunner validationRunner = new ConfigValidationRunner();
+ ConfigValidationResult result = validationRunner.validate(jobConfig);
assertEquals("User should not specify clear stage table flag without " +
"specifying name of the stage table",
Status.UNACCEPTABLE,
@@ -298,8 +298,8 @@ public class TestToInitializer {
assertTrue(result.getMessages().containsKey(
"toJobConfig"));
- jobConf.toJobConfig.clearStageTable = true;
- result = validationRunner.validate(jobConf);
+ jobConfig.toJobConfig.clearStageTable = true;
+ result = validationRunner.validate(jobConfig);
assertEquals("User should not specify clear stage table flag without " +
"specifying name of the stage table",
Status.UNACCEPTABLE,
@@ -310,17 +310,17 @@ public class TestToInitializer {
@Test
public void testStageTableWithoutTable() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
//specifying stage table without specifying table name
- jobConf.toJobConfig.stageTableName = stageTableName;
- jobConf.toJobConfig.sql = "";
+ jobConfig.toJobConfig.stageTableName = stageTableName;
+ jobConfig.toJobConfig.sql = "";
- ValidationRunner validationRunner = new ValidationRunner();
- ValidationResult result = validationRunner.validate(jobConf);
+ ConfigValidationRunner validationRunner = new ConfigValidationRunner();
+ ConfigValidationResult result = validationRunner.validate(jobConfig);
assertEquals("Stage table name cannot be specified without specifying " +
"table name", Status.UNACCEPTABLE, result.getStatus());
assertTrue(result.getMessages().containsKey(
@@ -330,16 +330,16 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testClearStageTable() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.tableName = schemalessTableName;
- jobConf.toJobConfig.stageTableName = stageTableName;
- jobConf.toJobConfig.clearStageTable = true;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.tableName = schemalessTableName;
+ jobConfig.toJobConfig.stageTableName = stageTableName;
+ jobConfig.toJobConfig.clearStageTable = true;
createTable(fullStageTableName);
executor.executeUpdate("INSERT INTO " + fullStageTableName +
" VALUES(1, 1.1, 'one')");
@@ -348,7 +348,7 @@ public class TestToInitializer {
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
assertEquals("Stage table should have been cleared", 0,
executor.getTableRowCount(stageTableName));
}
@@ -356,22 +356,22 @@ public class TestToInitializer {
@Test
@SuppressWarnings("unchecked")
public void testStageTable() throws Exception {
- LinkConfiguration connConf = new LinkConfiguration();
- ToJobConfiguration jobConf = new ToJobConfiguration();
+ LinkConfiguration linkConfig = new LinkConfiguration();
+ ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
- connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
- connConf.link.connectionString = GenericJdbcTestConstants.URL;
- jobConf.toJobConfig.tableName = schemalessTableName;
- jobConf.toJobConfig.stageTableName = stageTableName;
+ linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+ linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
+ jobConfig.toJobConfig.tableName = schemalessTableName;
+ jobConfig.toJobConfig.stageTableName = stageTableName;
createTable(fullStageTableName);
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
- initializer.initialize(initializerContext, connConf, jobConf);
+ initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullStageTableName +
" VALUES (?,?,?)");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java
index 47b186c..b17aa21 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java
@@ -18,21 +18,20 @@
*/
package org.apache.sqoop.connector.hdfs;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJobForms;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import org.apache.sqoop.model.MLinkConfig;
public class HdfsConfigUpgrader extends RepositoryUpgrader {
- private static final Logger LOG =
- Logger.getLogger(HdfsConfigUpgrader.class);
+ private static final Logger LOG = Logger.getLogger(HdfsConfigUpgrader.class);
/*
* For now, there is no real upgrade. So copy all data over,
@@ -41,37 +40,36 @@ public class HdfsConfigUpgrader extends RepositoryUpgrader {
*/
@Override
- public void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
+ public void upgrade(MLinkConfig original, MLinkConfig upgradeTarget) {
+ doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@Override
- public void upgrade(MJobForms original, MJobForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
+ public void upgrade(MConfigList original, MConfigList upgradeTarget) {
+ doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@SuppressWarnings("unchecked")
- private void doUpgrade(List<MForm> original, List<MForm> target) {
- // Easier to find the form in the original forms list if we use a map.
- // Since the constructor of MJobForms takes a list,
+ private void doUpgrade(List<MConfig> original, List<MConfig> target) {
+ // Easier to find the config in the original list if we use a map.
+ // Since the constructor takes a list,
// index is not guaranteed to be the same, so we need to look for
// equivalence
- Map<String, MForm> formMap = new HashMap<String, MForm>();
- for (MForm form : original) {
- formMap.put(form.getName(), form);
+ Map<String, MConfig> configMap = new HashMap<String, MConfig>();
+ for (MConfig config : original) {
+ configMap.put(config.getName(), config);
}
- for (MForm form : target) {
- List<MInput<?>> inputs = form.getInputs();
- MForm originalForm = formMap.get(form.getName());
- if (originalForm == null) {
- LOG.warn("Form: '" + form.getName() + "' not present in old " +
+ for (MConfig config : target) {
+ List<MInput<?>> inputs = config.getInputs();
+ MConfig originalConfig = configMap.get(config.getName());
+ if (originalConfig == null) {
+ LOG.warn("Config: '" + config.getName() + "' not present in old " +
"connector. So it and its inputs will not be transferred by the upgrader.");
continue;
}
for (MInput input : inputs) {
try {
- MInput originalInput = originalForm.getInput(input.getName());
+ MInput originalInput = originalConfig.getInput(input.getName());
input.setValue(originalInput.getValue());
} catch (SqoopException ex) {
LOG.warn("Input: '" + input.getName() + "' not present in old " +
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
index cd5350e..606b9fa 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
@@ -118,7 +118,7 @@ public class HdfsConnector extends SqoopConnector {
* @return Validator object
*/
@Override
- public Validator getValidator() {
+ public Validator getConfigValidator() {
return hdfsValidator;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
index 436d243..2c8b6c8 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
@@ -52,8 +52,8 @@ public class HdfsExtractor extends Extractor<LinkConfiguration, FromJobConfigura
@Override
public void extract(ExtractorContext context,
- LinkConfiguration connectionConfiguration,
- FromJobConfiguration jobConfiguration, HdfsPartition partition) {
+ LinkConfiguration linkConfig,
+ FromJobConfiguration fromJobConfig, HdfsPartition partition) {
conf = ((PrefixContext) context.getContext()).getConfiguration();
dataWriter = context.getDataWriter();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
index c2dc1a5..bb5e353 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
@@ -29,16 +29,16 @@ public class HdfsInitializer extends Initializer {
* promoted to all other part of the workflow automatically.
*
* @param context Initializer context object
- * @param linkConf Connector's link configuration object
+ * @param linkConfig Connector's link configuration object
* @param jobConf Connector's job configuration object
*/
@Override
- public void initialize(InitializerContext context, Object linkConf, Object jobConf) {
+ public void initialize(InitializerContext context, Object linkConfig, Object jobConf) {
}
@Override
- public Schema getSchema(InitializerContext context, Object linkConf, Object jobConf) {
+ public Schema getSchema(InitializerContext context, Object linkConfig, Object jobConfig) {
return new Schema("HDFS file");
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
index 4c546ba..660418d 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
@@ -42,19 +42,19 @@ public class HdfsLoader extends Loader<LinkConfiguration, ToJobConfiguration> {
* Load data to target.
*
* @param context Loader context object
- * @param linkConf Link configuration
- * @param toJobConf Job configuration
+ * @param linkConfig Link configuration
+ * @param toJobConfig Job configuration
* @throws Exception
*/
@Override
- public void load(LoaderContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) throws Exception {
+ public void load(LoaderContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) throws Exception {
DataReader reader = context.getDataReader();
Configuration conf = ((PrefixContext)context.getContext()).getConfiguration();
- String directoryName = toJobConf.toJobConfig.outputDirectory;
- String codecname = getCompressionCodecName(toJobConf);
+ String directoryName = toJobConfig.toJobConfig.outputDirectory;
+ String codecname = getCompressionCodecName(toJobConfig);
CompressionCodec codec = null;
if (codecname != null) {
@@ -73,12 +73,12 @@ public class HdfsLoader extends Loader<LinkConfiguration, ToJobConfiguration> {
}
}
- String filename = directoryName + "/" + UUID.randomUUID() + getExtension(toJobConf,codec);
+ String filename = directoryName + "/" + UUID.randomUUID() + getExtension(toJobConfig,codec);
try {
Path filepath = new Path(filename);
- GenericHdfsWriter filewriter = getWriter(toJobConf);
+ GenericHdfsWriter filewriter = getWriter(toJobConfig);
filewriter.initialize(filepath,conf,codec);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
index 6828de8..f40459f 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
@@ -68,12 +68,12 @@ public class HdfsPartitioner extends Partitioner<LinkConfiguration, FromJobConfi
@Override
public List<Partition> getPartitions(PartitionerContext context,
- LinkConfiguration linkConfiguration, FromJobConfiguration jobConfiguration) {
+ LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
Configuration conf = ((PrefixContext)context.getContext()).getConfiguration();
try {
- long numInputBytes = getInputSize(conf, jobConfiguration.fromJobConfig.inputDirectory);
+ long numInputBytes = getInputSize(conf, fromJobConfig.fromJobConfig.inputDirectory);
maxSplitSize = numInputBytes / context.getMaxPartitions();
if(numInputBytes % context.getMaxPartitions() != 0 ) {
@@ -118,7 +118,7 @@ public class HdfsPartitioner extends Partitioner<LinkConfiguration, FromJobConfi
}
// all the files in input set
- String indir = jobConfiguration.fromJobConfig.inputDirectory;
+ String indir = fromJobConfig.fromJobConfig.inputDirectory;
FileSystem fs = FileSystem.get(conf);
List<Path> paths = new LinkedList<Path>();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
index dfa3659..b995efd 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.connector.hdfs;
import org.apache.sqoop.connector.hdfs.configuration.*;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
+import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Validator;
/**
@@ -28,54 +28,45 @@ import org.apache.sqoop.validation.Validator;
public class HdfsValidator extends Validator {
@Override
- public Validation validateLink(Object connectionConfiguration) {
- Validation validation = new Validation(LinkConfiguration.class);
- // No validation on connection object
- return validation;
- }
-
-
- @Override
- public Validation validateJob(Object jobConfiguration) {
- //TODO: I'm pretty sure this needs to call either validateExportJob or validateImportJob, depending on context
- return super.validateJob(jobConfiguration);
+ public ConfigValidator validateConfigForJob(Object jobConfiguration) {
+ return super.validateConfigForJob(jobConfiguration);
}
@SuppressWarnings("unused")
- private Validation validateFromJob(Object jobConfiguration) {
- Validation validation = new Validation(FromJobConfiguration.class);
+ private ConfigValidator validateFromJob(Object jobConfiguration) {
+ ConfigValidator validation = new ConfigValidator(FromJobConfiguration.class);
FromJobConfiguration configuration = (FromJobConfiguration)jobConfiguration;
- validateInputForm(validation, configuration.fromJobConfig);
+ validateInputConfig(validation, configuration.fromJobConfig);
return validation;
}
@SuppressWarnings("unused")
- private Validation validateToJob(Object jobConfiguration) {
- Validation validation = new Validation(ToJobConfiguration.class);
+ private ConfigValidator validateToJob(Object jobConfiguration) {
+ ConfigValidator validation = new ConfigValidator(ToJobConfiguration.class);
ToJobConfiguration configuration = (ToJobConfiguration)jobConfiguration;
- validateOutputForm(validation, configuration.toJobConfig);
+ validateOutputConfig(validation, configuration.toJobConfig);
return validation;
}
- private void validateInputForm(Validation validation, FromJobConfig input) {
- if(input.inputDirectory == null || input.inputDirectory.isEmpty()) {
+ private void validateInputConfig(ConfigValidator validation, FromJobConfig inputConfig) {
+ if(inputConfig.inputDirectory == null || inputConfig.inputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "input", "inputDirectory", "Input directory is empty");
}
}
- private void validateOutputForm(Validation validation, ToJobConfig output) {
- if(output.outputDirectory == null || output.outputDirectory.isEmpty()) {
+ private void validateOutputConfig(ConfigValidator validation, ToJobConfig outputConfig) {
+ if(outputConfig.outputDirectory == null || outputConfig.outputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "output", "outputDirectory", "Output directory is empty");
}
- if(output.customCompression != null &&
- output.customCompression.trim().length() > 0 &&
- output.compression != ToCompression.CUSTOM) {
+ if(outputConfig.customCompression != null &&
+ outputConfig.customCompression.trim().length() > 0 &&
+ outputConfig.compression != ToCompression.CUSTOM) {
validation.addMessage(Status.UNACCEPTABLE, "output", "compression",
- "custom compression should be blank as " + output.compression + " is being used.");
+ "custom compression should be blank as " + outputConfig.compression + " is being used.");
}
- if(output.compression == ToCompression.CUSTOM &&
- (output.customCompression == null ||
- output.customCompression.trim().length() == 0)
+ if(outputConfig.compression == ToCompression.CUSTOM &&
+ (outputConfig.customCompression == null ||
+ outputConfig.customCompression.trim().length() == 0)
) {
validation.addMessage(Status.UNACCEPTABLE, "output", "compression",
"custom compression is blank.");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java
index 2c98051..037fe59 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.connector.hdfs.configuration;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
/**
*
*/
-@FormClass
+@ConfigClass
public class FromJobConfig {
@Input(size = 255) public String inputDirectory;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
index f861237..618366e 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
@@ -18,11 +18,11 @@
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
+import org.apache.sqoop.model.Config;
@ConfigurationClass
public class FromJobConfiguration {
- @Form public FromJobConfig fromJobConfig;
+ @Config public FromJobConfig fromJobConfig;
public FromJobConfiguration() {
fromJobConfig = new FromJobConfig();
[44/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConnector.java b/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
index d8bc94d..89ae440 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
@@ -17,6 +17,13 @@
*/
package org.apache.sqoop.model;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -24,39 +31,34 @@ import java.util.List;
import org.apache.sqoop.common.Direction;
import org.junit.Test;
-import static org.junit.Assert.*;
-
-/**
- * Test class for org.apache.sqoop.model.TestMConnector
- */
public class TestMConnector {
private MConnector createConnector(List<Direction> supportedDirections) {
- List<MForm> forms = new ArrayList<MForm>();
- MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
- input.setValue(100);
+ List<MConfig> configs = new ArrayList<MConfig>();
+ MIntegerInput inputs = new MIntegerInput("INTEGER-INPUT", false);
+ inputs.setValue(100);
MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
strInput.setValue("TEST-VALUE");
List<MInput<?>> list = new ArrayList<MInput<?>>();
- list.add(input);
+ list.add(inputs);
list.add(strInput);
- MForm form = new MForm("FORMNAME", list);
- forms.add(form);
+ MConfig config = new MConfig("CONFIGNAME", list);
+ configs.add(config);
- MConnectionForms connectionForms1 = new MConnectionForms(forms);
- MJobForms fromForm = null;
- MJobForms toForm = null;
+ MLinkConfig linkConfig = new MLinkConfig(configs);
+ MFromConfig fromConfig = null;
+ MToConfig toConfig = null;
if (supportedDirections.contains(Direction.FROM)) {
- fromForm = new MJobForms(forms);
+ fromConfig = new MFromConfig(configs);
}
if (supportedDirections.contains(Direction.TO)) {
- toForm = new MJobForms(forms);
+ toConfig = new MToConfig(configs);
}
return new MConnector("NAME", "CLASSNAME", "1.0",
- connectionForms1, fromForm, toForm);
+ linkConfig, fromConfig, toConfig);
}
/**
@@ -64,32 +66,32 @@ public class TestMConnector {
*/
@Test
public void testInitialization() {
- List<MForm> fromJobForms = new ArrayList<MForm>();
- List<MForm> toJobForms = new ArrayList<MForm>();
- MConnectionForms connectionForms1 = new MConnectionForms(fromJobForms);
- MJobForms fromJobForm1 = new MJobForms(fromJobForms);
- MJobForms toJobForm1 = new MJobForms(toJobForms);
+ List<MConfig> fromJobConfig = new ArrayList<MConfig>();
+ List<MConfig> toJobConfig = new ArrayList<MConfig>();
+ MLinkConfig linkConfig = new MLinkConfig(fromJobConfig);
+ MFromConfig fromConfig1 = new MFromConfig(fromJobConfig);
+ MToConfig toConfig1 = new MToConfig(toJobConfig);
MConnector connector1 = new MConnector("NAME", "CLASSNAME", "1.0",
- connectionForms1, fromJobForm1, toJobForm1);
+ linkConfig, fromConfig1, toConfig1);
assertEquals("NAME", connector1.getUniqueName());
assertEquals("CLASSNAME", connector1.getClassName());
assertEquals("1.0", connector1.getVersion());
MConnector connector2 = new MConnector("NAME", "CLASSNAME", "1.0",
- connectionForms1, fromJobForm1, toJobForm1);
+ linkConfig, fromConfig1, toConfig1);
assertEquals(connector2, connector1);
MConnector connector3 = new MConnector("NAME1", "CLASSNAME", "2.0",
- connectionForms1, fromJobForm1, toJobForm1);
+ linkConfig, fromConfig1, toConfig1);
assertFalse(connector1.equals(connector3));
try {
- connector1 = new MConnector(null, "CLASSNAME", "1.0", connectionForms1,
- fromJobForm1, toJobForm1); // Expecting null pointer exception
+ connector1 = new MConnector(null, "CLASSNAME", "1.0", linkConfig,
+ fromConfig1, toConfig1); // Expecting null pointer exception
} catch (NullPointerException e) {
assertTrue(true);
}
try {
- connector1 = new MConnector("NAME", null, "1.0", connectionForms1,
- fromJobForm1, toJobForm1); // Expecting null pointer exception
+ connector1 = new MConnector("NAME", null, "1.0", linkConfig,
+ fromConfig1, toConfig1); // Expecting null pointer exception
} catch (NullPointerException e) {
assertTrue(true);
}
@@ -97,48 +99,48 @@ public class TestMConnector {
@Test
public void testClone() {
- MConnector connector1 = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
- assertEquals("NAME", connector1.getUniqueName());
- assertEquals("CLASSNAME", connector1.getClassName());
- assertEquals("1.0", connector1.getVersion());
- //Clone with values. Checking values copying after the cloning. But form values will be null
- MConnector clone1 = connector1.clone(true);
- assertEquals("NAME", clone1.getUniqueName());
- assertEquals("CLASSNAME", clone1.getClassName());
- assertEquals("1.0", clone1.getVersion());
- MForm clonedForm1 = clone1.getConnectionForms().getForms().get(0);
- assertNull(clonedForm1.getInputs().get(0).getValue());
- assertNull(clonedForm1.getInputs().get(1).getValue());
-
- MForm clonedForm2 = clone1.getJobForms(Direction.FROM).getForms().get(0);
- assertNull(clonedForm2.getInputs().get(0).getValue());
- assertNull(clonedForm2.getInputs().get(1).getValue());
-
- MForm clonedForm3 = clone1.getJobForms(Direction.TO).getForms().get(0);
- assertNull(clonedForm3.getInputs().get(0).getValue());
- assertNull(clonedForm3.getInputs().get(1).getValue());
+ MConnector connector = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
+ assertEquals("NAME", connector.getUniqueName());
+ assertEquals("CLASSNAME", connector.getClassName());
+ assertEquals("1.0", connector.getVersion());
+ //Clone with values. Checking values copying after the cloning. But config values will be null
+ MConnector cloneConnector1 = connector.clone(true);
+ assertEquals("NAME", cloneConnector1.getUniqueName());
+ assertEquals("CLASSNAME", cloneConnector1.getClassName());
+ assertEquals("1.0", cloneConnector1.getVersion());
+ MConfig clonedLinkConfig = cloneConnector1.getLinkConfig().getConfigs().get(0);
+ assertNull(clonedLinkConfig.getInputs().get(0).getValue());
+ assertNull(clonedLinkConfig.getInputs().get(1).getValue());
+
+ MConfig clonedFromConfig = cloneConnector1.getConfig(Direction.FROM).getConfigs().get(0);
+ assertNull(clonedFromConfig.getInputs().get(0).getValue());
+ assertNull(clonedFromConfig.getInputs().get(1).getValue());
+
+ MConfig clonedToConfig = cloneConnector1.getConfig(Direction.TO).getConfigs().get(0);
+ assertNull(clonedToConfig.getInputs().get(0).getValue());
+ assertNull(clonedToConfig.getInputs().get(1).getValue());
//Clone without values. Inputs value will be null after cloning.
- MConnector clone2 = connector1.clone(false);
- clonedForm1 = clone2.getConnectionForms().getForms().get(0);
- assertNull(clonedForm1.getInputs().get(0).getValue());
- assertNull(clonedForm1.getInputs().get(1).getValue());
- clonedForm2 = clone2.getJobForms(Direction.FROM).getForms().get(0);
- assertNull(clonedForm2.getInputs().get(0).getValue());
- assertNull(clonedForm2.getInputs().get(1).getValue());
- clonedForm3 = clone2.getJobForms(Direction.TO).getForms().get(0);
- assertNull(clonedForm3.getInputs().get(0).getValue());
- assertNull(clonedForm3.getInputs().get(1).getValue());
+ MConnector clonedConnector2 = connector.clone(false);
+ clonedLinkConfig = clonedConnector2.getLinkConfig().getConfigs().get(0);
+ assertNull(clonedLinkConfig.getInputs().get(0).getValue());
+ assertNull(clonedLinkConfig.getInputs().get(1).getValue());
+ clonedFromConfig = clonedConnector2.getConfig(Direction.FROM).getConfigs().get(0);
+ assertNull(clonedFromConfig.getInputs().get(0).getValue());
+ assertNull(clonedFromConfig.getInputs().get(1).getValue());
+ clonedToConfig = clonedConnector2.getConfig(Direction.TO).getConfigs().get(0);
+ assertNull(clonedToConfig.getInputs().get(0).getValue());
+ assertNull(clonedToConfig.getInputs().get(1).getValue());
}
@Test
public void testFromDirection() {
MConnector connector = createConnector(Arrays.asList(Direction.FROM));
- // Clone should clone only one job form.
+ // Clone should clone only one job config.
MConnector clone = connector.clone(true);
- assertNotNull(clone.getJobForms(Direction.FROM));
- assertNull(clone.getJobForms(Direction.TO));
+ assertNotNull(clone.getFromConfig());
+ assertNull(clone.getToConfig());
assertEquals(connector, clone);
assertEquals(connector.toString(), clone.toString());
assertNotEquals(connector.hashCode(), clone.hashCode());
@@ -148,10 +150,10 @@ public class TestMConnector {
public void testToDirection() {
MConnector connector = createConnector(Arrays.asList(Direction.TO));
- // Clone should clone only one job form.
+ // Clone should clone only one job config.
MConnector clone = connector.clone(true);
- assertNull(clone.getJobForms(Direction.FROM));
- assertNotNull(clone.getJobForms(Direction.TO));
+ assertNull(clone.getFromConfig());
+ assertNotNull(clone.getToConfig());
assertEquals(connector, clone);
assertEquals(connector.toString(), clone.toString());
assertNotEquals(connector.hashCode(), clone.hashCode());
@@ -161,10 +163,10 @@ public class TestMConnector {
public void testNoDirection() {
MConnector connector = createConnector(Arrays.asList(new Direction[0]));
- // Clone should clone only one job form.
+ // Clone should clone only one job config.
MConnector clone = connector.clone(true);
- assertNull(clone.getJobForms(Direction.FROM));
- assertNull(clone.getJobForms(Direction.TO));
+ assertNull(clone.getFromConfig());
+ assertNull(clone.getToConfig());
assertEquals(connector, clone);
assertEquals(connector.toString(), clone.toString());
assertNotEquals(connector.hashCode(), clone.hashCode());
@@ -174,10 +176,10 @@ public class TestMConnector {
public void testBothDirections() {
MConnector connector = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
- // Clone should clone only one job form.
+ // Clone should clone only one job config.
MConnector clone = connector.clone(true);
- assertNotNull(clone.getJobForms(Direction.FROM));
- assertNotNull(clone.getJobForms(Direction.TO));
+ assertNotNull(clone.getFromConfig());
+ assertNotNull(clone.getToConfig());
assertEquals(connector, clone);
assertEquals(connector.toString(), clone.toString());
assertNotEquals(connector.hashCode(), clone.hashCode());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMDriver.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMDriver.java b/common/src/test/java/org/apache/sqoop/model/TestMDriver.java
new file mode 100644
index 0000000..aa1ee34
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/model/TestMDriver.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.sqoop.json.DriverBean;
+import org.junit.Test;
+
+public class TestMDriver {
+
+ @Test
+ public void testDriver() {
+ List<MConfig> driverConfig = new ArrayList<MConfig>();
+ driverConfig.add(new MConfig("driver-test", new ArrayList<MInput<?>>()));
+ MDriverConfig mDriverConfig = new MDriverConfig(driverConfig);
+
+ MDriver driver = new MDriver(mDriverConfig, DriverBean.CURRENT_DRIVER_VERSION);
+ assertEquals(1, driver.getDriverConfig().getConfigs().size());
+ assertEquals("driver-test", driver.getDriverConfig().getConfigs().get(0).getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java b/common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java
deleted file mode 100644
index 9c23cc3..0000000
--- a/common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-/**
- *
- */
-public class TestMDriverConfig {
-
- @Test
- public void testForms() {
- List<MForm> connectionFormList = new ArrayList<MForm>();
- List<MForm> jobFormList = new ArrayList<MForm>();
- connectionFormList.add(new MForm("connection-test", new ArrayList<MInput<?>>()));
- jobFormList.add(new MForm("job-test", new ArrayList<MInput<?>>()));
- MConnectionForms connectionForms = new MConnectionForms(connectionFormList);
- MJobForms jobForms = new MJobForms(jobFormList);
-
- MDriverConfig driver = new MDriverConfig(connectionForms, jobForms, "1");
- assertEquals(1, driver.getJobForms().getForms().size());
- assertEquals("job-test", driver.getJobForms().getForms().get(0).getName());
- assertEquals(1, driver.getConnectionForms().getForms().size());
- assertEquals("connection-test", driver.getConnectionForms().getForms().get(0).getName());
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMForm.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMForm.java b/common/src/test/java/org/apache/sqoop/model/TestMForm.java
deleted file mode 100644
index 536b650..0000000
--- a/common/src/test/java/org/apache/sqoop/model/TestMForm.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- * Test class for org.apache.sqoop.model.MForm
- */
-public class TestMForm {
-
- /**
- * Test for initialization
- */
- @Test
- public void testInitialization() {
- MInput<String> input1 = new MStringInput("sqoopsqoop1", true, (short) 5);
- MInput<String> input2 = new MStringInput("sqoopsqoop2", true, (short) 5);
-
- List<MInput<?>> list = new ArrayList<MInput<?>>();
- list.add(input1);
- list.add(input2);
- MForm mform = new MForm("form", list);
-
- assertEquals("form", mform.getName());
- assertEquals(2, mform.getInputs().size());
- }
-
- /**
- * Test for equals method
- */
- @Test
- public void testEquals() {
- MInput<Integer> input1 = new MIntegerInput("sqoopsqoop1", false);
- MInput<Integer> input2 = new MIntegerInput("sqoopsqoop2", false);
- List<MInput<?>> list1 = new ArrayList<MInput<?>>();
- list1.add(input1);
- list1.add(input2);
- MForm mform1 = new MForm("form", list1);
-
- MInput<Integer> input3 = new MIntegerInput("sqoopsqoop1", false);
- MInput<Integer> input4 = new MIntegerInput("sqoopsqoop2", false);
- List<MInput<?>> list2 = new ArrayList<MInput<?>>();
- list2.add(input3);
- list2.add(input4);
- MForm mform2 = new MForm("form", list2);
- assertEquals(mform2, mform1);
- }
-
- @Test
- public void testGetInputs() {
- MIntegerInput intInput = new MIntegerInput("Form.A", false);
- MMapInput mapInput = new MMapInput("Form.B", false);
- MStringInput stringInput = new MStringInput("Form.C", false, (short)3);
- MEnumInput enumInput = new MEnumInput("Form.D", false, new String[] {"I", "V"});
-
- List<MInput<?>> inputs = new ArrayList<MInput<?>>();
- inputs.add(intInput);
- inputs.add(mapInput);
- inputs.add(stringInput);
- inputs.add(enumInput);
-
- MForm form = new MForm("Form", inputs);
- assertEquals(intInput, form.getIntegerInput("Form.A"));
- assertEquals(mapInput, form.getMapInput("Form.B"));
- assertEquals(stringInput, form.getStringInput("Form.C"));
- assertEquals(enumInput, form.getEnumInput("Form.D"));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMFormList.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMFormList.java b/common/src/test/java/org/apache/sqoop/model/TestMFormList.java
deleted file mode 100644
index b8d3d37..0000000
--- a/common/src/test/java/org/apache/sqoop/model/TestMFormList.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- *
- */
-public class TestMFormList {
- @Test
- public void testGetInputs() {
- List<MForm> forms = new LinkedList<MForm>();
-
- MIntegerInput intInput = new MIntegerInput("Form1.A", false);
- MMapInput mapInput = new MMapInput("Form1.B", false);
-
- List<MInput<?>> inputs = new ArrayList<MInput<?>>();
- inputs.add(intInput);
- inputs.add(mapInput);
- forms.add(new MForm("Form1", inputs));
-
- MStringInput stringInput = new MStringInput("Form2.C", false, (short)3);
- MEnumInput enumInput = new MEnumInput("Form2.D", false, new String[] {"I", "V"});
-
- inputs = new ArrayList<MInput<?>>();
- inputs.add(stringInput);
- inputs.add(enumInput);
- forms.add(new MForm("Form2", inputs));
-
- MFormList form = new MFormList(forms);
- assertEquals(intInput, form.getIntegerInput("Form1.A"));
- assertEquals(mapInput, form.getMapInput("Form1.B"));
- assertEquals(stringInput, form.getStringInput("Form2.C"));
- assertEquals(enumInput, form.getEnumInput("Form2.D"));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMJob.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMJob.java b/common/src/test/java/org/apache/sqoop/model/TestMJob.java
index 8f2943e..848c2cc 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMJob.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMJob.java
@@ -17,17 +17,15 @@
*/
package org.apache.sqoop.model;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
import java.util.ArrayList;
import java.util.List;
import org.apache.sqoop.common.Direction;
import org.junit.Test;
-import static org.junit.Assert.*;
-
-/**
- * Test class for org.apache.sqoop.model.MJob
- */
public class TestMJob {
/**
* Test class for initialization
@@ -40,9 +38,9 @@ public class TestMJob {
assertEquals(456l, job.getConnectorId(Direction.TO));
assertEquals("Buffy", job.getCreationUser());
assertEquals("Vampire", job.getName());
- assertEquals(fromForms(), job.getConnectorPart(Direction.FROM));
- assertEquals(toForms(), job.getConnectorPart(Direction.TO));
- assertEquals(frameworkForms(), job.getFrameworkPart());
+ assertEquals(fromConfig(), job.getJobConfig(Direction.FROM));
+ assertEquals(toConfig(), job.getJobConfig(Direction.TO));
+ assertEquals(driverConfig(), job.getDriverConfig());
// Test copy constructor
MJob copy = new MJob(job);
@@ -50,19 +48,19 @@ public class TestMJob {
assertEquals(456l, copy.getConnectorId(Direction.TO));
assertEquals("Buffy", copy.getCreationUser());
assertEquals("Vampire", copy.getName());
- assertEquals(fromForms(), copy.getConnectorPart(Direction.FROM));
- assertEquals(toForms(), copy.getConnectorPart(Direction.TO));
- assertEquals(frameworkForms(), copy.getFrameworkPart());
+ assertEquals(fromConfig(), copy.getJobConfig(Direction.FROM));
+ assertEquals(toConfig(), copy.getJobConfig(Direction.TO));
+ assertEquals(driverConfig(), copy.getDriverConfig());
- // Test constructor for metadata upgrade (the order of forms is different)
- MJob upgradeCopy = new MJob(job, fromForms(), toForms(), frameworkForms());
+ // Test constructor for metadata upgrade (the order of configs is different)
+ MJob upgradeCopy = new MJob(job, fromConfig(), toConfig(), driverConfig());
assertEquals(123l, upgradeCopy.getConnectorId(Direction.FROM));
assertEquals(456l, upgradeCopy.getConnectorId(Direction.TO));
assertEquals("Buffy", upgradeCopy.getCreationUser());
assertEquals("Vampire", upgradeCopy.getName());
- assertEquals(fromForms(), upgradeCopy.getConnectorPart(Direction.FROM));
- assertEquals(toForms(), upgradeCopy.getConnectorPart(Direction.TO));
- assertEquals(frameworkForms(), upgradeCopy.getFrameworkPart());
+ assertEquals(fromConfig(), upgradeCopy.getJobConfig(Direction.FROM));
+ assertEquals(toConfig(), upgradeCopy.getJobConfig(Direction.TO));
+ assertEquals(driverConfig(), upgradeCopy.getDriverConfig());
}
@Test
@@ -70,42 +68,42 @@ public class TestMJob {
MJob job = job();
// Clone without value
- MJob withoutValue = job.clone(false);
- assertEquals(job, withoutValue);
- assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutValue.getPersistenceId());
- assertNull(withoutValue.getName());
- assertNull(withoutValue.getCreationUser());
- assertEquals(fromForms(), withoutValue.getConnectorPart(Direction.FROM));
- assertEquals(toForms(), withoutValue.getConnectorPart(Direction.TO));
- assertEquals(frameworkForms(), withoutValue.getFrameworkPart());
- assertNull(withoutValue.getConnectorPart(Direction.FROM)
- .getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
- assertNull(withoutValue.getConnectorPart(Direction.FROM)
- .getForm("FORMNAME").getInput("STRING-INPUT").getValue());
+ MJob withoutJobValue = job.clone(false);
+ assertEquals(job, withoutJobValue);
+ assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutJobValue.getPersistenceId());
+ assertNull(withoutJobValue.getName());
+ assertNull(withoutJobValue.getCreationUser());
+ assertEquals(fromConfig(), withoutJobValue.getJobConfig(Direction.FROM));
+ assertEquals(toConfig(), withoutJobValue.getJobConfig(Direction.TO));
+ assertEquals(driverConfig(), withoutJobValue.getDriverConfig());
+ assertNull(withoutJobValue.getJobConfig(Direction.FROM)
+ .getConfig("CONFIGFROMNAME").getInput("INTEGER-INPUT").getValue());
+ assertNull(withoutJobValue.getJobConfig(Direction.FROM)
+ .getConfig("CONFIGFROMNAME").getInput("STRING-INPUT").getValue());
// Clone with value
- MJob withValue = job.clone(true);
- assertEquals(job, withValue);
- assertEquals(job.getPersistenceId(), withValue.getPersistenceId());
- assertEquals(job.getName(), withValue.getName());
- assertEquals(job.getCreationUser(), withValue.getCreationUser());
- assertEquals(fromForms(), withValue.getConnectorPart(Direction.FROM));
- assertEquals(toForms(), withValue.getConnectorPart(Direction.TO));
- assertEquals(frameworkForms(), withValue.getFrameworkPart());
- assertEquals(100, withValue.getConnectorPart(Direction.FROM)
- .getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
- assertEquals("TEST-VALUE", withValue.getConnectorPart(Direction.FROM)
- .getForm("FORMNAME").getInput("STRING-INPUT").getValue()); }
+ MJob withJobValue = job.clone(true);
+ assertEquals(job, withJobValue);
+ assertEquals(job.getPersistenceId(), withJobValue.getPersistenceId());
+ assertEquals(job.getName(), withJobValue.getName());
+ assertEquals(job.getCreationUser(), withJobValue.getCreationUser());
+ assertEquals(fromConfig(), withJobValue.getJobConfig(Direction.FROM));
+ assertEquals(toConfig(), withJobValue.getJobConfig(Direction.TO));
+ assertEquals(driverConfig(), withJobValue.getDriverConfig());
+ assertEquals(100, withJobValue.getJobConfig(Direction.FROM)
+ .getConfig("CONFIGFROMNAME").getInput("INTEGER-INPUT").getValue());
+ assertEquals("TEST-VALUE", withJobValue.getJobConfig(Direction.FROM)
+ .getConfig("CONFIGFROMNAME").getInput("STRING-INPUT").getValue()); }
private MJob job() {
- MJob job = new MJob(123l, 456l, 1L, 2L, fromForms(), toForms(), frameworkForms());
+ MJob job = new MJob(123l, 456l, 1L, 2L, fromConfig(), toConfig(), driverConfig());
job.setName("Vampire");
job.setCreationUser("Buffy");
return job;
}
- private MJobForms fromForms() {
- List<MForm> forms = new ArrayList<MForm>();
+ private MFromConfig fromConfig() {
+ List<MConfig> configs = new ArrayList<MConfig>();
MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
input.setValue(100);
MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
@@ -113,28 +111,28 @@ public class TestMJob {
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
list.add(strInput);
- MForm form = new MForm("FORMNAME", list);
- forms.add(form);
- return new MJobForms(forms);
+ MConfig config = new MConfig("CONFIGFROMNAME", list);
+ configs.add(config);
+ return new MFromConfig(configs);
}
- private MJobForms toForms() {
- List<MForm> forms = new ArrayList<MForm>();
+ private MToConfig toConfig() {
+ List<MConfig> configs = new ArrayList<MConfig>();
MMapInput input = new MMapInput("MAP-INPUT", false);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
- MForm form = new MForm("form", list);
- forms.add(form);
- return new MJobForms(forms);
+ MConfig config = new MConfig("CONFIGTONAME", list);
+ configs.add(config);
+ return new MToConfig(configs);
}
- private MJobForms frameworkForms() {
- List<MForm> forms = new ArrayList<MForm>();
+ private MDriverConfig driverConfig() {
+ List<MConfig> configs = new ArrayList<MConfig>();
MMapInput input = new MMapInput("MAP-INPUT", false);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
- MForm form = new MForm("form", list);
- forms.add(form);
- return new MJobForms(forms);
+ MConfig config = new MConfig("CONFIGDRIVERNAME", list);
+ configs.add(config);
+ return new MDriverConfig(configs);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMJobConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMJobConfig.java b/common/src/test/java/org/apache/sqoop/model/TestMJobConfig.java
new file mode 100644
index 0000000..7d0641e
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/model/TestMJobConfig.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestMJobConfig {
+ /**
+ * Test for class initialization and values
+ */
+ @Test
+ public void testInitialization() {
+ List<MConfig> configs = new ArrayList<MConfig>();
+ MFromConfig fromJobConfig = new MFromConfig(configs);
+ List<MConfig> configs2 = new ArrayList<MConfig>();
+ MFromConfig fromJobConfig2 = new MFromConfig(configs2);
+ assertEquals(fromJobConfig2, fromJobConfig);
+ MConfig c = new MConfig("test", null);
+ configs2.add(c);
+ assertFalse(fromJobConfig.equals(fromJobConfig2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java b/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java
deleted file mode 100644
index e59b282..0000000
--- a/common/src/test/java/org/apache/sqoop/model/TestMJobForms.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- * Test class for org.apache.sqoop.model.MJobForms
- */
-public class TestMJobForms {
- /**
- * Test for class initialization and values
- */
- @Test
- public void testInitialization() {
- List<MForm> forms = new ArrayList<MForm>();
- MJobForms jobform1 = new MJobForms(forms);
- List<MForm> forms2 = new ArrayList<MForm>();
- MJobForms jobform2 = new MJobForms(forms2);
- assertEquals(jobform2, jobform1);
- // Add a form to list for checking not equals
- MForm m = new MForm("test", null);
- forms2.add(m);
- assertFalse(jobform1.equals(jobform2));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMLink.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMLink.java b/common/src/test/java/org/apache/sqoop/model/TestMLink.java
index 77fa2a9..9ad8954 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMLink.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMLink.java
@@ -24,9 +24,6 @@ import org.junit.Test;
import static org.junit.Assert.*;
-/**
- * Test class for org.apache.sqoop.model.MConnection
- */
public class TestMLink {
/**
@@ -39,8 +36,7 @@ public class TestMLink {
assertEquals(123l, link.getConnectorId());
assertEquals("Vampire", link.getName());
assertEquals("Buffy", link.getCreationUser());
- assertEquals(forms1(), link.getConnectorPart());
- assertEquals(forms2(), link.getFrameworkPart());
+ assertEquals(linkConfig(), link.getConnectorLinkConfig());
// Test copy constructor
MLink copy = new MLink(link);
@@ -48,17 +44,7 @@ public class TestMLink {
assertEquals("Vampire", copy.getName());
assertEquals("Buffy", copy.getCreationUser());
assertEquals(link.getCreationDate(), copy.getCreationDate());
- assertEquals(forms1(), copy.getConnectorPart());
- assertEquals(forms2(), copy.getFrameworkPart());
-
- // Test constructor for metadata upgrade (the order of forms is different)
- MLink upgradeCopy = new MLink(link, forms2(), forms1());
- assertEquals(123l, upgradeCopy.getConnectorId());
- assertEquals("Vampire", upgradeCopy.getName());
- assertEquals("Buffy", upgradeCopy.getCreationUser());
- assertEquals(link.getCreationDate(), upgradeCopy.getCreationDate());
- assertEquals(forms2(), upgradeCopy.getConnectorPart());
- assertEquals(forms1(), upgradeCopy.getFrameworkPart());
+ assertEquals(linkConfig(), copy.getConnectorLinkConfig());
}
@Test
@@ -66,37 +52,35 @@ public class TestMLink {
MLink link = link();
// Clone without value
- MLink withoutValue = link.clone(false);
- assertEquals(link, withoutValue);
- assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutValue.getPersistenceId());
- assertNull(withoutValue.getName());
- assertNull(withoutValue.getCreationUser());
- assertEquals(forms1(), withoutValue.getConnectorPart());
- assertEquals(forms2(), withoutValue.getFrameworkPart());
- assertNull(withoutValue.getConnectorPart().getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
- assertNull(withoutValue.getConnectorPart().getForm("FORMNAME").getInput("STRING-INPUT").getValue());
+ MLink withoutLinkValue = link.clone(false);
+ assertEquals(link, withoutLinkValue);
+ assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutLinkValue.getPersistenceId());
+ assertNull(withoutLinkValue.getName());
+ assertNull(withoutLinkValue.getCreationUser());
+ assertEquals(linkConfig(), withoutLinkValue.getConnectorLinkConfig());
+ assertNull(withoutLinkValue.getConnectorLinkConfig().getConfig("CONFIGNAME").getInput("INTEGER-INPUT").getValue());
+ assertNull(withoutLinkValue.getConnectorLinkConfig().getConfig("CONFIGNAME").getInput("STRING-INPUT").getValue());
// Clone with value
- MLink withValue = link.clone(true);
- assertEquals(link, withValue);
- assertEquals(link.getPersistenceId(), withValue.getPersistenceId());
- assertEquals(link.getName(), withValue.getName());
- assertEquals(link.getCreationUser(), withValue.getCreationUser());
- assertEquals(forms1(), withValue.getConnectorPart());
- assertEquals(forms2(), withValue.getFrameworkPart());
- assertEquals(100, withValue.getConnectorPart().getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
- assertEquals("TEST-VALUE", withValue.getConnectorPart().getForm("FORMNAME").getInput("STRING-INPUT").getValue());
+ MLink withLinkValue = link.clone(true);
+ assertEquals(link, withLinkValue);
+ assertEquals(link.getPersistenceId(), withLinkValue.getPersistenceId());
+ assertEquals(link.getName(), withLinkValue.getName());
+ assertEquals(link.getCreationUser(), withLinkValue.getCreationUser());
+ assertEquals(linkConfig(), withLinkValue.getConnectorLinkConfig());
+ assertEquals(100, withLinkValue.getConnectorLinkConfig().getConfig("CONFIGNAME").getInput("INTEGER-INPUT").getValue());
+ assertEquals("TEST-VALUE", withLinkValue.getConnectorLinkConfig().getConfig("CONFIGNAME").getInput("STRING-INPUT").getValue());
}
private MLink link() {
- MLink link = new MLink(123l, forms1(), forms2());
+ MLink link = new MLink(123l, linkConfig());
link.setName("Vampire");
link.setCreationUser("Buffy");
return link;
}
- private MConnectionForms forms1() {
- List<MForm> forms = new ArrayList<MForm>();
+ private MLinkConfig linkConfig() {
+ List<MConfig> configs = new ArrayList<MConfig>();
MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
input.setValue(100);
MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
@@ -104,19 +88,9 @@ public class TestMLink {
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
list.add(strInput);
- MForm form = new MForm("FORMNAME", list);
- forms.add(form);
- return new MConnectionForms(forms);
- }
-
- private MConnectionForms forms2() {
- List<MForm> forms = new ArrayList<MForm>();
- MMapInput input = new MMapInput("MAP-INPUT", false);
- List<MInput<?>> list = new ArrayList<MInput<?>>();
- list.add(input);
- MForm form = new MForm("form", list);
- forms.add(form);
- return new MConnectionForms(forms);
+ MConfig config = new MConfig("CONFIGNAME", list);
+ configs.add(config);
+ return new MLinkConfig(configs);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMLinkConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMLinkConfig.java b/common/src/test/java/org/apache/sqoop/model/TestMLinkConfig.java
new file mode 100644
index 0000000..62f61a6
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/model/TestMLinkConfig.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestMLinkConfig {
+
+ /**
+ * Test for class initialization and values
+ */
+ @Test
+ public void testInitialization() {
+ List<MConfig> configs = new ArrayList<MConfig>();
+ MLinkConfig linkConfig = new MLinkConfig(configs);
+ List<MConfig> testConfig = new ArrayList<MConfig>();
+ assertEquals(testConfig, linkConfig.getConfigs());
+ MLinkConfig linkConfig2 = new MLinkConfig(testConfig);
+ assertEquals(linkConfig2, linkConfig);
+ // Add a config to list for checking not equals
+ MConfig c = new MConfig("test", null);
+ testConfig.add(c);
+ assertFalse(linkConfig.equals(linkConfig2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/validation/TestValidation.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/TestValidation.java b/common/src/test/java/org/apache/sqoop/validation/TestValidation.java
index 85e9e1c..16a8bbe 100644
--- a/common/src/test/java/org/apache/sqoop/validation/TestValidation.java
+++ b/common/src/test/java/org/apache/sqoop/validation/TestValidation.java
@@ -17,16 +17,20 @@
*/
package org.apache.sqoop.validation;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.util.HashMap;
import java.util.Map;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.validation.Validation.FormInput;
-import org.apache.sqoop.validation.Validation.Message;
+import org.apache.sqoop.validation.ConfigValidator.ConfigInput;
+import org.apache.sqoop.validation.ConfigValidator.Message;
import org.junit.Test;
-import static org.junit.Assert.*;
-
/**
* Test class for org.apache.sqoop.validation.Validation
*/
@@ -38,42 +42,42 @@ public class TestValidation {
@Test
public void testInitialization() {
/* Check initialization with class */
- Validation validation = new Validation(Class.class);
+ ConfigValidator validation = new ConfigValidator(Class.class);
assertNotNull(validation);
assertEquals(Status.FINE, validation.getStatus());
assertEquals(0, validation.getMessages().size());
/* Check initialization with status and message as null */
- Validation validationNull = new Validation(null, null);
+ ConfigValidator validationNull = new ConfigValidator(null, null);
assertNotNull(validationNull);
assertNull(validationNull.getStatus());
assertNull(validationNull.getMessages());
/* Check initialization with status and message with values */
Status s1 = Status.FINE;
- Map<FormInput, Message> msg1 = new HashMap<Validation.FormInput, Validation.Message>();
- Validation validation1 = new Validation(s1, msg1);
+ Map<ConfigInput, Message> msg1 = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
+ ConfigValidator validation1 = new ConfigValidator(s1, msg1);
assertNotNull(validation1);
assertEquals(Status.FINE, validation1.getStatus());
assertEquals(0, validation1.getMessages().size());
/* Check initialization with status and message with values */
Status s2 = Status.ACCEPTABLE;
- Map<FormInput, Message> msg2 = new HashMap<Validation.FormInput, Validation.Message>();
- Validation validation2 = new Validation(s2, msg2);
+ Map<ConfigInput, Message> msg2 = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
+ ConfigValidator validation2 = new ConfigValidator(s2, msg2);
assertNotNull(validation2);
assertEquals(Status.ACCEPTABLE, validation2.getStatus());
assertEquals(0, validation2.getMessages().size());
/* Check initialization with status and message with values */
Status s3 = Status.ACCEPTABLE;
- Map<FormInput, Message> msg3 = new HashMap<Validation.FormInput, Validation.Message>();
- Validation.FormInput fi = new Validation.FormInput("form\\.input");
- Validation.Message message = new Validation.Message(Status.FINE, "sqoop");
+ Map<ConfigInput, Message> msg3 = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
+ ConfigValidator.ConfigInput fi = new ConfigValidator.ConfigInput("config\\.input");
+ ConfigValidator.Message message = new ConfigValidator.Message(Status.FINE, "sqoop");
msg3.put(fi, message);
- Validation validation3 = new Validation(s3, msg3);
- Validation.FormInput fiTest = new Validation.FormInput("form\\.input");
- Validation.Message messageTest = new Validation.Message(Status.FINE,
+ ConfigValidator validation3 = new ConfigValidator(s3, msg3);
+ ConfigValidator.ConfigInput fiTest = new ConfigValidator.ConfigInput("config\\.input");
+ ConfigValidator.Message messageTest = new ConfigValidator.Message(Status.FINE,
"sqoop");
assertEquals(messageTest, validation3.getMessages().get(fiTest));
assertEquals(Status.ACCEPTABLE, validation3.getStatus());
@@ -82,13 +86,13 @@ public class TestValidation {
/**
* Test for Validation.ForInput
*/
- public void testFormInput() {
- Validation.FormInput fi = new Validation.FormInput("test\\.test");
+ public void testConfigInput() {
+ ConfigValidator.ConfigInput fi = new ConfigValidator.ConfigInput("test\\.test");
assertNotNull(fi);
/* Passing null */
try {
- new Validation.FormInput(null);
+ new ConfigValidator.ConfigInput(null);
fail("Assert error is expected");
} catch (AssertionError e) {
assertTrue(true);
@@ -96,31 +100,31 @@ public class TestValidation {
/* Passing empty and check exception messages */
try {
- new Validation.FormInput("");
+ new ConfigValidator.ConfigInput("");
fail("SqoopException is expected");
} catch (SqoopException e) {
- assertEquals(ValidationError.VALIDATION_0003.getMessage(), e
+ assertEquals(ConfigValidationError.VALIDATION_0003.getMessage(), e
.getErrorCode().getMessage());
}
/* Passing value and check */
- Validation.FormInput fi2 = new Validation.FormInput("form\\.input");
- assertEquals("form\\", fi2.getForm());
+ ConfigValidator.ConfigInput fi2 = new ConfigValidator.ConfigInput("config\\.input");
+ assertEquals("config\\", fi2.getConfig());
assertEquals("input", fi2.getInput());
/* Check equals */
- Validation.FormInput fiOne = new Validation.FormInput("form\\.input");
- Validation.FormInput fiTwo = new Validation.FormInput("form\\.input");
+ ConfigValidator.ConfigInput fiOne = new ConfigValidator.ConfigInput("config\\.input");
+ ConfigValidator.ConfigInput fiTwo = new ConfigValidator.ConfigInput("config\\.input");
assertEquals(fiOne, fiTwo);
/* toString() method check */
- assertEquals("form\\.input", fiOne.toString());
+ assertEquals("config\\.input", fiOne.toString());
- // Checking null as input field (form validation)
- Validation.FormInput fi3 = new FormInput("form");
- assertEquals("form", fi3.getForm());
+ // Checking null as input field (config validation)
+ ConfigValidator.ConfigInput fi3 = new ConfigInput("config");
+ assertEquals("config", fi3.getConfig());
assertNull(fi3.getInput());
- assertEquals("form", fi3.toString());
+ assertEquals("config", fi3.toString());
}
@@ -129,17 +133,17 @@ public class TestValidation {
*/
public void testMessage() {
/* Passing null */
- Validation.Message msg1 = new Validation.Message(null, null);
+ ConfigValidator.Message msg1 = new ConfigValidator.Message(null, null);
assertNull(msg1.getStatus());
assertNull(msg1.getMessage());
/* Passing values */
- Validation.Message msg2 = new Validation.Message(Status.FINE, "sqoop");
+ ConfigValidator.Message msg2 = new ConfigValidator.Message(Status.FINE, "sqoop");
assertEquals(Status.FINE, msg2.getStatus());
assertEquals("sqoop", msg2.getMessage());
/* Check for equal */
- Validation.Message msg3 = new Validation.Message(Status.FINE, "sqoop");
+ ConfigValidator.Message msg3 = new ConfigValidator.Message(Status.FINE, "sqoop");
assertEquals(msg2, msg3);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java b/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
index 647abe0..579d1c5 100644
--- a/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
+++ b/common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
@@ -18,8 +18,8 @@
package org.apache.sqoop.validation;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Config;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.validators.Contains;
@@ -35,18 +35,18 @@ import static org.junit.Assert.assertTrue;
*/
public class TestValidationRunner {
- @FormClass(validators = {@Validator(FormA.FormValidator.class)})
- public static class FormA {
+ @ConfigClass(validators = {@Validator(ConfigA.ConfigValidator.class)})
+ public static class ConfigA {
@Input(validators = {@Validator(NotNull.class)})
String notNull;
- public static class FormValidator extends AbstractValidator<FormA> {
+ public static class ConfigValidator extends AbstractValidator<ConfigA> {
@Override
- public void validate(FormA form) {
- if(form.notNull == null) {
+ public void validate(ConfigA config) {
+ if(config.notNull == null) {
addMessage(Status.UNACCEPTABLE, "null");
}
- if("error".equals(form.notNull)) {
+ if("error".equals(config.notNull)) {
addMessage(Status.UNACCEPTABLE, "error");
}
}
@@ -54,83 +54,83 @@ public class TestValidationRunner {
}
@Test
- public void testValidateForm() {
- FormA form = new FormA();
- ValidationRunner runner = new ValidationRunner();
- ValidationResult result;
-
- // Null string should fail on Input level and should not call form level validators
- form.notNull = null;
- result = runner.validateForm("formName", form);
+ public void testValidateConfig() {
+ ConfigA config = new ConfigA();
+ ConfigValidationRunner runner = new ConfigValidationRunner();
+ ConfigValidationResult result;
+
+ // Null string should fail on Input level and should not call config level validators
+ config.notNull = null;
+ result = runner.validateConfig("configName", config);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
- assertTrue(result.getMessages().containsKey("formName.notNull"));
+ assertTrue(result.getMessages().containsKey("configName.notNull"));
- // String "error" should trigger form level error, but not Input level
- form.notNull = "error";
- result = runner.validateForm("formName", form);
+ // String "error" should trigger config level error, but not Input level
+ config.notNull = "error";
+ result = runner.validateConfig("configName", config);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
- assertTrue(result.getMessages().containsKey("formName"));
+ assertTrue(result.getMessages().containsKey("configName"));
// Acceptable state
- form.notNull = "This is truly random string";
- result = runner.validateForm("formName", form);
+ config.notNull = "This is truly random string";
+ result = runner.validateConfig("configName", config);
assertEquals(Status.FINE, result.getStatus());
assertEquals(0, result.getMessages().size());
}
- @FormClass
- public static class FormB {
+ @ConfigClass
+ public static class ConfigB {
@Input(validators = {@Validator(NotNull.class), @Validator(NotEmpty.class)})
String str;
}
- @FormClass
- public static class FormC {
+ @ConfigClass
+ public static class ConfigC {
@Input(validators = {@Validator(value = Contains.class, strArg = "findme")})
String str;
}
@Test
public void testMultipleValidatorsOnSingleInput() {
- FormB form = new FormB();
- ValidationRunner runner = new ValidationRunner();
- ValidationResult result;
+ ConfigB config = new ConfigB();
+ ConfigValidationRunner runner = new ConfigValidationRunner();
+ ConfigValidationResult result;
- form.str = null;
- result = runner.validateForm("formName", form);
+ config.str = null;
+ result = runner.validateConfig("configName", config);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
- assertTrue(result.getMessages().containsKey("formName.str"));
- assertEquals(2, result.getMessages().get("formName.str").size());
+ assertTrue(result.getMessages().containsKey("configName.str"));
+ assertEquals(2, result.getMessages().get("configName.str").size());
}
@Test
public void testValidatorWithParameters() {
- FormC form = new FormC();
- ValidationRunner runner = new ValidationRunner();
- ValidationResult result;
+ ConfigC config = new ConfigC();
+ ConfigValidationRunner runner = new ConfigValidationRunner();
+ ConfigValidationResult result;
// Sub string not found
- form.str = "Mordor";
- result = runner.validateForm("formName", form);
+ config.str = "Mordor";
+ result = runner.validateConfig("configName", config);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
- assertTrue(result.getMessages().containsKey("formName.str"));
+ assertTrue(result.getMessages().containsKey("configName.str"));
// Sub string found
- form.str = "Morfindmedor";
- result = runner.validateForm("formName", form);
+ config.str = "Morfindmedor";
+ result = runner.validateConfig("configName", config);
assertEquals(Status.FINE, result.getStatus());
assertEquals(0, result.getMessages().size());
}
@ConfigurationClass(validators = {@Validator(ConfigurationA.ClassValidator.class)})
public static class ConfigurationA {
- @Form FormA formA;
+ @Config ConfigA formA;
public ConfigurationA() {
- formA = new FormA();
+ formA = new ConfigA();
}
public static class ClassValidator extends AbstractValidator<ConfigurationA> {
@@ -149,24 +149,24 @@ public class TestValidationRunner {
@Test
public void testValidate() {
ConfigurationA conf = new ConfigurationA();
- ValidationRunner runner = new ValidationRunner();
- ValidationResult result;
+ ConfigValidationRunner runner = new ConfigValidationRunner();
+ ConfigValidationResult result;
- // Null string should fail on Input level and should not call form nor class level validators
+ // Null string should fail on Input level and should not call config nor class level validators
conf.formA.notNull = null;
result = runner.validate(conf);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
assertTrue(result.getMessages().containsKey("formA.notNull"));
- // String "error" should trigger form level error, but not Input nor class level
+ // String "error" should trigger config level error, but not Input nor class level
conf.formA.notNull = "error";
result = runner.validate(conf);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
assertTrue(result.getMessages().containsKey("formA"));
- // String "conf-error" should trigger class level error, but not Input nor Form level
+ // String "conf-error" should trigger class level error, but not Input nor Config level
conf.formA.notNull = "conf-error";
result = runner.validate(conf);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
index 62b2e0a..3a15274 100644
--- a/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
+++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
@@ -17,19 +17,17 @@
*/
package org.apache.sqoop.validation.validators;
-import org.apache.sqoop.validation.Message;
-import org.apache.sqoop.validation.Status;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
import java.util.List;
-import static org.junit.Assert.assertEquals;
+import org.apache.sqoop.validation.Message;
+import org.apache.sqoop.validation.Status;
+import org.junit.Test;
-/**
- */
public class TestClassAvailable {
- AbstractValidator validator = new ClassAvailable();
+ AbstractValidator<String> validator = new ClassAvailable();
@Test
public void test() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
index b4b6966..87ac2af 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
@@ -92,7 +92,7 @@ public class GenericJdbcConnector extends SqoopConnector {
}
@Override
- public Validator getValidator() {
+ public Validator getConfigValidator() {
return genericJdbcValidator;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
index 8deddb0..a069b3e 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
@@ -18,21 +18,20 @@
*/
package org.apache.sqoop.connector.jdbc;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJobForms;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import org.apache.sqoop.model.MLinkConfig;
public class GenericJdbcConnectorUpgrader extends RepositoryUpgrader {
- private static final Logger LOG =
- Logger.getLogger(GenericJdbcConnectorUpgrader.class);
+ private static final Logger LOG = Logger.getLogger(GenericJdbcConnectorUpgrader.class);
/*
* For now, there is no real upgrade. So copy all data over,
@@ -41,41 +40,40 @@ public class GenericJdbcConnectorUpgrader extends RepositoryUpgrader {
*/
@Override
- public void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
+ public void upgrade(MLinkConfig original, MLinkConfig upgradeTarget) {
+ doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@Override
- public void upgrade(MJobForms original, MJobForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
+ public void upgrade(MConfigList original, MConfigList upgradeTarget) {
+ doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@SuppressWarnings("unchecked")
- private void doUpgrade(List<MForm> original, List<MForm> target) {
- // Easier to find the form in the original forms list if we use a map.
- // Since the constructor of MJobForms takes a list,
+ private void doUpgrade(List<MConfig> original, List<MConfig> target) {
+ // Easier to find the config in the original list if we use a map.
+ // Since the constructor takes a list,
// index is not guaranteed to be the same, so we need to look for
// equivalence
- Map<String, MForm> formMap = new HashMap<String, MForm>();
- for (MForm form : original) {
- formMap.put(form.getName(), form);
+ Map<String, MConfig> configMap = new HashMap<String, MConfig>();
+ for (MConfig config : original) {
+ configMap.put(config.getName(), config);
}
- for (MForm form : target) {
- List<MInput<?>> inputs = form.getInputs();
- MForm originalForm = formMap.get(form.getName());
- if (originalForm == null) {
- LOG.warn("Form: '" + form.getName() + "' not present in old " +
- "connector. So it and its inputs will not be transferred by the upgrader.");
+ for (MConfig config : target) {
+ List<MInput<?>> inputs = config.getInputs();
+ MConfig orginalConfig = configMap.get(config.getName());
+ if (orginalConfig == null) {
+ LOG.warn("Config: '" + config.getName() + "' not present in old " +
+ "generic JDBC connector. So it and its inputs will not be transferred by the upgrader.");
continue;
}
for (MInput input : inputs) {
try {
- MInput originalInput = originalForm.getInput(input.getName());
+ MInput originalInput = orginalConfig.getInput(input.getName());
input.setValue(originalInput.getValue());
} catch (SqoopException ex) {
LOG.warn("Input: '" + input.getName() + "' not present in old " +
- "connector. So it will not be transferred by the upgrader.");
+ "generic JDBC connector. So it will not be transferred by the upgrader.");
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
index e52610a..af9320b 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
@@ -34,12 +34,12 @@ public class GenericJdbcExtractor extends Extractor<LinkConfiguration, FromJobCo
private long rowsRead = 0;
@Override
- public void extract(ExtractorContext context, LinkConfiguration linkConf,
- FromJobConfiguration fromJobConf, GenericJdbcPartition partition) {
- String driver = linkConf.link.jdbcDriver;
- String url = linkConf.link.connectionString;
- String username = linkConf.link.username;
- String password = linkConf.link.password;
+ public void extract(ExtractorContext context, LinkConfiguration linkConfig,
+ FromJobConfiguration fromJobConfig, GenericJdbcPartition partition) {
+ String driver = linkConfig.linkConfig.jdbcDriver;
+ String url = linkConfig.linkConfig.connectionString;
+ String username = linkConfig.linkConfig.username;
+ String password = linkConfig.linkConfig.password;
GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password);
String query = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
index d3a893f..3a783be 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
@@ -29,7 +29,7 @@ public class GenericJdbcFromDestroyer extends Destroyer<LinkConfiguration, FromJ
Logger.getLogger(GenericJdbcFromDestroyer.class);
@Override
- public void destroy(DestroyerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
+ public void destroy(DestroyerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
LOG.info("Running generic JDBC connector destroyer");
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
index 9d0c178..ef4ecc4 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
@@ -45,34 +45,34 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F
private GenericJdbcExecutor executor;
@Override
- public void initialize(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
- configureJdbcProperties(context.getContext(), linkConf, fromJobConf);
+ public void initialize(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
+ configureJdbcProperties(context.getContext(), linkConfig, fromJobConfig);
try {
- configurePartitionProperties(context.getContext(), linkConf, fromJobConf);
- configureTableProperties(context.getContext(), linkConf, fromJobConf);
+ configurePartitionProperties(context.getContext(), linkConfig, fromJobConfig);
+ configureTableProperties(context.getContext(), linkConfig, fromJobConfig);
} finally {
executor.close();
}
}
@Override
- public List<String> getJars(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
+ public List<String> getJars(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
List<String> jars = new LinkedList<String>();
- jars.add(ClassUtils.jarForClass(linkConf.link.jdbcDriver));
+ jars.add(ClassUtils.jarForClass(linkConfig.linkConfig.jdbcDriver));
return jars;
}
@Override
- public Schema getSchema(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
- configureJdbcProperties(context.getContext(), linkConf, fromJobConf);
+ public Schema getSchema(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
+ configureJdbcProperties(context.getContext(), linkConfig, fromJobConfig);
- String schemaName = fromJobConf.fromJobConfig.tableName;
+ String schemaName = fromJobConfig.fromJobConfig.tableName;
if(schemaName == null) {
schemaName = "Query";
- } else if(fromJobConf.fromJobConfig.schemaName != null) {
- schemaName = fromJobConf.fromJobConfig.schemaName + "." + schemaName;
+ } else if(fromJobConfig.fromJobConfig.schemaName != null) {
+ schemaName = fromJobConfig.fromJobConfig.schemaName + "." + schemaName;
}
Schema schema = new Schema(schemaName);
@@ -117,11 +117,11 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F
}
}
- private void configureJdbcProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
- String driver = connectionConfig.link.jdbcDriver;
- String url = connectionConfig.link.connectionString;
- String username = connectionConfig.link.username;
- String password = connectionConfig.link.password;
+ private void configureJdbcProperties(MutableContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
+ String driver = linkConfig.linkConfig.jdbcDriver;
+ String url = linkConfig.linkConfig.connectionString;
+ String username = linkConfig.linkConfig.username;
+ String password = linkConfig.linkConfig.password;
assert driver != null;
assert url != null;
@@ -129,7 +129,7 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F
executor = new GenericJdbcExecutor(driver, url, username, password);
}
- private void configurePartitionProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
+ private void configurePartitionProperties(MutableContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
// ----- configure column name -----
String partitionColumnName = fromJobConfig.fromJobConfig.partitionColumn;
@@ -234,7 +234,7 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F
}
}
- private void configureTableProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
+ private void configureTableProperties(MutableContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
String dataSql;
String fieldNames;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
index 991e686..6340a70 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
@@ -30,11 +30,11 @@ public class GenericJdbcLoader extends Loader<LinkConfiguration, ToJobConfigurat
private int batchesPerTransaction = DEFAULT_BATCHES_PER_TRANSACTION;
@Override
- public void load(LoaderContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) throws Exception{
- String driver = linkConf.link.jdbcDriver;
- String url = linkConf.link.connectionString;
- String username = linkConf.link.username;
- String password = linkConf.link.password;
+ public void load(LoaderContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) throws Exception{
+ String driver = linkConfig.linkConfig.jdbcDriver;
+ String url = linkConfig.linkConfig.connectionString;
+ String username = linkConfig.linkConfig.username;
+ String password = linkConfig.linkConfig.password;
GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password);
executor.setAutoCommit(false);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
index 6b11228..2411169 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
@@ -47,7 +47,8 @@ public class GenericJdbcPartitioner extends Partitioner<LinkConfiguration, FromJ
private Boolean partitionColumnNull;
@Override
- public List<Partition> getPartitions(PartitionerContext context,LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
+ public List<Partition> getPartitions(PartitionerContext context, LinkConfiguration linkConfig,
+ FromJobConfiguration fromJobConfig) {
List<Partition> partitions = new LinkedList<Partition>();
numberPartitions = context.getMaxPartitions();
@@ -56,7 +57,7 @@ public class GenericJdbcPartitioner extends Partitioner<LinkConfiguration, FromJ
partitionMinValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE);
partitionMaxValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE);
- partitionColumnNull = fromJobConf.fromJobConfig.partitionColumnNull;
+ partitionColumnNull = fromJobConfig.fromJobConfig.partitionColumnNull;
if (partitionColumnNull == null) {
partitionColumnNull = false;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
index 7bed1d9..e381651 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
@@ -28,26 +28,26 @@ public class GenericJdbcToDestroyer extends Destroyer<LinkConfiguration, ToJobCo
private static final Logger LOG = Logger.getLogger(GenericJdbcToDestroyer.class);
@Override
- public void destroy(DestroyerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
+ public void destroy(DestroyerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
LOG.info("Running generic JDBC connector destroyer");
- final String tableName = toJobConf.toJobConfig.tableName;
- final String stageTableName = toJobConf.toJobConfig.stageTableName;
+ final String tableName = toJobConfig.toJobConfig.tableName;
+ final String stageTableName = toJobConfig.toJobConfig.stageTableName;
final boolean stageEnabled = stageTableName != null &&
stageTableName.length() > 0;
if(stageEnabled) {
- moveDataToDestinationTable(linkConf,
+ moveDataToDestinationTable(linkConfig,
context.isSuccess(), stageTableName, tableName);
}
}
- private void moveDataToDestinationTable(LinkConfiguration linkConf,
+ private void moveDataToDestinationTable(LinkConfiguration linkConfig,
boolean success, String stageTableName, String tableName) {
GenericJdbcExecutor executor =
- new GenericJdbcExecutor(linkConf.link.jdbcDriver,
- linkConf.link.connectionString,
- linkConf.link.username,
- linkConf.link.password);
+ new GenericJdbcExecutor(linkConfig.linkConfig.jdbcDriver,
+ linkConfig.linkConfig.connectionString,
+ linkConfig.linkConfig.username,
+ linkConfig.linkConfig.password);
try {
if(success) {
LOG.info("Job completed, transferring data from stage fromTable to " +
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
index 5d0ec93..1747347 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
@@ -43,35 +43,35 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJ
Logger.getLogger(GenericJdbcToInitializer.class);
@Override
- public void initialize(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
- configureJdbcProperties(context.getContext(), linkConf, toJobConf);
+ public void initialize(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
+ configureJdbcProperties(context.getContext(), linkConfig, toJobConfig);
try {
- configureTableProperties(context.getContext(), linkConf, toJobConf);
+ configureTableProperties(context.getContext(), linkConfig, toJobConfig);
} finally {
executor.close();
}
}
@Override
- public List<String> getJars(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
+ public List<String> getJars(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
List<String> jars = new LinkedList<String>();
- jars.add(ClassUtils.jarForClass(linkConf.link.jdbcDriver));
+ jars.add(ClassUtils.jarForClass(linkConfig.linkConfig.jdbcDriver));
return jars;
}
@Override
- public Schema getSchema(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
- configureJdbcProperties(context.getContext(), linkConf, toJobConf);
+ public Schema getSchema(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
+ configureJdbcProperties(context.getContext(), linkConfig, toJobConfig);
- String schemaName = toJobConf.toJobConfig.tableName;
+ String schemaName = toJobConfig.toJobConfig.tableName;
if (schemaName == null) {
throw new SqoopException(GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0019,
"Table name extraction not supported yet.");
}
- if(toJobConf.toJobConfig.schemaName != null) {
- schemaName = toJobConf.toJobConfig.schemaName + "." + schemaName;
+ if(toJobConfig.toJobConfig.schemaName != null) {
+ schemaName = toJobConfig.toJobConfig.schemaName + "." + schemaName;
}
Schema schema = new Schema(schemaName);
@@ -110,11 +110,11 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJ
}
}
- private void configureJdbcProperties(MutableContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
- String driver = linkConf.link.jdbcDriver;
- String url = linkConf.link.connectionString;
- String username = linkConf.link.username;
- String password = linkConf.link.password;
+ private void configureJdbcProperties(MutableContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
+ String driver = linkConfig.linkConfig.jdbcDriver;
+ String url = linkConfig.linkConfig.connectionString;
+ String username = linkConfig.linkConfig.username;
+ String password = linkConfig.linkConfig.password;
assert driver != null;
assert url != null;
@@ -122,7 +122,7 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJ
executor = new GenericJdbcExecutor(driver, url, username, password);
}
- private void configureTableProperties(MutableContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConfig) {
+ private void configureTableProperties(MutableContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
String dataSql;
String schemaName = toJobConfig.toJobConfig.schemaName;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
index ad1ee5c..93989a4 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
@@ -22,7 +22,7 @@ import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
+import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Validator;
import java.sql.DriverManager;
@@ -34,30 +34,30 @@ import java.sql.SQLException;
public class GenericJdbcValidator extends Validator {
@Override
- public Validation validateLink(Object configuration) {
- Validation validation = new Validation(LinkConfiguration.class);
- LinkConfiguration linkConf = (LinkConfiguration)configuration;
+ public ConfigValidator validateConfigForLink(Object configuration) {
+ ConfigValidator validation = new ConfigValidator(LinkConfiguration.class);
+ LinkConfiguration linkConfig = (LinkConfiguration)configuration;
- if(linkConf.link.jdbcDriver == null) {
+ if(linkConfig.linkConfig.jdbcDriver == null) {
validation.addMessage(Status.UNACCEPTABLE, "link", "jdbcDriver", "Driver can't be empty");
} else {
try {
- Class.forName(linkConf.link.jdbcDriver);
+ Class.forName(linkConfig.linkConfig.jdbcDriver);
} catch (ClassNotFoundException e) {
validation.addMessage(Status.UNACCEPTABLE, "link", "jdbcDriver", "Can't load specified driver");
}
}
- if(linkConf.link.connectionString == null) {
+ if(linkConfig.linkConfig.connectionString == null) {
validation.addMessage(Status.UNACCEPTABLE, "link", "connectionString", "JDBC URL can't be empty");
- } else if(!linkConf.link.connectionString.startsWith("jdbc:")) {
+ } else if(!linkConfig.linkConfig.connectionString.startsWith("jdbc:")) {
validation.addMessage(Status.UNACCEPTABLE, "link", "connectionString", "This do not seem as a valid JDBC URL");
}
// See if we can connect to the database
try {
- DriverManager.getConnection(linkConf.link.connectionString,
- linkConf.link.username, linkConf.link.password);
+ DriverManager.getConnection(linkConfig.linkConfig.connectionString,
+ linkConfig.linkConfig.username, linkConfig.linkConfig.password);
} catch (SQLException e) {
validation.addMessage(Status.ACCEPTABLE, "link", "Can't connect to the database with given credentials: " + e.getMessage());
}
@@ -67,7 +67,7 @@ public class GenericJdbcValidator extends Validator {
}
@Override
- public Validation validateJob(Object jobConfiguration) {
+ public ConfigValidator validateConfigForJob(Object jobConfiguration) {
if (jobConfiguration instanceof FromJobConfiguration) {
return validateFromJobConfiguration((FromJobConfiguration)jobConfiguration);
} else if (jobConfiguration instanceof ToJobConfiguration) {
@@ -78,8 +78,8 @@ public class GenericJdbcValidator extends Validator {
}
}
- private Validation validateToJobConfiguration(ToJobConfiguration configuration) {
- Validation validation = new Validation(FromJobConfiguration.class);
+ private ConfigValidator validateToJobConfiguration(ToJobConfiguration configuration) {
+ ConfigValidator validation = new ConfigValidator(FromJobConfiguration.class);
if(configuration.toJobConfig.tableName == null && configuration.toJobConfig.sql == null) {
validation.addMessage(Status.UNACCEPTABLE, "toJobConfig", "Either table name or SQL must be specified");
@@ -102,8 +102,8 @@ public class GenericJdbcValidator extends Validator {
return validation;
}
- private Validation validateFromJobConfiguration(FromJobConfiguration configuration) {
- Validation validation = new Validation(FromJobConfiguration.class);
+ private ConfigValidator validateFromJobConfiguration(FromJobConfiguration configuration) {
+ ConfigValidator validation = new ConfigValidator(FromJobConfiguration.class);
if(configuration.fromJobConfig.tableName == null && configuration.fromJobConfig.sql == null) {
validation.addMessage(Status.UNACCEPTABLE, "fromJobConfig", "Either table name or SQL must be specified");
[42/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
index b689854..5d48a29 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.connector.hdfs.configuration;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
-@FormClass
+@ConfigClass
public class LinkConfig {
//Todo: Didn't find anything that belongs here...
- // Since empty forms don't work (DERBYREPO_0008:The form contains no input metadata), I'm putting a dummy form here
+ // Since empty forms don't work (DERBYREPO_0008:The config contains no input metadata), I'm putting a dummy config here
@Input(size = 255) public String dummy;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
index 4970821..c0cd336 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
@@ -18,14 +18,14 @@
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
+import org.apache.sqoop.model.Config;
@ConfigurationClass
public class LinkConfiguration {
- @Form
- public LinkConfig link;
+ @Config
+ public LinkConfig linkConfig;
public LinkConfiguration() {
- link = new LinkConfig();
+ linkConfig = new LinkConfig();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
index b1308db..2dfd738 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.connector.hdfs.configuration;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
/**
*
*/
-@FormClass
+@ConfigClass
public class ToJobConfig {
@Input public ToFormat outputFormat;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
index bba249c..c91a975 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
@@ -18,11 +18,11 @@
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
+import org.apache.sqoop.model.Config;
@ConfigurationClass
public class ToJobConfiguration {
- @Form
+ @Config
public ToJobConfig toJobConfig;
public ToJobConfiguration() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
index b603f2f..9b8c6ba 100644
--- a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
+++ b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
@@ -18,12 +18,12 @@
############################
# Link Config
#
-link.label = Link configuration
-link.help = You must supply the information requested in order to \
+linkConfig.label = Link configuration
+linkConfig.help = You must supply the information requested in order to \
create a connection object.
-link.dummy.label = Dummy parameter needed to get HDFS connector to register
-link.dummy.help = You can write anything here. Doesn't matter.
+linkConfig.dummy.label = Dummy parameter needed to get HDFS connector to register
+linkConfig.dummy.help = You can write anything here. Doesn't matter.
# To Job Config
#
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
index 765bedd..f5fbab7 100644
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
@@ -18,6 +18,13 @@
*/
package org.apache.sqoop.connector.idf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Binary;
@@ -26,13 +33,6 @@ import org.apache.sqoop.schema.type.Text;
import org.junit.Before;
import org.junit.Test;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
public class TestCSVIntermediateDataFormat {
private final String BYTE_FIELD_ENCODING = "ISO-8859-1";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
index dbfdc03..54bdd13 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
@@ -19,19 +19,18 @@ package org.apache.sqoop.connector;
import java.io.IOException;
import java.net.URL;
-import java.util.ArrayList;
import java.util.Properties;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.core.ConfigurationConstants;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.core.ConfigurationConstants;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MToConfig;
public final class ConnectorHandler {
@@ -92,26 +91,25 @@ public final class ConnectorHandler {
connectorClassName, ex);
}
- // Initialize Metadata
- MJobForms fromJobForms = null;
- MJobForms toJobForms = null;
+ MFromConfig fromConfig = null;
+ MToConfig toConfig = null;
if (connector.getSupportedDirections().contains(Direction.FROM)) {
- fromJobForms = new MJobForms(FormUtils.toForms(
+ fromConfig = new MFromConfig(ConfigUtils.toConfigs(
connector.getJobConfigurationClass(Direction.FROM)));
}
if (connector.getSupportedDirections().contains(Direction.TO)) {
- toJobForms = new MJobForms(FormUtils.toForms(
+ toConfig = new MToConfig(ConfigUtils.toConfigs(
connector.getJobConfigurationClass(Direction.TO)));
}
- MConnectionForms connectionForms = new MConnectionForms(
- FormUtils.toForms(connector.getLinkConfigurationClass()));
+ MLinkConfig connectionForms = new MLinkConfig(
+ ConfigUtils.toConfigs(connector.getLinkConfigurationClass()));
String connectorVersion = connector.getVersion();
mConnector = new MConnector(connectorUniqueName, connectorClassName, connectorVersion,
- connectionForms, fromJobForms, toJobForms);
+ connectionForms, fromConfig, toConfig);
if (LOG.isInfoEnabled()) {
LOG.info("Connector [" + connectorClassName + "] initialized.");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
index c87df84..5226926 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
@@ -119,7 +119,7 @@ public class ConnectorManager implements Reconfigurable {
return handler.getConnector().getBundle(locale);
}
- public MConnector getConnectorMetadata(long connectorId) {
+ public MConnector getConnectorConfig(long connectorId) {
ConnectorHandler handler = handlerMap.get(nameMap.get(connectorId));
if(handler == null) {
return null;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/Driver.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/Driver.java b/core/src/main/java/org/apache/sqoop/driver/Driver.java
index 5297bde..f1b45bb 100644
--- a/core/src/main/java/org/apache/sqoop/driver/Driver.java
+++ b/core/src/main/java/org/apache/sqoop/driver/Driver.java
@@ -17,6 +17,7 @@
*/
package org.apache.sqoop.driver;
+import java.util.List;
import java.util.Locale;
import java.util.ResourceBundle;
@@ -26,12 +27,12 @@ import org.apache.sqoop.core.ConfigurationConstants;
import org.apache.sqoop.core.Reconfigurable;
import org.apache.sqoop.core.SqoopConfiguration;
import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
-import org.apache.sqoop.driver.configuration.LinkConfiguration;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.validation.Validator;
@@ -92,14 +93,14 @@ public class Driver implements Reconfigurable {
}
/**
- * Driver config structure
+ * Driver structure
*/
- private MDriverConfig mDriverConfig;
+ private MDriver mDriver;
/**
* Validator instance
*/
- private final Validator validator;
+ private final Validator driverValidator;
/**
* Driver config upgrader instance
@@ -111,38 +112,30 @@ public class Driver implements Reconfigurable {
*/
private static final boolean DEFAULT_AUTO_UPGRADE = false;
- public static final String CURRENT_DRIVER_VERSION = "1";
-
- public Class getJobConfigurationClass() {
- return JobConfiguration.class;
- }
-
- public Class getLinkConfigurationClass() {
- return LinkConfiguration.class;
+ public Class getDriverConfigurationGroupClass() {
+ return DriverConfiguration.class;
}
public Driver() {
- MConnectionForms connectionForms = new MConnectionForms(
- FormUtils.toForms(getLinkConfigurationClass())
- );
- mDriverConfig = new MDriverConfig(connectionForms, new MJobForms(FormUtils.toForms(getJobConfigurationClass())),
- CURRENT_DRIVER_VERSION);
+ List<MConfig> driverConfig = ConfigUtils.toConfigs(getDriverConfigurationGroupClass());
+ mDriver = new MDriver(new MDriverConfig(driverConfig), DriverBean.CURRENT_DRIVER_VERSION);
// Build validator
- validator = new DriverValidator();
+ driverValidator = new DriverConfigValidator();
// Build upgrader
driverConfigUpgrader = new DriverConfigUpgrader();
}
public synchronized void initialize() {
- initialize(SqoopConfiguration.getInstance().getContext().getBoolean(ConfigurationConstants.DRIVER_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE));
+ initialize(SqoopConfiguration.getInstance().getContext()
+ .getBoolean(ConfigurationConstants.DRIVER_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE));
}
public synchronized void initialize(boolean autoUpgrade) {
LOG.trace("Begin Driver Config initialization");
// Register driver config in repository
- mDriverConfig = RepositoryManager.getInstance().getRepository().registerDriverConfig(mDriverConfig, autoUpgrade);
+ mDriver = RepositoryManager.getInstance().getRepository().registerDriver(mDriver, autoUpgrade);
SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this));
@@ -154,15 +147,15 @@ public class Driver implements Reconfigurable {
}
public Validator getValidator() {
- return validator;
+ return driverValidator;
}
public RepositoryUpgrader getDriverConfigRepositoryUpgrader() {
return driverConfigUpgrader;
}
- public MDriverConfig getDriverConfig() {
- return mDriverConfig;
+ public MDriver getDriver() {
+ return mDriver;
}
public ResourceBundle getBundle(Locale locale) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
index 8d6eb78..847b73d 100644
--- a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
@@ -18,56 +18,54 @@
*/
package org.apache.sqoop.driver;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJobForms;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import org.apache.sqoop.model.MLinkConfig;
public class DriverConfigUpgrader extends RepositoryUpgrader{
private static final Logger LOG = Logger.getLogger(DriverConfigUpgrader.class);
@Override
- public void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
+ public void upgrade(MLinkConfig original, MLinkConfig upgradeTarget) {
+ // NOTE(VB): There are no link configs anymore for driver, this code remains for previous versions
}
@Override
- public void upgrade(MJobForms original, MJobForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
-
+ public void upgrade(MConfigList original, MConfigList upgradeTarget) {
+ doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@SuppressWarnings("unchecked")
- private void doUpgrade(List<MForm> original, List<MForm> target) {
- // Easier to find the form in the original forms list if we use a map.
- // Since the constructor of MJobForms takes a list,
+ private void doUpgrade(List<MConfig> original, List<MConfig> target) {
+ // Easier to find the config in the original list if we use a map.
+ // Since the constructor takes a list,
// index is not guaranteed to be the same, so we need to look for
// equivalence
- Map<String, MForm> formMap = new HashMap<String, MForm>();
- for (MForm form : original) {
- formMap.put(form.getName(), form);
+ Map<String, MConfig> configMap = new HashMap<String, MConfig>();
+ for (MConfig config : original) {
+ configMap.put(config.getName(), config);
}
- for (MForm form : target) {
- List<MInput<?>> inputs = form.getInputs();
- MForm originalForm = formMap.get(form.getName());
- if(originalForm == null) {
- LOG.warn("Form: " + form.getName() + " not present in old " +
+ for (MConfig config : target) {
+ List<MInput<?>> inputs = config.getInputs();
+ MConfig originalConfig = configMap.get(config.getName());
+ if(originalConfig == null) {
+ LOG.warn("Config: " + config.getName() + " not present in old " +
"driver config. So it will not be transferred by the upgrader.");
continue;
}
for (MInput input : inputs) {
try {
- MInput originalInput = originalForm.getInput(input.getName());
+ MInput originalInput = originalConfig.getInput(input.getName());
input.setValue(originalInput.getValue());
} catch (SqoopException ex) {
LOG.warn("Input: " + input.getName() + " not present in old " +
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
new file mode 100644
index 0000000..9c3b660
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
+import org.apache.sqoop.driver.configuration.ThrottlingConfig;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.ConfigValidator;
+import org.apache.sqoop.validation.Validator;
+
+public class DriverConfigValidator extends Validator {
+ @Override
+ public ConfigValidator validateConfigForJob(Object jobConfiguration) {
+ ConfigValidator validation = new ConfigValidator(DriverConfiguration.class);
+ DriverConfiguration conf = (DriverConfiguration)jobConfiguration;
+ validateThrottlingConfig(validation,conf.throttlingConfig);
+
+ return validation;
+ };
+
+ private void validateThrottlingConfig(ConfigValidator validation, ThrottlingConfig throttlingConfig) {
+ if(throttlingConfig.numExtractors != null && throttlingConfig.numExtractors < 1) {
+ validation.addMessage(Status.UNACCEPTABLE, "throttlingConfig", "numExtractors", "You need to specify more than one extractor");
+ }
+
+ if(throttlingConfig.numLoaders != null && throttlingConfig.numLoaders < 1) {
+ validation.addMessage(Status.UNACCEPTABLE, "throttlingConfig", "numLoaders", "You need to specify more than one loader");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
deleted file mode 100644
index 9cc51dd..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver;
-
-import org.apache.sqoop.driver.configuration.LinkConfiguration;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
-import org.apache.sqoop.driver.configuration.ThrottlingForm;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
-import org.apache.sqoop.validation.Validator;
-
-public class DriverValidator extends Validator {
- @Override
- public Validation validateLink(Object linkConfiguration) {
- Validation validation = new Validation(LinkConfiguration.class);
- // No validation on link object
- return validation;
- }
-
- @Override
- public Validation validateJob(Object jobConfiguration) {
- Validation validation = new Validation(JobConfiguration.class);
- JobConfiguration conf = (JobConfiguration)jobConfiguration;
- validateThrottlingForm(validation,conf.throttling);
-
- return validation;
- };
-
- private void validateThrottlingForm(Validation validation, ThrottlingForm throttling) {
- if(throttling.extractors != null && throttling.extractors < 1) {
- validation.addMessage(Status.UNACCEPTABLE, "throttling", "extractors", "You need to specify more than one extractor");
- }
-
- if(throttling.loaders != null && throttling.loaders < 1) {
- validation.addMessage(Status.UNACCEPTABLE, "throttling", "loaders", "You need to specify more than one loader");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/JobManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobManager.java b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
index e91c436..df2a5ab 100644
--- a/core/src/main/java/org/apache/sqoop/driver/JobManager.java
+++ b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
@@ -30,13 +30,13 @@ import org.apache.sqoop.connector.spi.SqoopConnector;
import org.apache.sqoop.core.Reconfigurable;
import org.apache.sqoop.core.SqoopConfiguration;
import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
import org.apache.sqoop.job.etl.Destroyer;
import org.apache.sqoop.job.etl.DestroyerContext;
import org.apache.sqoop.job.etl.Initializer;
import org.apache.sqoop.job.etl.InitializerContext;
import org.apache.sqoop.job.etl.Transferable;
-import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MSubmission;
@@ -311,35 +311,27 @@ public class JobManager implements Reconfigurable {
SqoopConnector toConnector = getConnector(toConnection.getConnectorId());
validateSupportedDirection(toConnector, Direction.TO);
- // Transform config to fromConnector specific classes
- Object fromConnectionConfig = ClassUtils.instantiate(fromConnector
- .getLinkConfigurationClass());
- FormUtils.fromForms(fromConnection.getConnectorPart().getForms(), fromConnectionConfig);
+ // link config for the FROM part of the job
+ Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass());
+ ConfigUtils.fromConfigs(fromConnection.getConnectorLinkConfig().getConfigs(), fromLinkConfig);
- // Transform config to toConnector specific classes
- Object toConnectorConfig = ClassUtils
- .instantiate(toConnector.getLinkConfigurationClass());
- FormUtils.fromForms(toConnection.getConnectorPart().getForms(), toConnectorConfig);
+ // link config for the TO part of the job
+ Object toLinkConfig = ClassUtils.instantiate(toConnector.getLinkConfigurationClass());
+ ConfigUtils.fromConfigs(toConnection.getConnectorLinkConfig().getConfigs(), toLinkConfig);
+ // from config for the job
Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
- FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromJob);
+ ConfigUtils.fromConfigs(job.getJobConfig(Direction.FROM).getConfigs(), fromJob);
+ // to config for the job
Object toJob = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
- FormUtils.fromForms(job.getConnectorPart(Direction.TO).getForms(), toJob);
+ ConfigUtils.fromConfigs(job.getJobConfig(Direction.TO).getConfigs(), toJob);
- // Transform framework specific configs
- // Q(VB) : Aren't the following 2 exactly the same?
- Object fromDriverConnection = ClassUtils.instantiate(Driver.getInstance()
- .getLinkConfigurationClass());
- FormUtils.fromForms(fromConnection.getFrameworkPart().getForms(), fromDriverConnection);
+ // the only driver config for the job
+ Object driverConfig = ClassUtils
+ .instantiate(Driver.getInstance().getDriverConfigurationGroupClass());
+ ConfigUtils.fromConfigs(job.getDriverConfig().getConfigs(), driverConfig);
- Object toDriverConnection = ClassUtils.instantiate(Driver.getInstance()
- .getLinkConfigurationClass());
- FormUtils.fromForms(toConnection.getFrameworkPart().getForms(), toDriverConnection);
-
- Object frameworkJob = ClassUtils.instantiate(Driver.getInstance()
- .getJobConfigurationClass());
- FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);
// Create a job request for submit/execution
JobRequest jobRequest = executionEngine.createJobRequest();
@@ -347,14 +339,14 @@ public class JobManager implements Reconfigurable {
jobRequest.setSummary(submission);
jobRequest.setConnector(Direction.FROM, fromConnector);
jobRequest.setConnector(Direction.TO, toConnector);
- jobRequest.setConnectorLinkConfig(Direction.FROM, fromConnectionConfig);
- jobRequest.setConnectorLinkConfig(Direction.TO, toConnectorConfig);
- jobRequest.setConnectorJobConfig(Direction.FROM, fromJob);
- jobRequest.setConnectorJobConfig(Direction.TO, toJob);
- // TODO(Abe): Should we actually have 2 different Driver Connection config objects?
- jobRequest.setFrameworkLinkConfig(Direction.FROM, fromDriverConnection);
- jobRequest.setFrameworkLinkConfig(Direction.TO, toDriverConnection);
- jobRequest.setFrameworkJobConfig(frameworkJob);
+
+ jobRequest.setConnectorLinkConfig(Direction.FROM, fromLinkConfig);
+ jobRequest.setConnectorLinkConfig(Direction.TO, toLinkConfig);
+
+ jobRequest.setJobConfig(Direction.FROM, fromJob);
+ jobRequest.setJobConfig(Direction.TO, toJob);
+
+ jobRequest.setDriverConfig(driverConfig);
jobRequest.setJobName(job.getName());
jobRequest.setJobId(job.getPersistenceId());
jobRequest.setNotificationUrl(notificationBaseUrl + jobId);
@@ -453,12 +445,12 @@ public class JobManager implements Reconfigurable {
// Initialize submission from the connector perspective
initializer.initialize(initializerContext, jobRequest.getConnectorLinkConfig(direction),
- jobRequest.getConnectorJobConfig(direction));
+ jobRequest.getJobConfig(direction));
return initializer.getSchema(initializerContext,
jobRequest.getConnectorLinkConfig(direction),
- jobRequest.getConnectorJobConfig(direction));
+ jobRequest.getJobConfig(direction));
}
private void addConnectorInitializerJars(JobRequest jobRequest, Direction direction) {
@@ -468,7 +460,7 @@ public class JobManager implements Reconfigurable {
// Add job specific jars to
jobRequest.addJars(initializer.getJars(initializerContext,
jobRequest.getConnectorLinkConfig(direction),
- jobRequest.getConnectorJobConfig(direction)));
+ jobRequest.getJobConfig(direction)));
}
private Initializer getConnectorInitializer(JobRequest jobRequest, Direction direction) {
@@ -488,12 +480,12 @@ public class JobManager implements Reconfigurable {
}
void prepareJob(JobRequest request) {
- JobConfiguration jobConfiguration = (JobConfiguration) request.getFrameworkJobConfig();
+ DriverConfiguration jobConfiguration = (DriverConfiguration) request.getDriverConfig();
// We're directly moving configured number of extractors and loaders to
// underlying request object. In the future we might need to throttle this
// count based on other running jobs to meet our SLAs.
- request.setExtractors(jobConfiguration.throttling.extractors);
- request.setLoaders(jobConfiguration.throttling.loaders);
+ request.setExtractors(jobConfiguration.throttlingConfig.numExtractors);
+ request.setLoaders(jobConfiguration.throttlingConfig.numLoaders);
// Delegate rest of the job to execution engine
executionEngine.prepareJob(request);
@@ -532,9 +524,9 @@ public class JobManager implements Reconfigurable {
// destroy submission from connector perspective
fromDestroyer.destroy(fromDestroyerContext, request.getConnectorLinkConfig(Direction.FROM),
- request.getConnectorJobConfig(Direction.FROM));
+ request.getJobConfig(Direction.FROM));
toDestroyer.destroy(toDestroyerContext, request.getConnectorLinkConfig(Direction.TO),
- request.getConnectorJobConfig(Direction.TO));
+ request.getJobConfig(Direction.TO));
}
public MSubmission stop(long jobId, HttpEventContext ctx) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
index 63e1e49..2666320 100644
--- a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
+++ b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
@@ -78,11 +78,11 @@ public class JobRequest {
*/
Object fromConnectorLinkConfig;
Object toConnectorLinkConfig;
- Object fromConnectorJobConfig;
- Object toConnectorJobConfig;
- Object fromFrameworkLinkConfig;
- Object toFrameworkLinkConfig;
- Object frameworkJobConfig;
+
+ Object fromConfig;
+ Object toConfig;
+
+ Object driverConfig;
/**
* Connector context (submission specific configuration)
@@ -124,10 +124,9 @@ public class JobRequest {
this.toConnector = null;
this.fromConnectorLinkConfig = null;
this.toConnectorLinkConfig = null;
- this.fromConnectorJobConfig = null;
- this.toConnectorJobConfig = null;
- this.fromFrameworkLinkConfig = null;
- this.toFrameworkLinkConfig = null;
+ this.fromConfig = null;
+ this.toConfig = null;
+ this.driverConfig = null;
}
public MSubmission getSummary() {
@@ -244,64 +243,38 @@ public class JobRequest {
}
}
- public Object getConnectorJobConfig(Direction type) {
- switch(type) {
- case FROM:
- return fromConnectorJobConfig;
-
- case TO:
- return toConnectorJobConfig;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setConnectorJobConfig(Direction type, Object config) {
- switch(type) {
- case FROM:
- fromConnectorJobConfig = config;
- break;
- case TO:
- toConnectorJobConfig = config;
- break;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public Object getFrameworkLinkConfig(Direction type) {
+ public Object getJobConfig(Direction type) {
switch(type) {
case FROM:
- return fromFrameworkLinkConfig;
+ return fromConfig;
case TO:
- return toFrameworkLinkConfig;
+ return toConfig;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
- public void setFrameworkLinkConfig(Direction type, Object config) {
+ public void setJobConfig(Direction type, Object config) {
switch(type) {
case FROM:
- fromFrameworkLinkConfig = config;
+ fromConfig = config;
break;
case TO:
- toFrameworkLinkConfig = config;
+ toConfig = config;
break;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
- public Object getFrameworkJobConfig() {
- return frameworkJobConfig;
+ public Object getDriverConfig() {
+ return driverConfig;
}
- public void setFrameworkJobConfig(Object config) {
- frameworkJobConfig = config;
+ public void setDriverConfig(Object config) {
+ driverConfig = config;
}
public MutableMapContext getConnectorContext(Direction type) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
new file mode 100644
index 0000000..d4e2254
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.ConfigurationClass;
+import org.apache.sqoop.model.Config;
+
+/**
+ * Representing the core job configuration
+ */
+@ConfigurationClass
+public class DriverConfiguration {
+ @Config
+ public ThrottlingConfig throttlingConfig;
+
+ public DriverConfiguration() {
+ throttlingConfig = new ThrottlingConfig();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
deleted file mode 100644
index 908a4eb..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-
-/**
- * Representing the core job configuration
- */
-@ConfigurationClass
-public class JobConfiguration {
- @Form
- public ThrottlingForm throttling;
-
- public JobConfiguration() {
- throttling = new ThrottlingForm();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
deleted file mode 100644
index 3202844..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-
-/**
- * Representing the core link configuration
- */
-@ConfigurationClass
-public class LinkConfiguration {
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
new file mode 100644
index 0000000..357d8e5
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.ConfigClass;
+import org.apache.sqoop.model.Input;
+
+/**
+ * Config to set up number of loaders and extractors
+ */
+@ConfigClass
+public class ThrottlingConfig {
+
+ @Input public Integer numExtractors;
+
+ @Input public Integer numLoaders;
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
deleted file mode 100644
index e73007e..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-
-/**
- * Form to set up number of loaders and extractors
- */
-@FormClass
-public class ThrottlingForm {
-
- @Input public Integer extractors;
-
- @Input public Integer loaders;
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
index 3466116..3ade247 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
@@ -25,7 +25,7 @@ import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
@@ -51,7 +51,7 @@ public class JdbcRepository extends Repository {
/**
* Do what is needed to be done with given link object.
*
- * @param conn Connection to metadata repository.
+ * @param conn Connection to the repository.
* @return Arbitrary value
*/
Object doIt(Connection conn) throws Exception;
@@ -158,8 +158,8 @@ public class JdbcRepository extends Repository {
public Object doIt(Connection conn) throws Exception {
String connectorUniqueName = mConnector.getUniqueName();
- MConnector result = handler.findConnector(connectorUniqueName, conn);
- if (result == null) {
+ MConnector connectorResult = handler.findConnector(connectorUniqueName, conn);
+ if (connectorResult == null) {
handler.registerConnector(mConnector, conn);
return mConnector;
} else {
@@ -167,23 +167,23 @@ public class JdbcRepository extends Repository {
// For now, use the "string" versions itself - later we should
// probably include a build number or something that is
// monotonically increasing.
- if (result.getUniqueName().equals(mConnector.getUniqueName()) &&
- mConnector.getVersion().compareTo(result.getVersion()) > 0) {
+ if (connectorResult.getUniqueName().equals(mConnector.getUniqueName()) &&
+ mConnector.getVersion().compareTo(connectorResult.getVersion()) > 0) {
if (autoUpgrade) {
- upgradeConnector(result, mConnector);
+ upgradeConnector(connectorResult, mConnector);
return mConnector;
} else {
throw new SqoopException(RepositoryError.JDBCREPO_0026,
"Connector: " + mConnector.getUniqueName());
}
}
- if (!result.equals(mConnector)) {
+ if (!connectorResult.equals(mConnector)) {
throw new SqoopException(RepositoryError.JDBCREPO_0013,
"Connector: " + mConnector.getUniqueName()
+ " given: " + mConnector
- + " found: " + result);
+ + " found: " + connectorResult);
}
- return result;
+ return connectorResult;
}
}
});
@@ -220,27 +220,27 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public MDriverConfig registerDriverConfig(final MDriverConfig mDriverConfig, final boolean autoUpgrade) {
- return (MDriverConfig) doWithConnection(new DoWithConnection() {
+ public MDriver registerDriver(final MDriver mDriver, final boolean autoUpgrade) {
+ return (MDriver) doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- MDriverConfig result = handler.findDriverConfig(conn);
- if (result == null) {
- handler.registerDriverConfig(mDriverConfig, conn);
- return mDriverConfig;
+ MDriver existingDriverConfig = handler.findDriver(conn);
+ if (existingDriverConfig == null) {
+ handler.registerDriver(mDriver, conn);
+ return mDriver;
} else {
// We're currently not serializing version into repository
// so let's just compare the structure to see if we need upgrade.
- if(!mDriverConfig.equals(result)) {
+ if(!mDriver.equals(existingDriverConfig)) {
if (autoUpgrade) {
- upgradeDriverConfig(mDriverConfig);
- return mDriverConfig;
+ upgradeDriver(mDriver);
+ return mDriver;
} else {
throw new SqoopException(RepositoryError.JDBCREPO_0026,
- "DriverConfig: " + mDriverConfig.getPersistenceId());
+ "DriverConfig: " + mDriver.getPersistenceId());
}
}
- return result;
+ return existingDriverConfig;
}
}
});
@@ -664,11 +664,11 @@ public class JdbcRepository extends Repository {
}
- protected void updateDriverConfig(final MDriverConfig mDriverConfig, RepositoryTransaction tx) {
+ protected void updateDriver(final MDriver mDriver, RepositoryTransaction tx) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) throws Exception {
- handler.updateDriverConfig(mDriverConfig, conn);
+ handler.updateDriver(mDriver, conn);
return null;
}
}, (JdbcRepositoryTransaction) tx);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
index a743491..97de893 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
@@ -113,11 +113,11 @@ public abstract class JdbcRepositoryHandler {
* which may not have changed). After this operation the repository is
* guaranteed to only have the new forms specified in this object.
*
- * @param mDriverConfig The new data to be inserted into the repository for
+ * @param mDriver The new data to be inserted into the repository for
* the driverConfig.
* @param conn JDBC link for querying repository
*/
- public abstract void updateDriverConfig(MDriverConfig mDriverConfig, Connection conn);
+ public abstract void updateDriver(MDriver mDriver, Connection conn);
/**
@@ -127,7 +127,7 @@ public abstract class JdbcRepositoryHandler {
* @return null if driverConfig are not yet present in repository or
* loaded representation.
*/
- public abstract MDriverConfig findDriverConfig(Connection conn);
+ public abstract MDriver findDriver(Connection conn);
/**
* Register driver config in repository.
@@ -138,7 +138,7 @@ public abstract class JdbcRepositoryHandler {
* @param driverConfig Driver config that should be registered.
* @param conn JDBC link for querying repository.
*/
- public abstract void registerDriverConfig(MDriverConfig driverConfig, Connection conn);
+ public abstract void registerDriver(MDriver driverConfig, Connection conn);
/**
* Return true if repository tables exists and are suitable for use.
@@ -243,7 +243,7 @@ public abstract class JdbcRepositoryHandler {
*
* @param linkId Link id
* @param conn Connection to the repository
- * @return Deserialized form of the link that is saved in repository
+ * @return Deserialized config of the link that is saved in repository
*/
public abstract MLink findLink(long linkId, Connection conn);
@@ -323,7 +323,7 @@ public abstract class JdbcRepositoryHandler {
*
* @param jobId Job id
* @param conn Connection to the repository
- * @return Deserialized form of the job that is present in the repository
+ * @return Deserialized config of the job that is present in the repository
*/
public abstract MJob findJob(long jobId, Connection conn);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/Repository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java
index ea1608a..95c7a4d 100644
--- a/core/src/main/java/org/apache/sqoop/repository/Repository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java
@@ -28,18 +28,21 @@ import org.apache.sqoop.connector.ConnectorManager;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.connector.spi.SqoopConnector;
import org.apache.sqoop.driver.Driver;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.utils.ClassUtils;
-import org.apache.sqoop.validation.Validation;
+import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Validator;
@@ -76,15 +79,26 @@ public abstract class Repository {
/**
* Registers given connector in the repository and return registered
- * variant. This method might return an exception in case that
- * given connector are already registered with different structure.
+ * variant. This method might return an exception in case that
+ * given connector are already registered with different structure
*
* @param mConnector the connector to be registered
- * autoupgrade whether to upgrade driver config automatically
+ * @param autoUpgrade whether to upgrade driver config automatically
* @return Registered connector structure
*/
public abstract MConnector registerConnector(MConnector mConnector, boolean autoUpgrade);
+ /**
+ * Registers given driver and its config in the repository and return registered
+ * variant. This method might return an exception in case that the
+ * given driverConfig are already registered with different structure
+ *
+ * @param mDriverConfig driverConfig to be registered
+ * @param autoUpgrade whether to upgrade driverConfig automatically
+ * @return Registered connector structure
+ */
+ public abstract MDriver registerDriver(MDriver mDriverConfig, boolean autoUpgrade);
+
/**
* Search for connector with given name in repository.
*
@@ -103,18 +117,6 @@ public abstract class Repository {
*/
public abstract List<MConnector> findConnectors();
-
- /**
- * Registers given driverConfig in the repository and return registered
- * variant. This method might return an exception in case that the
- * given driverConfig are already registered with different structure.
- *
- * @param mDriverConfig driverConfig to be registered
- * autoupgrade whether to upgrade driverConfig automatically
- * @return Registered connector structure
- */
- public abstract MDriverConfig registerDriverConfig(MDriverConfig mDriverConfig, boolean autoUpgrade);
-
/**
* Save given link to repository. This link must not be already
* present in the repository otherwise exception will be thrown.
@@ -164,7 +166,7 @@ public abstract class Repository {
* Find link with given id in repository.
*
* @param id Link id
- * @return Deserialized form of the link that is saved in repository
+ * @return Deserialized config of the link that is saved in repository
*/
public abstract MLink findLink(long id);
@@ -184,7 +186,7 @@ public abstract class Repository {
public abstract void createJob(MJob job);
/**
- * Update given job metadata in repository. This object must already be saved
+ * Update given job entity in repository. This object must already be saved
* in repository otherwise exception will be thrown.
*
* @param job Job object that should be updated in the repository
@@ -192,7 +194,7 @@ public abstract class Repository {
public abstract void updateJob(MJob job);
/**
- * Update given job metadata in repository. This object must already be saved
+ * Update given job entity in repository. This object must already be saved
* in repository otherwise exception will be thrown.
*
* @param job Job object that should be updated in the repository
@@ -204,7 +206,7 @@ public abstract class Repository {
public abstract void updateJob(MJob job, RepositoryTransaction tx);
/**
- * Enable or disable job with given id from metadata repository
+ * Enable or disable job with given id from entity repository
*
* @param id Job object that is going to be enabled or disabled
* @param enabled Enable or disable
@@ -212,7 +214,7 @@ public abstract class Repository {
public abstract void enableJob(long id, boolean enabled);
/**
- * Delete job with given id from metadata repository.
+ * Delete job with given id from entity repository.
*
* @param id Job id that should be removed
*/
@@ -222,7 +224,7 @@ public abstract class Repository {
* Find job object with given id.
*
* @param id Job id
- * @return Deserialized form of job loaded from repository
+ * @return Deserialized config of job loaded from repository
*/
public abstract MJob findJob(long id);
@@ -288,8 +290,7 @@ public abstract class Repository {
* @param connectorID Connector ID whose links should be fetched
* @return List of MLink that use <code>connectorID</code>.
*/
- public abstract List<MLink> findLinksForConnector(long
- connectorID);
+ public abstract List<MLink> findLinksForConnector(long connectorID);
/**
* Retrieve jobs which use the given link.
@@ -297,17 +298,16 @@ public abstract class Repository {
* @param connectorID Connector ID whose jobs should be fetched
* @return List of MJobs that use <code>linkID</code>.
*/
- public abstract List<MJob> findJobsForConnector(long
- connectorID);
+ public abstract List<MJob> findJobsForConnector(long connectorID);
/**
* Update the connector with the new data supplied in the
- * <tt>newConnector</tt>. Also Update all forms associated with this
- * connector in the repository with the forms specified in
+ * <tt>newConnector</tt>. Also Update all configs associated with this
+ * connector in the repository with the configs specified in
* <tt>mConnector</tt>. <tt>mConnector </tt> must
- * minimally have the connectorID and all required forms (including ones
+ * minimally have the connectorID and all required configs (including ones
* which may not have changed). After this operation the repository is
- * guaranteed to only have the new forms specified in this object.
+ * guaranteed to only have the new configs specified in this object.
*
* @param newConnector The new data to be inserted into the repository for
* this connector.
@@ -319,22 +319,22 @@ public abstract class Repository {
protected abstract void updateConnector(MConnector newConnector, RepositoryTransaction tx);
/**
- * Update the driverConfig with the new data supplied in the
- * <tt>mDriverConfig</tt>. Also Update all forms associated with the driverConfig
- * in the repository with the forms specified in
+ * Update the driver with the new data supplied in the
+ * <tt>mDriverConfig</tt>. Also Update all configs associated with the driverConfig
+ * in the repository with the configs specified in
* <tt>mDriverConfig</tt>. <tt>mDriverConfig </tt> must
- * minimally have the connectorID and all required forms (including ones
+ * minimally have the connectorID and all required configs (including ones
* which may not have changed). After this operation the repository is
- * guaranteed to only have the new forms specified in this object.
+ * guaranteed to only have the new configs specified in this object.
*
- * @param mDriverConfig The new data to be inserted into the repository for
+ * @param mDriver The new data to be inserted into the repository for
* the driverConfig.
* @param tx The repository transaction to use to push the data to the
* repository. If this is null, a new transaction will be created.
* method will not call begin, commit,
* rollback or close on this transaction.
*/
- protected abstract void updateDriverConfig(MDriverConfig mDriverConfig, RepositoryTransaction tx);
+ protected abstract void updateDriver(MDriver mDriver, RepositoryTransaction tx);
/**
* Delete all inputs for a job
@@ -365,6 +365,13 @@ public abstract class Repository {
}
}
+ private void deleteJobs(List<MJob> jobs, RepositoryTransaction tx) {
+ for (MJob job : jobs) {
+ deleteJobInputs(job.getPersistenceId(), tx);
+ }
+ }
+
+
/**
* Upgrade the connector with the same {@linkplain MConnector#uniqueName}
* in the repository with values from <code>newConnector</code>.
@@ -377,7 +384,7 @@ public abstract class Repository {
* upgraded.
*/
public final void upgradeConnector(MConnector oldConnector, MConnector newConnector) {
- LOG.info("Upgrading metadata for connector: " + oldConnector.getUniqueName());
+ LOG.info("Upgrading connector: " + oldConnector.getUniqueName());
long connectorID = oldConnector.getPersistenceId();
newConnector.setPersistenceId(connectorID);
/* Algorithms:
@@ -385,8 +392,8 @@ public abstract class Repository {
* 2. Get all links associated with the connector.
* 3. Get all jobs associated with the connector.
* 4. Delete the inputs for all of the jobs and links (in that order)
- * 5. Remove all inputs and forms associated with the connector, and
- * register the new forms and inputs.
+ * 5. Remove all inputs and configs associated with the connector, and
+ * register the new configs and inputs.
* 6. Create new links and jobs with connector part being the ones
* returned by the upgrader.
* 7. Validate new links and jobs with connector's validator
@@ -401,85 +408,63 @@ public abstract class Repository {
ConnectorManager.getInstance().getConnector(newConnector
.getUniqueName());
- Validator validator = connector.getValidator();
-
+ Validator connectorConfigValidator = connector.getConfigValidator();
boolean upgradeSuccessful = true;
-
RepositoryUpgrader upgrader = connector.getRepositoryUpgrader();
- List<MLink> links = findLinksForConnector(
- connectorID);
- List<MJob> jobs = findJobsForConnector(connectorID);
+ List<MLink> linksByConnector = findLinksForConnector(connectorID);
+ List<MJob> jobsByConnector = findJobsForConnector(connectorID);
// -- BEGIN TXN --
tx = getTransaction();
tx.begin();
- deletelinksAndJobs(links, jobs, tx);
+ deletelinksAndJobs(linksByConnector, jobsByConnector, tx);
updateConnector(newConnector, tx);
- for (MLink link : links) {
- // Make a new copy of the forms from the connector,
- // else the values will get set in the forms in the connector for
- // each link.
- List<MForm> forms = newConnector.getConnectionForms().clone(false).getForms();
- MConnectionForms newlinkForms = new MConnectionForms(forms);
- upgrader.upgrade(link.getConnectorPart(), newlinkForms);
- MLink newlink = new MLink(link, newlinkForms, link.getFrameworkPart());
-
- // Transform form structures to objects for validations
+ for (MLink oldLink : linksByConnector) {
+ // Make a new copy of the configs
+ List<MConfig> linkConfig = newConnector.getLinkConfig().clone(false).getConfigs();
+ MLinkConfig newLinkConfig = new MLinkConfig(linkConfig);
+ MLinkConfig oldLinkConfig = oldLink.getConnectorLinkConfig();
+ upgrader.upgrade(oldLinkConfig, newLinkConfig);
+
+ MLink newlink = new MLink(oldLink, newLinkConfig);
+
Object newConfigurationObject = ClassUtils.instantiate(connector.getLinkConfigurationClass());
- FormUtils.fromForms(newlink.getConnectorPart().getForms(), newConfigurationObject);
+ ConfigUtils.fromConfigs(newlink.getConnectorLinkConfig().getConfigs(), newConfigurationObject);
- Validation validation = validator.validateLink(newConfigurationObject);
- if (validation.getStatus().canProceed()) {
+ ConfigValidator configValidator = connectorConfigValidator.validateConfigForLink(newConfigurationObject);
+ if (configValidator.getStatus().canProceed()) {
updateLink(newlink, tx);
} else {
- logInvalidModelObject("link", newlink, validation);
+ logInvalidModelObject("link", newlink, configValidator);
upgradeSuccessful = false;
}
}
- for (MJob job : jobs) {
- // Make a new copy of the forms from the connector,
- // else the values will get set in the forms in the connector for
+ for (MJob job : jobsByConnector) {
+ // Make a new copy of the configs
+ // else the values will get set in the configs in the connector for
// each job.
- List<MForm> fromForms = newConnector.getJobForms(Direction.FROM).clone(false).getForms();
- List<MForm> toForms = newConnector.getJobForms(Direction.TO).clone(false).getForms();
+ List<MConfig> fromConfig = newConnector.getConfig(Direction.FROM).clone(false).getConfigs();
+ List<MConfig> toConfig = newConnector.getConfig(Direction.TO).clone(false).getConfigs();
- // New FROM direction forms, old TO direction forms.
+ // New FROM direction configs, old TO direction configs.
if (job.getConnectorId(Direction.FROM) == newConnector.getPersistenceId()) {
- MJobForms newFromJobForms = new MJobForms(fromForms);
- MJobForms oldToJobForms = job.getConnectorPart(Direction.TO);
- upgrader.upgrade(job.getConnectorPart(Direction.FROM), newFromJobForms);
- MJob newJob = new MJob(job, newFromJobForms, oldToJobForms, job.getFrameworkPart());
- updateJob(newJob, tx);
+ MFromConfig newFromConfig = new MFromConfig(fromConfig);
+ MFromConfig oldFromCOnfig = job.getFromJobConfig();
+ upgrader.upgrade(oldFromCOnfig, newFromConfig);
- // Transform form structures to objects for validations
-// Object newFromConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.FROM));
-// FormUtils.fromForms(newJob.getConnectorPart(Direction.FROM).getForms(), newFromConfigurationObject);
-// Validation fromValidation = validator.validateJob(newFromConfigurationObject);
-// if (fromValidation.getStatus().canProceed()) {
-// updateJob(newJob, tx);
-// } else {
-// logInvalidModelObject("job", newJob, fromValidation);
-// upgradeSuccessful = false;
-// }
+ MToConfig oldToConfig = job.getToJobConfig();
+ MJob newJob = new MJob(job, newFromConfig, oldToConfig, job.getDriverConfig());
+ updateJob(newJob, tx);
}
- // Old FROM direction forms, new TO direction forms.
+ // Old FROM direction configs, new TO direction configs.
if (job.getConnectorId(Direction.TO) == newConnector.getPersistenceId()) {
- MJobForms oldFromJobForms = job.getConnectorPart(Direction.FROM);
- MJobForms newToJobForms = new MJobForms(toForms);
- upgrader.upgrade(job.getConnectorPart(Direction.TO), newToJobForms);
- MJob newJob = new MJob(job, oldFromJobForms, newToJobForms, job.getFrameworkPart());
- updateJob(newJob, tx);
- // Transform form structures to objects for validations
-// Object newToConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.TO));
-// FormUtils.fromForms(newJob.getConnectorPart(Direction.TO).getForms(), newToConfigurationObject);
-// Validation toValidation = validator.validateJob(newToConfigurationObject);
-// if (toValidation.getStatus().canProceed()) {
-// updateJob(newJob, tx);
-// } else {
-// logInvalidModelObject("job", newJob, toValidation);
-// upgradeSuccessful = false;
-// }
+ MToConfig oldToConfig = job.getToJobConfig();
+ MToConfig newToConfig = new MToConfig(toConfig);
+ upgrader.upgrade(oldToConfig, newToConfig);
+ MFromConfig oldFromConfig = job.getFromJobConfig();
+ MJob newJob = new MJob(job, oldFromConfig, newToConfig, job.getDriverConfig());
+ updateJob(newJob, tx);
}
}
@@ -506,60 +491,35 @@ public abstract class Repository {
}
}
- public final void upgradeDriverConfig(MDriverConfig driverConfig) {
- LOG.info("Upgrading driver config");
+ public final void upgradeDriver(MDriver driver) {
+ LOG.info("Upgrading driver");
RepositoryTransaction tx = null;
try {
- RepositoryUpgrader upgrader = Driver.getInstance()
+ RepositoryUpgrader driverConfigUpgrader = Driver.getInstance()
.getDriverConfigRepositoryUpgrader();
- List<MLink> links = findLinks();
List<MJob> jobs = findJobs();
Validator validator = Driver.getInstance().getValidator();
-
boolean upgradeSuccessful = true;
// -- BEGIN TXN --
tx = getTransaction();
tx.begin();
- deletelinksAndJobs(links, jobs, tx);
- updateDriverConfig(driverConfig, tx);
- for (MLink link : links) {
- // Make a new copy of the forms from the connector,
- // else the values will get set in the forms in the connector for
- // each link.
- // @TODO(Abe): From/To link forms.
- List<MForm> forms = driverConfig.getConnectionForms().clone(false).getForms();
- MConnectionForms newlinkForms = new MConnectionForms(forms);
- upgrader.upgrade(link.getFrameworkPart(), newlinkForms);
- MLink newlink = new MLink(link, link.getConnectorPart(), newlinkForms);
-
- // Transform form structures to objects for validations
- Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getLinkConfigurationClass());
- FormUtils.fromForms(newlink.getFrameworkPart().getForms(), newConfigurationObject);
-
- Validation validation = validator.validateLink(newConfigurationObject);
- if (validation.getStatus().canProceed()) {
- updateLink(newlink, tx);
- } else {
- logInvalidModelObject("link", newlink, validation);
- upgradeSuccessful = false;
- }
- }
+ deleteJobs(jobs, tx);
+ updateDriver(driver, tx);
+
for (MJob job : jobs) {
- // Make a new copy of the forms from the framework,
- // else the values will get set in the forms in the connector for
- // each link.
- List<MForm> forms = driverConfig.getJobForms().clone(false).getForms();
- MJobForms newJobForms = new MJobForms(forms);
- upgrader.upgrade(job.getFrameworkPart(), newJobForms);
- MJob newJob = new MJob(job, job.getConnectorPart(Direction.FROM), job.getConnectorPart(Direction.TO), newJobForms);
-
- // Transform form structures to objects for validations
- Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getJobConfigurationClass());
- FormUtils.fromForms(newJob.getFrameworkPart().getForms(), newConfigurationObject);
-
- Validation validation = validator.validateJob(newConfigurationObject);
+ // Make a new copy of the configs
+ MDriverConfig driverConfig = driver.getDriverConfig().clone(false);
+ MDriver newDriver = new MDriver(driverConfig, DriverBean.CURRENT_DRIVER_VERSION);
+ driverConfigUpgrader.upgrade(job.getDriverConfig(), newDriver.getDriverConfig());
+ MJob newJob = new MJob(job, job.getFromJobConfig(), job.getToJobConfig(), newDriver.getDriverConfig());
+
+ // Transform config structures to objects for validations
+ Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getDriverConfigurationGroupClass());
+ ConfigUtils.fromConfigs(newJob.getDriverConfig().getConfigs(), newConfigurationObject);
+
+ ConfigValidator validation = validator.validateConfigForJob(newConfigurationObject);
if (validation.getStatus().canProceed()) {
updateJob(newJob, tx);
} else {
@@ -587,14 +547,14 @@ public abstract class Repository {
if(tx != null) {
tx.close();
}
- LOG.info("Driver config upgrade finished");
+ LOG.info("Driver upgrade finished");
}
}
- private void logInvalidModelObject(String objectType, MPersistableEntity entity, Validation validation) {
+ private void logInvalidModelObject(String objectType, MPersistableEntity entity, ConfigValidator validation) {
LOG.error("Upgrader created invalid " + objectType + " with id" + entity.getPersistenceId());
- for(Map.Entry<Validation.FormInput, Validation.Message> entry : validation.getMessages().entrySet()) {
+ for(Map.Entry<ConfigValidator.ConfigInput, ConfigValidator.Message> entry : validation.getMessages().entrySet()) {
LOG.error("\t" + entry.getKey() + ": " + entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
index ae7be82..c2f8505 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
@@ -118,7 +118,7 @@ public class RepositoryManager implements Reconfigurable {
provider.initialize(context);
if(!immutableRepository) {
- LOG.info("Creating or upgrading on disk structures if necessary");
+ LOG.info("Creating or update respository internals at bootup");
provider.getRepository().createOrUpdateInternals();
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/resources/driver-config.properties
----------------------------------------------------------------------
diff --git a/core/src/main/resources/driver-config.properties b/core/src/main/resources/driver-config.properties
index 9ec5d9e..78c3b38 100644
--- a/core/src/main/resources/driver-config.properties
+++ b/core/src/main/resources/driver-config.properties
@@ -19,12 +19,12 @@
# Throttling From
#
-throttling.label = Throttling resources
-throttling.help = Set throttling boundaries to not overload your systems
+throttthrottlingConfigling.label = Throttling resources
+throttlingConfig.help = Set throttling boundaries to not overload your systems
-throttling.extractors.label = Extractors
-throttling.extractors.help = Number of extractors that Sqoop will use
+throttlingConfig.numExtractors.label = Extractors
+throttlingConfig.numExtractors.help = Number of extractors that Sqoop will use
-throttling.loaders.label = Loaders
-throttling.loaders.help = Number of loaders that Sqoop will use
+throttlingConfig.numLoaders.label = Loaders
+throttlingConfig.numLoaders.help = Number of loaders that Sqoop will use
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
index 9c39d23..dc4e8c8 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
@@ -18,16 +18,19 @@
*/
package org.apache.sqoop.driver;
-import org.apache.sqoop.driver.DriverConfigUpgrader;
-import org.apache.sqoop.model.*;
-import org.junit.Before;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import java.util.LinkedList;
import java.util.List;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MStringInput;
+import org.junit.Before;
+import org.junit.Test;
/**
*/
@@ -39,35 +42,14 @@ public class TestDriverConfigUpgrader {
public void initializeUpgrader() {
upgrader = new DriverConfigUpgrader();
}
-
- /**
- * We take the same forms on input and output and we
- * expect that all values will be correctly transferred.
- */
- @Test
- public void testConnectionUpgrade() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection1();
-
- original.getStringInput("f1.s1").setValue("A");
- original.getStringInput("f1.s2").setValue("B");
- original.getIntegerInput("f1.i").setValue(3);
-
- upgrader.upgrade(original, target);
-
- assertEquals("A", target.getStringInput("f1.s1").getValue());
- assertEquals("B", target.getStringInput("f1.s2").getValue());
- assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
- }
-
/**
- * We take the same forms on input and output and we
+ * We take the same configs on input and output and we
* expect that all values will be correctly transferred.
*/
@Test
- public void testJobUpgrade() {
- MJobForms original = job1();
- MJobForms target = job1();
+ public void testJobConfigTyeUpgrade() {
+ MConfigList original = job();
+ MConfigList target = job();
original.getStringInput("f1.s1").setValue("A");
original.getStringInput("f1.s2").setValue("B");
@@ -85,8 +67,8 @@ public class TestDriverConfigUpgrader {
*/
@Test
public void testNonExistingInput() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection2();
+ MConfigList original = job1();
+ MConfigList target = job2();
original.getStringInput("f1.s1").setValue("A");
original.getStringInput("f1.s2").setValue("B");
@@ -104,9 +86,9 @@ public class TestDriverConfigUpgrader {
* therefore is missing in the original.
*/
@Test
- public void testNonExistingForm() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection3();
+ public void testNonExistingConfig() {
+ MConfigList original = job1();
+ MConfigList target = job3();
original.getStringInput("f1.s1").setValue("A");
original.getStringInput("f1.s2").setValue("B");
@@ -119,25 +101,25 @@ public class TestDriverConfigUpgrader {
assertNull(target.getIntegerInput("f2.i").getValue());
}
- MJobForms job1() {
- return new MJobForms(forms1());
+ MConfigList job() {
+ return new MConfigList(configs1());
}
- MConnectionForms connection1() {
- return new MConnectionForms(forms1());
+ MConfigList job1() {
+ return new MConfigList(configs1());
}
- MConnectionForms connection2() {
- return new MConnectionForms(forms2());
+ MConfigList job2() {
+ return new MConfigList(configs2());
}
- MConnectionForms connection3() {
- return new MConnectionForms(forms3());
+ MConfigList job3() {
+ return new MConfigList(configs3());
}
- List<MForm> forms1() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f1", inputs1("f1")));
+ List<MConfig> configs1() {
+ List<MConfig> list = new LinkedList<MConfig>();
+ list.add(new MConfig("f1", inputs1("f1")));
return list;
}
@@ -149,9 +131,9 @@ public class TestDriverConfigUpgrader {
return list;
}
- List<MForm> forms2() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f1", inputs2("f1")));
+ List<MConfig> configs2() {
+ List<MConfig> list = new LinkedList<MConfig>();
+ list.add(new MConfig("f1", inputs2("f1")));
return list;
}
@@ -163,9 +145,9 @@ public class TestDriverConfigUpgrader {
return list;
}
- List<MForm> forms3() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f2", inputs1("f2")));
+ List<MConfig> configs3() {
+ List<MConfig> list = new LinkedList<MConfig>();
+ list.add(new MConfig("f2", inputs1("f2")));
return list;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
index aa3af89..3b475c6 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
@@ -20,6 +20,13 @@ package org.apache.sqoop.driver;
import java.util.Arrays;
import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.ConnectorManager;
@@ -34,13 +41,6 @@ import org.apache.sqoop.request.HttpEventContext;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
public class TestJobManager {
private JobManager jobManager;
private SqoopConnector sqoopConnectorMock;
@@ -99,7 +99,7 @@ public class TestJobManager {
@Test
public void testGetLink() {
- MLink testLink = new MLink(123l, null, null);
+ MLink testLink = new MLink(123l, null);
testLink.setEnabled(true);
MLink mConnectionSpy = org.mockito.Mockito.spy(testLink);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
@@ -111,7 +111,7 @@ public class TestJobManager {
@Test
public void testDisabledLink() {
- MLink testConnection = new MLink(123l, null, null);
+ MLink testConnection = new MLink(123l, null);
testConnection.setPersistenceId(1234);
testConnection.setEnabled(false);
SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection id: "
[14/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/JobManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/JobManager.java b/core/src/main/java/org/apache/sqoop/framework/JobManager.java
deleted file mode 100644
index 8149d1c..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/JobManager.java
+++ /dev/null
@@ -1,710 +0,0 @@
-
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.common.MapContext;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.framework.configuration.JobConfiguration;
-import org.apache.sqoop.request.HttpEventContext;
-import org.apache.sqoop.connector.idf.IntermediateDataFormat;
-import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.core.Reconfigurable;
-import org.apache.sqoop.core.SqoopConfiguration;
-import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
-import org.apache.sqoop.job.etl.*;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MSubmission;
-import org.apache.sqoop.repository.Repository;
-import org.apache.sqoop.repository.RepositoryManager;
-import org.apache.sqoop.schema.Schema;
-import org.apache.sqoop.submission.SubmissionStatus;
-import org.apache.sqoop.submission.counter.Counters;
-import org.apache.sqoop.utils.ClassUtils;
-import org.json.simple.JSONValue;
-
-import java.util.Date;
-import java.util.List;
-
-public class JobManager implements Reconfigurable {
- /**
- * Logger object.
- */
- private static final Logger LOG = Logger.getLogger(JobManager.class);
-
- /**
- * Private instance to singleton of this class.
- */
- private static JobManager instance;
- /**
- * Create default object by default.
- *
- * Every Sqoop server application needs one so this should not be performance
- * issue.
- */
- static {
- instance = new JobManager();
- }
-
- /**
- * Return current instance.
- *
- * @return Current instance
- */
- public static JobManager getInstance() {
- return instance;
- }
-
- /**
- * Allows to set instance in case that it's need.
- *
- * This method should not be normally used as the default instance should be
- * sufficient. One target user use case for this method are unit tests.
- *
- * @param newInstance
- * New instance
- */
- public static void setInstance(JobManager newInstance) {
- instance = newInstance;
- }
-
- /**
- * Default interval for purging old submissions from repository.
- */
- private static final long DEFAULT_PURGE_THRESHOLD = 24 * 60 * 60 * 1000;
-
- /**
- * Default sleep interval for purge thread.
- */
- private static final long DEFAULT_PURGE_SLEEP = 24 * 60 * 60 * 1000;
-
- /**
- * Default interval for update thread.
- */
- private static final long DEFAULT_UPDATE_SLEEP = 60 * 5 * 1000;
-
- /**
- * Configured submission engine instance
- */
- private SubmissionEngine submissionEngine;
-
- /**
- * Configured execution engine instance
- */
- private ExecutionEngine executionEngine;
-
- /**
- * Purge thread that will periodically remove old submissions from repository.
- */
- private PurgeThread purgeThread = null;
-
- /**
- * Update thread that will periodically check status of running submissions.
- */
- private UpdateThread updateThread = null;
-
- /**
- * Synchronization variable between threads.
- */
- private boolean running = true;
-
- /**
- * Specifies how old submissions should be removed from repository.
- */
- private long purgeThreshold;
-
- /**
- * Number of milliseconds for purge thread to sleep.
- */
- private long purgeSleep;
-
- /**
- * Number of milliseconds for update thread to slepp.
- */
- private long updateSleep;
-
- /**
- * Base notification URL.
- *
- * Framework manager will always add job id.
- */
- private String notificationBaseUrl;
-
- /**
- * Set notification base URL.
- *
- * @param url
- * Base URL
- */
- public void setNotificationBaseUrl(String url) {
- LOG.debug("Setting notification base URL to " + url);
- notificationBaseUrl = url;
- }
-
- /**
- * Get base notification url.
- *
- * @return String representation of the URL
- */
- public String getNotificationBaseUrl() {
- return notificationBaseUrl;
- }
-
- public synchronized void destroy() {
- LOG.trace("Begin submission engine manager destroy");
-
- running = false;
-
- try {
- purgeThread.interrupt();
- purgeThread.join();
- } catch (InterruptedException e) {
- // TODO(jarcec): Do I want to wait until it actually finish here?
- LOG.error("Interrupted joining purgeThread");
- }
-
- try {
- updateThread.interrupt();
- updateThread.join();
- } catch (InterruptedException e) {
- // TODO(jarcec): Do I want to wait until it actually finish here?
- LOG.error("Interrupted joining updateThread");
- }
-
- if (submissionEngine != null) {
- submissionEngine.destroy();
- }
-
- if (executionEngine != null) {
- executionEngine.destroy();
- }
- }
-
- public synchronized void initialize() {
- LOG.trace("Begin submission engine manager initialization");
- MapContext context = SqoopConfiguration.getInstance().getContext();
-
- // Let's load configured submission engine
- String submissionEngineClassName =
- context.getString(FrameworkConstants.SYSCFG_SUBMISSION_ENGINE);
-
- submissionEngine = (SubmissionEngine) ClassUtils
- .instantiate(submissionEngineClassName);
- if (submissionEngine == null) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0001,
- submissionEngineClassName);
- }
-
- submissionEngine.initialize(context,
- FrameworkConstants.PREFIX_SUBMISSION_ENGINE_CONFIG);
-
- // Execution engine
- String executionEngineClassName =
- context.getString(FrameworkConstants.SYSCFG_EXECUTION_ENGINE);
-
- executionEngine = (ExecutionEngine) ClassUtils
- .instantiate(executionEngineClassName);
- if (executionEngine == null) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0007,
- executionEngineClassName);
- }
-
- // We need to make sure that user has configured compatible combination of
- // submission engine and execution engine
- if (!submissionEngine
- .isExecutionEngineSupported(executionEngine.getClass())) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0008);
- }
-
- executionEngine.initialize(context,
- FrameworkConstants.PREFIX_EXECUTION_ENGINE_CONFIG);
-
- // Set up worker threads
- purgeThreshold = context.getLong(
- FrameworkConstants.SYSCFG_SUBMISSION_PURGE_THRESHOLD,
- DEFAULT_PURGE_THRESHOLD
- );
- purgeSleep = context.getLong(
- FrameworkConstants.SYSCFG_SUBMISSION_PURGE_SLEEP,
- DEFAULT_PURGE_SLEEP
- );
-
- purgeThread = new PurgeThread();
- purgeThread.start();
-
- updateSleep = context.getLong(
- FrameworkConstants.SYSCFG_SUBMISSION_UPDATE_SLEEP,
- DEFAULT_UPDATE_SLEEP
- );
-
- updateThread = new UpdateThread();
- updateThread.start();
-
- SqoopConfiguration.getInstance().getProvider()
- .registerListener(new CoreConfigurationListener(this));
-
- LOG.info("Submission manager initialized: OK");
- }
-
- public MSubmission submit(long jobId, HttpEventContext ctx) {
-
- MSubmission mSubmission = createJobSubmission(ctx, jobId);
- JobRequest jobRequest = createJobRequest(jobId, mSubmission);
- // Bootstrap job to execute
- prepareJob(jobRequest);
- // Make sure that this job id is not currently running and submit the job
- // only if it's not.
- synchronized (getClass()) {
- MSubmission lastSubmission = RepositoryManager.getInstance().getRepository()
- .findSubmissionLastForJob(jobId);
- if (lastSubmission != null && lastSubmission.getStatus().isRunning()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0002, "Job with id " + jobId);
- }
- // TODO(Abe): Call multiple destroyers.
- // TODO(jarcec): We might need to catch all exceptions here to ensure
- // that Destroyer will be executed in all cases.
- // NOTE: the following is a blocking call
- boolean success = submissionEngine.submit(jobRequest);
- if (!success) {
- destroySubmission(jobRequest);
- mSubmission.setStatus(SubmissionStatus.FAILURE_ON_SUBMIT);
- }
- RepositoryManager.getInstance().getRepository().createSubmission(mSubmission);
- }
- return mSubmission;
- }
-
- private JobRequest createJobRequest(long jobId, MSubmission submission) {
- // get job
- MJob job = getJob(jobId);
-
- // get from/to connections for the job
- MConnection fromConnection = getConnection(job.getConnectionId(Direction.FROM));
- MConnection toConnection = getConnection(job.getConnectionId(Direction.TO));
-
- // get from/to connectors for the connection
- SqoopConnector fromConnector = getConnector(fromConnection.getConnectorId());
- validateSupportedDirection(fromConnector, Direction.FROM);
- SqoopConnector toConnector = getConnector(toConnection.getConnectorId());
- validateSupportedDirection(toConnector, Direction.TO);
-
- // Transform config to fromConnector specific classes
- Object fromConnectionConfig = ClassUtils.instantiate(fromConnector
- .getConnectionConfigurationClass());
- FormUtils.fromForms(fromConnection.getConnectorPart().getForms(), fromConnectionConfig);
-
- // Transform config to toConnector specific classes
- Object toConnectorConfig = ClassUtils
- .instantiate(toConnector.getConnectionConfigurationClass());
- FormUtils.fromForms(toConnection.getConnectorPart().getForms(), toConnectorConfig);
-
- Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
- FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromJob);
-
- Object toJob = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
- FormUtils.fromForms(job.getConnectorPart(Direction.TO).getForms(), toJob);
-
- // Transform framework specific configs
- // Q(VB) : Aren't the following 2 exactly the same?
- Object fromFrameworkConnection = ClassUtils.instantiate(FrameworkManager.getInstance()
- .getConnectionConfigurationClass());
- FormUtils.fromForms(fromConnection.getFrameworkPart().getForms(), fromFrameworkConnection);
-
- Object toFrameworkConnection = ClassUtils.instantiate(FrameworkManager.getInstance()
- .getConnectionConfigurationClass());
- FormUtils.fromForms(toConnection.getFrameworkPart().getForms(), toFrameworkConnection);
-
- Object frameworkJob = ClassUtils.instantiate(FrameworkManager.getInstance()
- .getJobConfigurationClass());
- FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);
-
- // Create a job request for submit/execution
- JobRequest jobRequest = executionEngine.createJobRequest();
- // Save important variables to the job request
- jobRequest.setSummary(submission);
- jobRequest.setConnector(Direction.FROM, fromConnector);
- jobRequest.setConnector(Direction.TO, toConnector);
- jobRequest.setConnectorConnectionConfig(Direction.FROM, fromConnectionConfig);
- jobRequest.setConnectorConnectionConfig(Direction.TO, toConnectorConfig);
- jobRequest.setConnectorJobConfig(Direction.FROM, fromJob);
- jobRequest.setConnectorJobConfig(Direction.TO, toJob);
- // TODO(Abe): Should we actually have 2 different Framework Connection config objects?
- jobRequest.setFrameworkConnectionConfig(Direction.FROM, fromFrameworkConnection);
- jobRequest.setFrameworkConnectionConfig(Direction.TO, toFrameworkConnection);
- jobRequest.setConfigFrameworkJob(frameworkJob);
- jobRequest.setJobName(job.getName());
- jobRequest.setJobId(job.getPersistenceId());
- jobRequest.setNotificationUrl(notificationBaseUrl + jobId);
- Class<? extends IntermediateDataFormat<?>> dataFormatClass =
- fromConnector.getIntermediateDataFormat();
- jobRequest.setIntermediateDataFormat(fromConnector.getIntermediateDataFormat());
-
-
- jobRequest.setFrom(fromConnector.getFrom());
- jobRequest.setTo(toConnector.getTo());
-
- addStandardJars(jobRequest);
- addConnectorJars(jobRequest, fromConnector, toConnector, dataFormatClass);
- addConnectorInitializerJars(jobRequest, Direction.FROM);
- addConnectorInitializerJars(jobRequest, Direction.TO);
-
- Schema fromSchema = getSchemaFromConnector(jobRequest, Direction.FROM);
- Schema toSchema = getSchemaFromConnector(jobRequest, Direction.TO);
-
- // TODO(Gwen): Need better logic here once the Schema refactor: SQOOP-1378
- if (fromSchema != null) {
- jobRequest.getSummary().setFromSchema(fromSchema);
- }
- else {
- jobRequest.getSummary().setFromSchema(toSchema);
- }
- LOG.debug("Using entities: " + jobRequest.getFrom() + ", " + jobRequest.getTo());
- return jobRequest;
- }
-
- private void addConnectorJars(JobRequest jobRequest, SqoopConnector fromConnector,
- SqoopConnector toConnector, Class<? extends IntermediateDataFormat<?>> dataFormatClass) {
- jobRequest.addJarForClass(fromConnector.getClass());
- jobRequest.addJarForClass(toConnector.getClass());
- jobRequest.addJarForClass(dataFormatClass);
- }
-
- private void addStandardJars(JobRequest jobRequest) {
- // Let's register all important jars
- // sqoop-common
- jobRequest.addJarForClass(MapContext.class);
- // sqoop-core
- jobRequest.addJarForClass(FrameworkManager.class);
- // sqoop-spi
- jobRequest.addJarForClass(SqoopConnector.class);
- // Execution engine jar
- jobRequest.addJarForClass(executionEngine.getClass());
- // Extra libraries that Sqoop code requires
- jobRequest.addJarForClass(JSONValue.class);
- }
-
- MSubmission createJobSubmission(HttpEventContext ctx, long jobId) {
- MSubmission summary = new MSubmission(jobId);
- summary.setCreationUser(ctx.getUsername());
- summary.setLastUpdateUser(ctx.getUsername());
- return summary;
- }
-
- SqoopConnector getConnector(long connnectorId) {
- return ConnectorManager.getInstance().getConnector(connnectorId);
- }
-
- void validateSupportedDirection(SqoopConnector connector, Direction direction) {
- // Make sure that connector supports the given direction
- if (!connector.getSupportedDirections().contains(direction)) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0011, "Connector: "
- + connector.getClass().getCanonicalName());
- }
- }
-
- MConnection getConnection(long connectionId) {
- MConnection connection = RepositoryManager.getInstance().getRepository()
- .findConnection(connectionId);
- if (!connection.getEnabled()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0010, "Connection id: "
- + connection.getPersistenceId());
- }
- return connection;
- }
-
- MJob getJob(long jobId) {
- MJob job = RepositoryManager.getInstance().getRepository().findJob(jobId);
- if (job == null) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0004, "Unknown job id: " + jobId);
- }
-
- if (!job.getEnabled()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0009, "Job id: " + job.getPersistenceId());
- }
- return job;
- }
-
- private Schema getSchemaFromConnector(JobRequest jobRequest, Direction direction) {
-
- Initializer initializer = getConnectorInitializer(jobRequest, direction);
-
- // Initializer context
- InitializerContext initializerContext = getInitializerContext(jobRequest, direction);
-
- // Initialize submission from the connector perspective
- initializer.initialize(initializerContext, jobRequest.getConnectorConnectionConfig(direction),
- jobRequest.getConnectorJobConfig(direction));
-
- // TODO(Abe): Alter behavior of Schema here.
- return initializer.getSchema(initializerContext,
- jobRequest.getConnectorConnectionConfig(direction),
- jobRequest.getConnectorJobConfig(direction));
- }
-
- private void addConnectorInitializerJars(JobRequest jobRequest, Direction direction) {
-
- Initializer initializer = getConnectorInitializer(jobRequest, direction);
- InitializerContext initializerContext = getInitializerContext(jobRequest, direction);
- // Add job specific jars to
- jobRequest.addJars(initializer.getJars(initializerContext,
- jobRequest.getConnectorConnectionConfig(direction),
- jobRequest.getConnectorJobConfig(direction)));
- }
-
- private Initializer getConnectorInitializer(JobRequest jobRequest, Direction direction) {
- Transferable transferable = direction.equals(Direction.FROM) ? jobRequest.getFrom() : jobRequest.getTo();
- Class<? extends Initializer> initializerClass = transferable.getInitializer();
- Initializer initializer = (Initializer) ClassUtils.instantiate(initializerClass);
-
- if (initializer == null) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0006,
- "Can't create connector initializer instance: " + initializerClass.getName());
- }
- return initializer;
- }
-
- private InitializerContext getInitializerContext(JobRequest jobRequest, Direction direction) {
- return new InitializerContext(jobRequest.getConnectorContext(direction));
- }
-
- void prepareJob(JobRequest request) {
- JobConfiguration jobConfiguration = (JobConfiguration) request.getConfigFrameworkJob();
- // We're directly moving configured number of extractors and loaders to
- // underlying request object. In the future we might need to throttle this
- // count based on other running jobs to meet our SLAs.
- request.setExtractors(jobConfiguration.throttling.extractors);
- request.setLoaders(jobConfiguration.throttling.loaders);
-
- // Delegate rest of the job to execution engine
- executionEngine.prepareJob(request);
- }
-
- /**
- * Callback that will be called only if we failed to submit the job to the
- * remote cluster.
- */
- void destroySubmission(JobRequest request) {
- Transferable from = request.getFrom();
- Transferable to = request.getTo();
-
- Class<? extends Destroyer> fromDestroyerClass = from.getDestroyer();
- Class<? extends Destroyer> toDestroyerClass = to.getDestroyer();
- Destroyer fromDestroyer = (Destroyer) ClassUtils.instantiate(fromDestroyerClass);
- Destroyer toDestroyer = (Destroyer) ClassUtils.instantiate(toDestroyerClass);
-
- if (fromDestroyer == null) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0006,
- "Can't create toDestroyer instance: " + fromDestroyerClass.getName());
- }
-
- if (toDestroyer == null) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0006,
- "Can't create toDestroyer instance: " + toDestroyerClass.getName());
- }
-
- // TODO(Abe): Update context to manage multiple connectors. As well as summary.
- DestroyerContext fromDestroyerContext = new DestroyerContext(
- request.getConnectorContext(Direction.FROM), false, request.getSummary()
- .getFromSchema());
- DestroyerContext toDestroyerContext = new DestroyerContext(
- request.getConnectorContext(Direction.TO), false, request.getSummary()
- .getToSchema());
-
- // destroy submission from connector perspective
- fromDestroyer.destroy(fromDestroyerContext, request.getConnectorConnectionConfig(Direction.FROM),
- request.getConnectorJobConfig(Direction.FROM));
- toDestroyer.destroy(toDestroyerContext, request.getConnectorConnectionConfig(Direction.TO),
- request.getConnectorJobConfig(Direction.TO));
- }
-
- public MSubmission stop(long jobId, HttpEventContext ctx) {
-
- Repository repository = RepositoryManager.getInstance().getRepository();
- MSubmission mSubmission = repository.findSubmissionLastForJob(jobId);
-
- if (mSubmission == null || !mSubmission.getStatus().isRunning()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0003, "Job with id " + jobId
- + " is not running");
- }
- submissionEngine.stop(mSubmission.getExternalId());
-
- mSubmission.setLastUpdateUser(ctx.getUsername());
-
- // Fetch new information to verify that the stop command has actually worked
- update(mSubmission);
-
- // Return updated structure
- return mSubmission;
- }
-
- public MSubmission status(long jobId) {
- Repository repository = RepositoryManager.getInstance().getRepository();
- MSubmission mSubmission = repository.findSubmissionLastForJob(jobId);
-
- if (mSubmission == null) {
- return new MSubmission(jobId, new Date(), SubmissionStatus.NEVER_EXECUTED);
- }
-
- // If the submission is in running state, let's update it
- if (mSubmission.getStatus().isRunning()) {
- update(mSubmission);
- }
-
- return mSubmission;
- }
-
- private void update(MSubmission submission) {
- double progress = -1;
- Counters counters = null;
- String externalId = submission.getExternalId();
- SubmissionStatus newStatus = submissionEngine.status(externalId);
- String externalLink = submissionEngine.externalLink(externalId);
-
- if (newStatus.isRunning()) {
- progress = submissionEngine.progress(externalId);
- } else {
- counters = submissionEngine.counters(externalId);
- }
-
- submission.setStatus(newStatus);
- submission.setProgress(progress);
- submission.setCounters(counters);
- submission.setExternalLink(externalLink);
- submission.setLastUpdateDate(new Date());
-
- RepositoryManager.getInstance().getRepository()
- .updateSubmission(submission);
- }
-
- @Override
- public synchronized void configurationChanged() {
- LOG.info("Begin submission engine manager reconfiguring");
- MapContext newContext = SqoopConfiguration.getInstance().getContext();
- MapContext oldContext = SqoopConfiguration.getInstance().getOldContext();
-
- String newSubmissionEngineClassName = newContext
- .getString(FrameworkConstants.SYSCFG_SUBMISSION_ENGINE);
- if (newSubmissionEngineClassName == null
- || newSubmissionEngineClassName.trim().length() == 0) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0001,
- newSubmissionEngineClassName);
- }
-
- String oldSubmissionEngineClassName = oldContext
- .getString(FrameworkConstants.SYSCFG_SUBMISSION_ENGINE);
- if (!newSubmissionEngineClassName.equals(oldSubmissionEngineClassName)) {
- LOG.warn("Submission engine cannot be replaced at the runtime. " +
- "You might need to restart the server.");
- }
-
- String newExecutionEngineClassName = newContext
- .getString(FrameworkConstants.SYSCFG_EXECUTION_ENGINE);
- if (newExecutionEngineClassName == null
- || newExecutionEngineClassName.trim().length() == 0) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0007,
- newExecutionEngineClassName);
- }
-
- String oldExecutionEngineClassName = oldContext
- .getString(FrameworkConstants.SYSCFG_EXECUTION_ENGINE);
- if (!newExecutionEngineClassName.equals(oldExecutionEngineClassName)) {
- LOG.warn("Execution engine cannot be replaced at the runtime. " +
- "You might need to restart the server.");
- }
-
- // Set up worker threads
- purgeThreshold = newContext.getLong(
- FrameworkConstants.SYSCFG_SUBMISSION_PURGE_THRESHOLD,
- DEFAULT_PURGE_THRESHOLD
- );
- purgeSleep = newContext.getLong(
- FrameworkConstants.SYSCFG_SUBMISSION_PURGE_SLEEP,
- DEFAULT_PURGE_SLEEP
- );
- purgeThread.interrupt();
-
- updateSleep = newContext.getLong(
- FrameworkConstants.SYSCFG_SUBMISSION_UPDATE_SLEEP,
- DEFAULT_UPDATE_SLEEP
- );
- updateThread.interrupt();
-
- LOG.info("Submission engine manager reconfigured.");
- }
-
- private class PurgeThread extends Thread {
- public PurgeThread() {
- super("PurgeThread");
- }
-
- public void run() {
- LOG.info("Starting submission manager purge thread");
-
- while (running) {
- try {
- LOG.info("Purging old submissions");
- Date threshold = new Date((new Date()).getTime() - purgeThreshold);
- RepositoryManager.getInstance().getRepository()
- .purgeSubmissions(threshold);
- Thread.sleep(purgeSleep);
- } catch (InterruptedException e) {
- LOG.debug("Purge thread interrupted", e);
- }
- }
-
- LOG.info("Ending submission manager purge thread");
- }
- }
-
- private class UpdateThread extends Thread {
- public UpdateThread() {
- super("UpdateThread");
- }
-
- public void run() {
- LOG.info("Starting submission manager update thread");
-
- while (running) {
- try {
- LOG.debug("Updating running submissions");
-
- // Let's get all running submissions from repository to check them out
- List<MSubmission> unfinishedSubmissions =
- RepositoryManager.getInstance().getRepository()
- .findSubmissionsUnfinished();
-
- for (MSubmission submission : unfinishedSubmissions) {
- update(submission);
- }
-
- Thread.sleep(updateSleep);
- } catch (InterruptedException e) {
- LOG.debug("Purge thread interrupted", e);
- }
- }
-
- LOG.info("Ending submission manager update thread");
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/JobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/JobRequest.java b/core/src/main/java/org/apache/sqoop/framework/JobRequest.java
deleted file mode 100644
index 1f77693..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/JobRequest.java
+++ /dev/null
@@ -1,356 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.common.DirectionError;
-import org.apache.sqoop.common.MutableMapContext;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.idf.IntermediateDataFormat;
-import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.job.etl.Transferable;
-import org.apache.sqoop.model.MSubmission;
-import org.apache.sqoop.utils.ClassUtils;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Submission details class is used when creating new submission and contains
- * all information that we need to create a new submission (including mappers,
- * reducers, ...).
- */
-public class JobRequest {
-
- /**
- * Submission summary
- */
- MSubmission summary;
-
- /**
- * Original job name
- */
- String jobName;
-
- /**
- * Associated job (from metadata perspective) id
- */
- long jobId;
-
- /**
- * Connector instances associated with this submission request
- */
- SqoopConnector fromConnector;
- SqoopConnector toConnector;
-
- /**
- * List of required local jars for the job
- */
- List<String> jars;
-
- /**
- * From entity
- */
- Transferable from;
-
- /**
- * To entity
- */
- Transferable to;
-
- /**
- * All configuration objects
- */
- Object fromConnectorConnectionConfig;
- Object toConnectorConnectionConfig;
- Object fromConnectorJobConfig;
- Object toConnectorJobConfig;
- Object fromFrameworkConnectionConfig;
- Object toFrameworkConnectionConfig;
- Object configFrameworkJob;
-
- /**
- * Connector context (submission specific configuration)
- */
- MutableMapContext fromConnectorContext;
- MutableMapContext toConnectorContext;
-
- /**
- * Framework context (submission specific configuration)
- */
- MutableMapContext frameworkContext;
-
- /**
- * Optional notification URL for job progress
- */
- String notificationUrl;
-
- /**
- * Number of extractors
- */
- Integer extractors;
-
- /**
- * Number of loaders
- */
- Integer loaders;
-
- /**
- * The intermediate data format this submission should use.
- */
- Class<? extends IntermediateDataFormat> intermediateDataFormat;
-
- public JobRequest() {
- this.jars = new LinkedList<String>();
- this.fromConnectorContext = new MutableMapContext();
- this.toConnectorContext = new MutableMapContext();
- this.frameworkContext = new MutableMapContext();
- this.fromConnector = null;
- this.toConnector = null;
- this.fromConnectorConnectionConfig = null;
- this.toConnectorConnectionConfig = null;
- this.fromConnectorJobConfig = null;
- this.toConnectorJobConfig = null;
- this.fromFrameworkConnectionConfig = null;
- this.toFrameworkConnectionConfig = null;
- }
-
- public MSubmission getSummary() {
- return summary;
- }
-
- public void setSummary(MSubmission summary) {
- this.summary = summary;
- }
-
- public String getJobName() {
- return jobName;
- }
-
- public void setJobName(String jobName) {
- this.jobName = jobName;
- }
-
- public long getJobId() {
- return jobId;
- }
-
- public void setJobId(long jobId) {
- this.jobId = jobId;
- }
-
- public SqoopConnector getConnector(Direction type) {
- switch(type) {
- case FROM:
- return fromConnector;
-
- case TO:
- return toConnector;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setConnector(Direction type, SqoopConnector connector) {
- switch(type) {
- case FROM:
- fromConnector = connector;
- break;
-
- case TO:
- toConnector = connector;
- break;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public List<String> getJars() {
- return jars;
- }
-
- public void addJar(String jar) {
- if(!jars.contains(jar)) {
- jars.add(jar);
- }
- }
-
- public void addJarForClass(Class klass) {
- addJar(ClassUtils.jarForClass(klass));
- }
-
- public void addJars(List<String> jars) {
- for(String j : jars) {
- addJar(j);
- }
- }
-
- public Transferable getFrom() {
- return from;
- }
-
- public void setFrom(Transferable from) {
- this.from = from;
- }
-
- public Transferable getTo() {
- return to;
- }
-
- public void setTo(Transferable to) {
- this.to = to;
- }
-
- public Object getConnectorConnectionConfig(Direction type) {
- switch(type) {
- case FROM:
- return fromConnectorConnectionConfig;
-
- case TO:
- return toConnectorConnectionConfig;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setConnectorConnectionConfig(Direction type, Object config) {
- switch(type) {
- case FROM:
- fromConnectorConnectionConfig = config;
- break;
- case TO:
- toConnectorConnectionConfig = config;
- break;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public Object getConnectorJobConfig(Direction type) {
- switch(type) {
- case FROM:
- return fromConnectorJobConfig;
-
- case TO:
- return toConnectorJobConfig;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setConnectorJobConfig(Direction type, Object config) {
- switch(type) {
- case FROM:
- fromConnectorJobConfig = config;
- break;
- case TO:
- toConnectorJobConfig = config;
- break;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public Object getFrameworkConnectionConfig(Direction type) {
- switch(type) {
- case FROM:
- return fromFrameworkConnectionConfig;
-
- case TO:
- return toFrameworkConnectionConfig;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setFrameworkConnectionConfig(Direction type, Object config) {
- switch(type) {
- case FROM:
- fromFrameworkConnectionConfig = config;
- break;
- case TO:
- toFrameworkConnectionConfig = config;
- break;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public Object getConfigFrameworkJob() {
- return configFrameworkJob;
- }
-
- public void setConfigFrameworkJob(Object config) {
- configFrameworkJob = config;
- }
-
- public MutableMapContext getConnectorContext(Direction type) {
- switch(type) {
- case FROM:
- return fromConnectorContext;
-
- case TO:
- return toConnectorContext;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public MutableMapContext getFrameworkContext() {
- return frameworkContext;
- }
-
- public String getNotificationUrl() {
- return notificationUrl;
- }
-
- public void setNotificationUrl(String url) {
- this.notificationUrl = url;
- }
-
- public Integer getExtractors() {
- return extractors;
- }
-
- public void setExtractors(Integer extractors) {
- this.extractors = extractors;
- }
-
- public Integer getLoaders() {
- return loaders;
- }
-
- public void setLoaders(Integer loaders) {
- this.loaders = loaders;
- }
-
- public Class<? extends IntermediateDataFormat> getIntermediateDataFormat() {
- return intermediateDataFormat;
- }
-
- public void setIntermediateDataFormat(Class<? extends IntermediateDataFormat> intermediateDataFormat) {
- this.intermediateDataFormat = intermediateDataFormat;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java b/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java
deleted file mode 100644
index 732be3b..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.common.MapContext;
-import org.apache.sqoop.submission.counter.Counters;
-import org.apache.sqoop.submission.SubmissionStatus;
-
-/**
- * Submission engine is responsible in conveying the information about the
- * job instances (submissions) to remote (hadoop) cluster.
- */
-public abstract class SubmissionEngine {
-
- /**
- * Initialize submission engine
- *
- * @param context Configuration context
- * @param prefix Submission engine prefix
- */
- public void initialize(MapContext context, String prefix) {
- }
-
- /**
- * Destroy submission engine when stopping server
- */
- public void destroy() {
- }
-
- /**
- * Callback to verify that configured submission engine and execution engine
- * are compatible.
- *
- * @param executionEngineClass Configured execution class.
- * @return True if such execution engine is supported
- */
- public abstract boolean isExecutionEngineSupported(Class executionEngineClass);
-
- /**
- * Submit new job to remote (hadoop) cluster. This method *must* fill
- * submission.getSummary.setExternalId(), otherwise Sqoop framework won't
- * be able to track progress on this job!
- *
- * @return Return true if we were able to submit job to remote cluster.
- */
- public abstract boolean submit(JobRequest submission);
-
- /**
- * Hard stop for given submission.
- *
- * @param submissionId Submission internal id.
- */
- public abstract void stop(String submissionId);
-
- /**
- * Return status of given submission.
- *
- * @param submissionId Submission internal id.
- * @return Current submission status.
- */
- public abstract SubmissionStatus status(String submissionId);
-
- /**
- * Return submission progress.
- *
- * Expected is number from interval <0, 1> denoting how far the processing
- * has gone or -1 in case that this submission engine do not supports
- * progress reporting.
- *
- * @param submissionId Submission internal id.
- * @return {-1} union <0, 1>
- */
- public double progress(String submissionId) {
- return -1;
- }
-
- /**
- * Return statistics for given submission id.
- *
- * Sqoop framework will call counters only for submission in state SUCCEEDED,
- * it's consider exceptional state to call this method for other states.
- *
- * @param submissionId Submission internal id.
- * @return Submission statistics
- */
- public Counters counters(String submissionId) {
- return null;
- }
-
- /**
- * Return link to external web page with given submission.
- *
- * @param submissionId Submission internal id.
- * @return Null in case that external page is not supported or available or
- * HTTP link to given submission.
- */
- public String externalLink(String submissionId) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java b/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java
deleted file mode 100644
index 897d3c7..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-
-/**
- * Framework class representing connection configuration
- */
-@ConfigurationClass
-public class ConnectionConfiguration {
-
- @Form SecurityForm security = new SecurityForm();
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/configuration/JobConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/configuration/JobConfiguration.java b/core/src/main/java/org/apache/sqoop/framework/configuration/JobConfiguration.java
deleted file mode 100644
index 0abc611..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/configuration/JobConfiguration.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-
-@ConfigurationClass
-public class JobConfiguration {
- @Form
- public ThrottlingForm throttling;
-
- public JobConfiguration() {
- throttling = new ThrottlingForm();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/configuration/SecurityForm.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/configuration/SecurityForm.java b/core/src/main/java/org/apache/sqoop/framework/configuration/SecurityForm.java
deleted file mode 100644
index 8ab50ed..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/configuration/SecurityForm.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-
-/**
- * Security form
- */
-@FormClass
-public class SecurityForm {
- @Input public Integer maxConnections;
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/framework/configuration/ThrottlingForm.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/configuration/ThrottlingForm.java b/core/src/main/java/org/apache/sqoop/framework/configuration/ThrottlingForm.java
deleted file mode 100644
index c435f6b..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/configuration/ThrottlingForm.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-
-/**
- * Form to set up number of loaders and extractors
- */
-@FormClass
-public class ThrottlingForm {
-
- @Input public Integer extractors;
-
- @Input public Integer loaders;
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
index fa119a5..3466116 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
@@ -23,9 +23,9 @@ import java.util.List;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
@@ -43,13 +43,13 @@ public class JdbcRepository extends Repository {
}
/**
- * Private interface to wrap specific code that requires fresh connection to
- * repository with general code that will get the connection and handle
+ * Private interface to wrap specific code that requires fresh link to
+ * repository with general code that will get the link and handle
* exceptions.
*/
private interface DoWithConnection {
/**
- * Do what is needed to be done with given connection object.
+ * Do what is needed to be done with given link object.
*
* @param conn Connection to metadata repository.
* @return Arbitrary value
@@ -62,7 +62,7 @@ public class JdbcRepository extends Repository {
}
/**
- * Handle transaction and connection functionality and delegate action to
+ * Handle transaction and link functionality and delegate action to
* given delegator.
*
* @param delegator Code for specific action
@@ -77,7 +77,7 @@ public class JdbcRepository extends Repository {
boolean shouldCloseTxn = false;
try {
- // Get transaction and connection
+ // Get transaction and link
Connection conn;
if (tx == null) {
tx = getTransaction();
@@ -205,6 +205,7 @@ public class JdbcRepository extends Repository {
/**
* {@inheritDoc}
*/
+ @SuppressWarnings("unchecked")
@Override
public List<MConnector> findConnectors() {
return (List<MConnector>) doWithConnection(new DoWithConnection() {
@@ -219,24 +220,24 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public MFramework registerFramework(final MFramework mFramework, final boolean autoUpgrade) {
- return (MFramework) doWithConnection(new DoWithConnection() {
+ public MDriverConfig registerDriverConfig(final MDriverConfig mDriverConfig, final boolean autoUpgrade) {
+ return (MDriverConfig) doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- MFramework result = handler.findFramework(conn);
+ MDriverConfig result = handler.findDriverConfig(conn);
if (result == null) {
- handler.registerFramework(mFramework, conn);
- return mFramework;
+ handler.registerDriverConfig(mDriverConfig, conn);
+ return mDriverConfig;
} else {
- // We're currently not serializing framework version into repository
+ // We're currently not serializing version into repository
// so let's just compare the structure to see if we need upgrade.
- if(!mFramework.equals(result)) {
+ if(!mDriverConfig.equals(result)) {
if (autoUpgrade) {
- upgradeFramework(mFramework);
- return mFramework;
+ upgradeDriverConfig(mDriverConfig);
+ return mDriverConfig;
} else {
throw new SqoopException(RepositoryError.JDBCREPO_0026,
- "Framework: " + mFramework.getPersistenceId());
+ "DriverConfig: " + mDriverConfig.getPersistenceId());
}
}
return result;
@@ -249,15 +250,15 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public void createConnection(final MConnection connection) {
+ public void createLink(final MLink link) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- if(connection.hasPersistenceId()) {
+ if(link.hasPersistenceId()) {
throw new SqoopException(RepositoryError.JDBCREPO_0015);
}
- handler.createConnection(connection, conn);
+ handler.createLink(link, conn);
return null;
}
});
@@ -267,28 +268,27 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public void updateConnection(final MConnection connection) {
- updateConnection(connection, null);
+ public void updateLink(final MLink link) {
+ updateLink(link, null);
}
/**
* {@inheritDoc}
*/
@Override
- public void updateConnection(final MConnection connection,
- RepositoryTransaction tx) {
+ public void updateLink(final MLink link, RepositoryTransaction tx) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- if(!connection.hasPersistenceId()) {
+ if (!link.hasPersistenceId()) {
throw new SqoopException(RepositoryError.JDBCREPO_0016);
}
- if(!handler.existsConnection(connection.getPersistenceId(), conn)) {
- throw new SqoopException(RepositoryError.JDBCREPO_0017,
- "Invalid id: " + connection.getPersistenceId());
+ if (!handler.existsLink(link.getPersistenceId(), conn)) {
+ throw new SqoopException(RepositoryError.JDBCREPO_0017, "Invalid id: "
+ + link.getPersistenceId());
}
- handler.updateConnection(connection, conn);
+ handler.updateLink(link, conn);
return null;
}
}, (JdbcRepositoryTransaction) tx);
@@ -298,16 +298,16 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public void enableConnection(final long connectionId, final boolean enabled) {
+ public void enableLink(final long linkId, final boolean enabled) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- if(!handler.existsConnection(connectionId, conn)) {
+ if(!handler.existsLink(linkId, conn)) {
throw new SqoopException(RepositoryError.JDBCREPO_0017,
- "Invalid id: " + connectionId);
+ "Invalid id: " + linkId);
}
- handler.enableConnection(connectionId, enabled, conn);
+ handler.enableLink(linkId, enabled, conn);
return null;
}
});
@@ -317,20 +317,20 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public void deleteConnection(final long connectionId) {
+ public void deleteLink(final long linkId) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- if(!handler.existsConnection(connectionId, conn)) {
+ if(!handler.existsLink(linkId, conn)) {
throw new SqoopException(RepositoryError.JDBCREPO_0017,
- "Invalid id: " + connectionId);
+ "Invalid id: " + linkId);
}
- if(handler.inUseConnection(connectionId, conn)) {
+ if(handler.inUseLink(linkId, conn)) {
throw new SqoopException(RepositoryError.JDBCREPO_0021,
- "Id in use: " + connectionId);
+ "Id in use: " + linkId);
}
- handler.deleteConnection(connectionId, conn);
+ handler.deleteLink(linkId, conn);
return null;
}
});
@@ -340,11 +340,11 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public MConnection findConnection(final long connectionId) {
- return (MConnection) doWithConnection(new DoWithConnection() {
+ public MLink findLink(final long connectionId) {
+ return (MLink) doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- return handler.findConnection(connectionId, conn);
+ return handler.findLink(connectionId, conn);
}
});
}
@@ -354,11 +354,11 @@ public class JdbcRepository extends Repository {
*/
@SuppressWarnings("unchecked")
@Override
- public List<MConnection> findConnections() {
- return (List<MConnection>) doWithConnection(new DoWithConnection() {
+ public List<MLink> findLinks() {
+ return (List<MLink>) doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- return handler.findConnections(conn);
+ return handler.findLinks(conn);
}
});
}
@@ -601,12 +601,12 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public List<MConnection> findConnectionsForConnector(final long
+ public List<MLink> findLinksForConnector(final long
connectorID) {
- return (List<MConnection>) doWithConnection(new DoWithConnection() {
+ return (List<MLink>) doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) throws Exception {
- return handler.findConnectionsForConnector(connectorID, conn);
+ return handler.findLinksForConnector(connectorID, conn);
}
});
}
@@ -637,12 +637,11 @@ public class JdbcRepository extends Repository {
}
@Override
- protected void deleteConnectionInputs(final long connectionID,
- RepositoryTransaction tx) {
+ protected void deleteLinkInputs(final long linkId, RepositoryTransaction tx) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) throws Exception {
- handler.deleteConnectionInputs(connectionID, conn);
+ handler.deleteLinkInputs(linkId, conn);
return null;
}
}, (JdbcRepositoryTransaction) tx);
@@ -665,12 +664,11 @@ public class JdbcRepository extends Repository {
}
- protected void updateFramework(final MFramework mFramework,
- RepositoryTransaction tx) {
+ protected void updateDriverConfig(final MDriverConfig mDriverConfig, RepositoryTransaction tx) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) throws Exception {
- handler.updateFramework(mFramework, conn);
+ handler.updateDriverConfig(mDriverConfig, conn);
return null;
}
}, (JdbcRepositoryTransaction) tx);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
index 4de3134..a743491 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
@@ -21,9 +21,9 @@ import java.sql.Connection;
import java.util.Date;
import java.util.List;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
@@ -41,11 +41,10 @@ public abstract class JdbcRepositoryHandler {
/**
* Search for connector with given name in repository.
- *
- * And return corresponding metadata structure.
+ * And return corresponding connector structure.
*
* @param shortName Connector unique name
- * @param conn JDBC connection for querying repository.
+ * @param conn JDBC link for querying repository.
* @return null if connector is not yet registered in repository or
* loaded representation.
*/
@@ -65,26 +64,26 @@ public abstract class JdbcRepositoryHandler {
* already registered or present in the repository.
*
* @param mc Connector that should be registered.
- * @param conn JDBC connection for querying repository.
+ * @param conn JDBC link for querying repository.
*/
public abstract void registerConnector(MConnector mc, Connection conn);
/**
- * Retrieve connections which use the given connector.
- * @param connectorID Connector ID whose connections should be fetched
- * @param conn JDBC connection for querying repository
- * @return List of MConnections that use <code>connectorID</code>.
+ * Retrieve links which use the given connector.
+ * @param connectorID Connector ID whose links should be fetched
+ * @param conn JDBC link for querying repository
+ * @return List of MLinks that use <code>connectorID</code>.
*/
- public abstract List<MConnection> findConnectionsForConnector(long
+ public abstract List<MLink> findLinksForConnector(long
connectorID, Connection conn);
/**
- * Retrieve jobs which use the given connection.
+ * Retrieve jobs which use the given link.
*
* @param connectorID Connector ID whose jobs should be fetched
- * @param conn JDBC connection for querying repository
- * @return List of MJobs that use <code>connectionID</code>.
+ * @param conn JDBC link for querying repository
+ * @return List of MJobs that use <code>linkID</code>.
*/
public abstract List<MJob> findJobsForConnector(long connectorID,
Connection conn);
@@ -99,47 +98,47 @@ public abstract class JdbcRepositoryHandler {
*
* @param mConnector The new data to be inserted into the repository for
* this connector.
- * @param conn JDBC connection for querying repository
+ * @param conn JDBC link for querying repository
*/
public abstract void updateConnector(MConnector mConnector, Connection conn);
/**
- * Update the framework with the new data supplied in the
- * <tt>mFramework</tt>.
+ * Update the driverConfig with the new data supplied in the
+ * <tt>mDriverConfig</tt>.
* Also Update all forms in the repository
- * with the forms specified in <tt>mFramework</tt>. <tt>mFramework </tt> must
+ * with the forms specified in <tt>mDriverConfig</tt>. <tt>mDriverConfig </tt> must
* minimally have the connectorID and all required forms (including ones
* which may not have changed). After this operation the repository is
* guaranteed to only have the new forms specified in this object.
*
- * @param mFramework The new data to be inserted into the repository for
- * the framework.
- * @param conn JDBC connection for querying repository
+ * @param mDriverConfig The new data to be inserted into the repository for
+ * the driverConfig.
+ * @param conn JDBC link for querying repository
*/
- public abstract void updateFramework(MFramework mFramework, Connection conn);
+ public abstract void updateDriverConfig(MDriverConfig mDriverConfig, Connection conn);
/**
- * Search for framework metadata in the repository.
+ * Search for driverConfigin the repository.
*
- * @param conn JDBC connection for querying repository.
- * @return null if framework metadata are not yet present in repository or
+ * @param conn JDBC link for querying repository.
+ * @return null if driverConfig are not yet present in repository or
* loaded representation.
*/
- public abstract MFramework findFramework(Connection conn);
+ public abstract MDriverConfig findDriverConfig(Connection conn);
/**
- * Register framework metadata in repository.
+ * Register driver config in repository.
*
- * Save framework metadata into repository. Metadata should not be already
+ * Save driver config into repository. Driver config should not be already
* registered or present in the repository.
*
- * @param mf Framework metadata that should be registered.
- * @param conn JDBC connection for querying repository.
+ * @param driverConfig Driver config that should be registered.
+ * @param conn JDBC link for querying repository.
*/
- public abstract void registerFramework(MFramework mf, Connection conn);
+ public abstract void registerDriverConfig(MDriverConfig driverConfig, Connection conn);
/**
* Return true if repository tables exists and are suitable for use.
@@ -169,95 +168,92 @@ public abstract class JdbcRepositoryHandler {
public abstract void shutdown();
/**
- * Specify query that Sqoop framework can use to validate connection to
+ * Specify query that Sqoop can use to validate link to
* repository. This query should return at least one row.
*
* @return Query or NULL in case that this repository do not support or do not
- * want to validate live connections.
+ * want to validate live links.
*/
public abstract String validationQuery();
/**
- * Save given connection to repository. This connection must not be already
+ * Save given link to repository. This link must not be already
* present in the repository otherwise exception will be thrown.
*
- * @param connection Connection object to serialize into repository.
- * @param conn Connection to metadata repository
+ * @param link Link object to serialize into repository.
+ * @param conn Connection to the repository
*/
- public abstract void createConnection(MConnection connection,
- Connection conn);
+ public abstract void createLink(MLink link, Connection conn);
/**
- * Update given connection representation in repository. This connection
+ * Update given link representation in repository. This link
* object must already exists in the repository otherwise exception will be
* thrown.
*
- * @param connection Connection object that should be updated in repository.
- * @param conn Connection to metadata repository
+ * @param link Link object that should be updated in repository.
+ * @param conn Connection to the repository
*/
- public abstract void updateConnection(MConnection connection,
- Connection conn);
+ public abstract void updateLink(MLink link, Connection conn);
/**
- * Check if given connection exists in metastore.
+ * Check if given link exists in repository.
*
- * @param connetionId Connection id
- * @param conn Connection to metadata repository
- * @return True if the connection exists
+ * @param linkId Link id
+ * @param conn Connection to the repository
+ * @return True if the link exists
*/
- public abstract boolean existsConnection(long connetionId, Connection conn);
+ public abstract boolean existsLink(long linkId, Connection conn);
/**
* Check if given Connection id is referenced somewhere and thus can't
* be removed.
*
- * @param connectionId Connection id
- * @param conn Connection to metadata repository
+ * @param linkId Link id
+ * @param conn Connection to the repository
* @return
*/
- public abstract boolean inUseConnection(long connectionId, Connection conn);
+ public abstract boolean inUseLink(long linkId, Connection conn);
/**
- * Enable or disable connection with given id from metadata repository
+ * Enable or disable link with given id from the repository
*
- * @param connectionId Connection object that is going to be enabled or disabled
+ * @param linkId Link object that is going to be enabled or disabled
* @param enabled Enable or disable
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
- public abstract void enableConnection(long connectionId, boolean enabled, Connection conn);
+ public abstract void enableLink(long linkId, boolean enabled, Connection conn);
/**
- * Delete connection with given id from metadata repository.
+ * Delete link with given id from the repository.
*
- * @param connectionId Connection object that should be removed from repository
- * @param conn Connection to metadata repository
+ * @param linkId Link object that should be removed from repository
+ * @param conn Connection to the repository
*/
- public abstract void deleteConnection(long connectionId, Connection conn);
+ public abstract void deleteLink(long linkId, Connection conn);
/**
- * Delete the input values for the connection with given id from the
+ * Delete the input values for the link with given id from the
* repository.
- * @param id Connection object whose inputs should be removed from repository
- * @param conn Connection to metadata repository
+ * @param id Link object whose inputs should be removed from repository
+ * @param conn Connection to the repository
*/
- public abstract void deleteConnectionInputs(long id, Connection conn);
+ public abstract void deleteLinkInputs(long id, Connection conn);
/**
- * Find connection with given id in repository.
+ * Find link with given id in repository.
*
- * @param connectionId Connection id
- * @param conn Connection to metadata repository
- * @return Deserialized form of the connection that is saved in repository
+ * @param linkId Link id
+ * @param conn Connection to the repository
+ * @return Deserialized form of the link that is saved in repository
*/
- public abstract MConnection findConnection(long connectionId,
- Connection conn);
+ public abstract MLink findLink(long linkId, Connection conn);
/**
- * Get all connection objects.
+ * Get all link objects.
*
- * @param conn Connection to metadata repository
- * @return List will all saved connection objects
+ * @param conn Connection to the repository
+ * @return List will all saved link objects
*/
- public abstract List<MConnection> findConnections(Connection conn);
+ public abstract List<MLink> findLinks(Connection conn);
/**
@@ -265,7 +261,7 @@ public abstract class JdbcRepositoryHandler {
* present in the repository otherwise exception will be thrown.
*
* @param job Job object to serialize into repository.
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
public abstract void createJob(MJob job, Connection conn);
@@ -275,15 +271,15 @@ public abstract class JdbcRepositoryHandler {
* thrown.
*
* @param job Job object that should be updated in repository.
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
public abstract void updateJob(MJob job, Connection conn);
/**
- * Check if given job exists in metastore.
+ * Check if given job exists in the repository.
*
* @param jobId Job id
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
* @return True if the job exists
*/
public abstract boolean existsJob(long jobId, Connection conn);
@@ -293,7 +289,7 @@ public abstract class JdbcRepositoryHandler {
* be removed.
*
* @param jobId Job id
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
* @return
*/
public abstract boolean inUseJob(long jobId, Connection conn);
@@ -303,22 +299,22 @@ public abstract class JdbcRepositoryHandler {
*
* @param jobId Job id
* @param enabled Enable or disable
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
public abstract void enableJob(long jobId, boolean enabled, Connection conn);
/**
* Delete the input values for the job with given id from the repository.
* @param id Job object whose inputs should be removed from repository
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
public abstract void deleteJobInputs(long id, Connection conn);
/**
- * Delete job with given id from metadata repository. This method will
+ * Delete job with given id from the repository. This method will
* delete all inputs for this job also.
*
* @param jobId Job object that should be removed from repository
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
public abstract void deleteJob(long jobId, Connection conn);
@@ -326,7 +322,7 @@ public abstract class JdbcRepositoryHandler {
* Find job with given id in repository.
*
* @param jobId Job id
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
* @return Deserialized form of the job that is present in the repository
*/
public abstract MJob findJob(long jobId, Connection conn);
@@ -334,7 +330,7 @@ public abstract class JdbcRepositoryHandler {
/**
* Get all job objects.
*
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
* @return List will all saved job objects
*/
public abstract List<MJob> findJobs(Connection conn);
@@ -343,16 +339,15 @@ public abstract class JdbcRepositoryHandler {
* Save given submission in repository.
*
* @param submission Submission object
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
- public abstract void createSubmission(MSubmission submission,
- Connection conn);
+ public abstract void createSubmission(MSubmission submission, Connection conn);
/**
* Check if submission with given id already exists in repository.
*
* @param submissionId Submission internal id
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
public abstract boolean existsSubmission(long submissionId, Connection conn);
@@ -360,39 +355,38 @@ public abstract class JdbcRepositoryHandler {
* Update given submission in repository.
*
* @param submission Submission object
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
- public abstract void updateSubmission(MSubmission submission,
- Connection conn);
+ public abstract void updateSubmission(MSubmission submission, Connection conn);
/**
* Remove submissions older then threshold from repository.
*
* @param threshold Threshold date
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
*/
public abstract void purgeSubmissions(Date threshold, Connection conn);
/**
* Return list of unfinished submissions (as far as repository is concerned).
*
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
* @return List of unfinished submissions.
*/
public abstract List<MSubmission> findSubmissionsUnfinished(Connection conn);
/**
- * Return list of all submissions from metadata repository.
+ * Return list of all submissions from the repository.
*
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
* @return List of all submissions.
*/
public abstract List<MSubmission> findSubmissions(Connection conn);
/**
- * Return list of submissions from metadata repository for given jobId.
+ * Return list of submissions from the repository for given jobId.
* @param jobId Job id
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
* @return List of submissions
*/
public abstract List<MSubmission> findSubmissionsForJob(long jobId, Connection conn);
@@ -401,7 +395,7 @@ public abstract class JdbcRepositoryHandler {
* Find last submission for given jobId.
*
* @param jobId Job id
- * @param conn Connection to metadata repository
+ * @param conn Connection to the repository
* @return Most recent submission
*/
public abstract MSubmission findSubmissionLastForJob(long jobId,
[41/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
index 19b0023..e6e4760 100644
--- a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
+++ b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
@@ -17,6 +17,23 @@
*/
package org.apache.sqoop.repository;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Collections;
@@ -29,40 +46,27 @@ import org.apache.sqoop.connector.ConnectorManager;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.connector.spi.SqoopConnector;
import org.apache.sqoop.driver.Driver;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MToConfig;
+import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.Validator;
import org.junit.Before;
import org.junit.Test;
import org.mockito.InOrder;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.inOrder;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
public class TestJdbcRepository {
private JdbcRepository repoSpy;
@@ -73,8 +77,8 @@ public class TestJdbcRepository {
private Validator validatorMock;
private RepositoryUpgrader upgraderMock;
- private Validation validRepoMock;
- private Validation invalidRepoMock;
+ private ConfigValidator validRepoMock;
+ private ConfigValidator invalidRepoMock;
@Before
public void setUp() throws Exception {
@@ -91,20 +95,20 @@ public class TestJdbcRepository {
ConnectorManager.setInstance(connectorMgrMock);
Driver.setInstance(driverMock);
- validRepoMock = mock(Validation.class);
+ validRepoMock = mock(ConfigValidator.class);
when(validRepoMock.getStatus()).thenReturn(Status.ACCEPTABLE);
- invalidRepoMock = mock(Validation.class);
+ invalidRepoMock = mock(ConfigValidator.class);
when(invalidRepoMock.getStatus()).thenReturn(Status.UNACCEPTABLE);
- doNothing().when(upgraderMock).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
- doNothing().when(upgraderMock).upgrade(any(MJobForms.class), any(MJobForms.class));
+ doNothing().when(upgraderMock).upgrade(any(MLinkConfig.class), any(MLinkConfig.class));
+ doNothing().when(upgraderMock).upgrade(any(MFromConfig.class), any(MFromConfig.class));
}
/**
* Test the procedure when the connector auto upgrade option is enabled
*/
@Test
- public void testConnectorEnableAutoUpgrade() {
+ public void testConnectorConfigEnableAutoUpgrade() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1, "1.0");
@@ -155,22 +159,22 @@ public class TestJdbcRepository {
*/
@Test
public void testDriverConfigEnableAutoUpgrade() {
- MDriverConfig newDriverConfig = driverConfig();
- MDriverConfig oldDriverConfig = anotherDriverConfig();
+ MDriver newDriverConfig = driver();
+ MDriver oldDriverConfig = anotherDriver();
- when(repoHandlerMock.findDriverConfig(any(Connection.class))).thenReturn(oldDriverConfig);
+ when(repoHandlerMock.findDriver(any(Connection.class))).thenReturn(oldDriverConfig);
// make the upgradeDriverConfig to throw an exception to prove that it has been called
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"upgradeDriverConfig() has been called.");
- doThrow(exception).when(repoHandlerMock).findLinks(any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).findJobs(any(Connection.class));
try {
- repoSpy.registerDriverConfig(newDriverConfig, true);
+ repoSpy.registerDriver(newDriverConfig, true);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findDriverConfig(any(Connection.class));
- verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findDriver(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -183,16 +187,16 @@ public class TestJdbcRepository {
*/
@Test
public void testDriverConfigDisableAutoUpgrade() {
- MDriverConfig newDriverConfig = driverConfig();
- MDriverConfig oldDriverConfig = anotherDriverConfig();
+ MDriver newDriverConfig = driver();
+ MDriver oldDriverConfig = anotherDriver();
- when(repoHandlerMock.findDriverConfig(any(Connection.class))).thenReturn(oldDriverConfig);
+ when(repoHandlerMock.findDriver(any(Connection.class))).thenReturn(oldDriverConfig);
try {
- repoSpy.registerDriverConfig(newDriverConfig, false);
+ repoSpy.registerDriver(newDriverConfig, false);
} catch (SqoopException ex) {
assertEquals(ex.getErrorCode(), RepositoryError.JDBCREPO_0026);
- verify(repoHandlerMock, times(1)).findDriverConfig(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findDriver(any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -205,18 +209,18 @@ public class TestJdbcRepository {
* jobs using the old connector are still valid for the new connector
*/
@Test
- public void testConnectorUpgradeWithValidLinksAndJobs() {
+ public void testConnectorConfigUpgradeWithValidLinksAndJobs() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
// prepare the sqoop connector
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
- when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
- when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(validatorMock.validateConfigForLink(any(MLink.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateConfigForJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(sqconnector.getConfigValidator()).thenReturn(validatorMock);
when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
- when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
+ when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationGroup.class);
+ when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(DriverConfiguration.class);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
// prepare the links and jobs
@@ -252,160 +256,82 @@ public class TestJdbcRepository {
txOrder.verify(repoTransactionMock, times(1)).commit();
txOrder.verify(repoTransactionMock, times(1)).close();
txOrder.verifyNoMoreInteractions();
- upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
- upgraderOrder.verify(upgraderMock, times(4)).upgrade(any(MJobForms.class), any(MJobForms.class));
+ upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MLinkConfig.class), any(MLinkConfig.class));
+ upgraderOrder.verify(upgraderMock, times(4)).upgrade(any(MFromConfig.class), any(MFromConfig.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validatorMock, times(2)).validateLink(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateConfigForLink(anyObject());
// @TODO(Abe): Re-enable job validation?
- validatorOrder.verify(validatorMock, times(0)).validateJob(anyObject());
+ validatorOrder.verify(validatorMock, times(0)).validateConfigForJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
}
/**
- * @TODO(Abe): To re-enable with Validation in Repository upgrade.
- * Test the connector upgrade procedure, when all the links and
- * jobs using the old connector are invalid for the new connector
- */
-// @Test
-// public void testConnectorUpgradeWithInvalidLinksAndJobs() {
-// MConnector newConnector = connector(1, "1.1");
-// MConnector oldConnector = connector(1);
-//
-// // prepare the sqoop connector
-// SqoopConnector sqconnector = mock(SqoopConnector.class);
-// when(validator.validateLink(any(MLink.class))).thenReturn(invalid);
-// when(validator.validateJob(any(MJob.class))).thenReturn(invalid);
-// when(sqconnector.getValidator()).thenReturn(validator);
-// when(sqconnector.getDriverConfigRepositoryUpgrader()).thenReturn(upgrader);
-// when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
-// when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
-// when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
-//
-// // prepare the links and jobs
-// List<MLink> linkList = links(link(1,1), link(2,1));
-// List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
-//
-// doReturn(linkList).when(repo).findLinksForConnector(anyLong());
-// doReturn(jobList).when(repo).findJobsForConnector(anyLong());
-// doNothing().when(repo).updateLink(any(MLink.class), any(RepositoryTransaction.class));
-// doNothing().when(repo).updateJob(any(MJob.class), any(RepositoryTransaction.class));
-// doNothing().when(repo).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
-//
-// try {
-// repo.upgradeConnector(oldConnector, newConnector);
-// } catch (SqoopException ex) {
-// assertEquals(ex.getErrorCode(), RepositoryError.JDBCREPO_0027);
-//
-// InOrder repoOrder = inOrder(repo);
-// InOrder txOrder = inOrder(tx);
-// InOrder upgraderOrder = inOrder(upgrader);
-// InOrder validatorOrder = inOrder(validator);
-//
-// repoOrder.verify(repo, times(1)).findLinksForConnector(anyLong());
-// repoOrder.verify(repo, times(1)).findJobsForConnector(anyLong());
-// repoOrder.verify(repo, times(1)).getTransaction();
-// repoOrder.verify(repo, times(1)).deleteJobInputs(1, tx);
-// repoOrder.verify(repo, times(1)).deleteJobInputs(2, tx);
-// repoOrder.verify(repo, times(1)).deleteLinkInputs(1, tx);
-// repoOrder.verify(repo, times(1)).deleteLinkInputs(2, tx);
-// repoOrder.verify(repo, times(1)).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
-// repoOrder.verifyNoMoreInteractions();
-// txOrder.verify(tx, times(1)).begin();
-// txOrder.verify(tx, times(1)).rollback();
-// txOrder.verify(tx, times(1)).close();
-// txOrder.verifyNoMoreInteractions();
-// upgraderOrder.verify(upgrader, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
-// upgraderOrder.verify(upgrader, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
-// upgraderOrder.verifyNoMoreInteractions();
-// validatorOrder.verify(validator, times(2)).validateLink(anyObject());
-// validatorOrder.verify(validator, times(2)).validateJob(anyObject());
-// validatorOrder.verifyNoMoreInteractions();
-// return ;
-// }
-//
-// fail("Should throw out an exception with code: " + RepositoryError.JDBCREPO_0027);
-// }
-
- /**
- * Test the driverConfig upgrade procedure, when all the links and
- * jobs using the old connector are still valid for the new connector
+ * Test the driverConfig upgrade procedure, when all jobs
+ * using the old connector are still valid for the new connector
*/
@Test
- public void testDriverConfigUpgradeWithValidLinksAndJobs() {
- MDriverConfig newDriverConfig = driverConfig();
+ public void testDriverConfigUpgradeWithValidJobs() {
+ MDriver newDriverConfig = driver();
- when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
- when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateConfigForLink(any(MLink.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateConfigForJob(any(MJob.class))).thenReturn(validRepoMock);
when(driverMock.getValidator()).thenReturn(validatorMock);
when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- when(driverMock.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(driverMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(driverMock.getDriverConfigurationGroupClass()).thenReturn(EmptyConfigurationGroup.class);
- List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(linkList).when(repoSpy).findLinks();
doReturn(jobList).when(repoSpy).findJobs();
doNothing().when(repoSpy).updateLink(any(MLink.class), any(RepositoryTransaction.class));
doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
- doNothing().when(repoSpy).updateDriverConfig(any(MDriverConfig.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateDriver(any(MDriver.class), any(RepositoryTransaction.class));
- repoSpy.upgradeDriverConfig(newDriverConfig);
+ repoSpy.upgradeDriver(newDriverConfig);
InOrder repoOrder = inOrder(repoSpy);
InOrder txOrder = inOrder(repoTransactionMock);
InOrder upgraderOrder = inOrder(upgraderMock);
InOrder validatorOrder = inOrder(validatorMock);
- repoOrder.verify(repoSpy, times(1)).findLinks();
repoOrder.verify(repoSpy, times(1)).findJobs();
repoOrder.verify(repoSpy, times(1)).getTransaction();
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(1, repoTransactionMock);
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(1, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).updateDriverConfig(any(MDriverConfig.class), any(RepositoryTransaction.class));
- repoOrder.verify(repoSpy, times(2)).updateLink(any(MLink.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(1)).updateDriver(any(MDriver.class), any(RepositoryTransaction.class));
repoOrder.verify(repoSpy, times(2)).updateJob(any(MJob.class), any(RepositoryTransaction.class));
repoOrder.verifyNoMoreInteractions();
txOrder.verify(repoTransactionMock, times(1)).begin();
txOrder.verify(repoTransactionMock, times(1)).commit();
txOrder.verify(repoTransactionMock, times(1)).close();
txOrder.verifyNoMoreInteractions();
- upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
- upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
+ upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConfigList.class), any(MConfigList.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validatorMock, times(2)).validateLink(anyObject());
- validatorOrder.verify(validatorMock, times(2)).validateJob(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateConfigForJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
}
/**
- * Test the driverConfig upgrade procedure, when all the links and
- * jobs using the old connector are invalid for the new connector
+ * Test the driverConfig upgrade procedure, when all the jobs
+ * using the old connector are invalid for the new connector
*/
@Test
- public void testDriverConfigUpgradeWithInvalidLinksAndJobs() {
- MDriverConfig newDriverConfig = driverConfig();
+ public void testDriverConfigUpgradeWithInvalidJobs() {
+ MDriver newDriverConfig = driver();
- when(validatorMock.validateLink(any(MLink.class))).thenReturn(invalidRepoMock);
- when(validatorMock.validateJob(any(MJob.class))).thenReturn(invalidRepoMock);
+ when(validatorMock.validateConfigForLink(any(MLink.class))).thenReturn(invalidRepoMock);
+ when(validatorMock.validateConfigForJob(any(MJob.class))).thenReturn(invalidRepoMock);
when(driverMock.getValidator()).thenReturn(validatorMock);
when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- when(driverMock.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(driverMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(driverMock.getDriverConfigurationGroupClass()).thenReturn(EmptyConfigurationGroup.class);
- List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(linkList).when(repoSpy).findLinks();
doReturn(jobList).when(repoSpy).findJobs();
- doNothing().when(repoSpy).updateLink(any(MLink.class), any(RepositoryTransaction.class));
doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
- doNothing().when(repoSpy).updateDriverConfig(any(MDriverConfig.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateDriver(any(MDriver.class), any(RepositoryTransaction.class));
try {
- repoSpy.upgradeDriverConfig(newDriverConfig);
+ repoSpy.upgradeDriver(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getErrorCode(), RepositoryError.JDBCREPO_0027);
@@ -414,24 +340,20 @@ public class TestJdbcRepository {
InOrder upgraderOrder = inOrder(upgraderMock);
InOrder validatorOrder = inOrder(validatorMock);
- repoOrder.verify(repoSpy, times(1)).findLinks();
repoOrder.verify(repoSpy, times(1)).findJobs();
repoOrder.verify(repoSpy, times(1)).getTransaction();
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(1, repoTransactionMock);
repoOrder.verify(repoSpy, times(1)).deleteJobInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(1, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).deleteLinkInputs(2, repoTransactionMock);
- repoOrder.verify(repoSpy, times(1)).updateDriverConfig(any(MDriverConfig.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(1)).updateDriver(any(MDriver.class), any(RepositoryTransaction.class));
repoOrder.verifyNoMoreInteractions();
txOrder.verify(repoTransactionMock, times(1)).begin();
txOrder.verify(repoTransactionMock, times(1)).rollback();
txOrder.verify(repoTransactionMock, times(1)).close();
txOrder.verifyNoMoreInteractions();
- upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
- upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
+ upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConfigList.class), any(MConfigList.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validatorMock, times(2)).validateLink(anyObject());
- validatorOrder.verify(validatorMock, times(2)).validateJob(anyObject());
+ // driver configs are per job.
+ validatorOrder.verify(validatorMock, times(2)).validateConfigForJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
return ;
}
@@ -444,12 +366,12 @@ public class TestJdbcRepository {
* find links for a given connector
*/
@Test
- public void testConnectorUpgradeHandlerFindLinksForConnectorError() {
+ public void testConnectorConfigUpgradeHandlerWithFindLinksForConnectorError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getConfigValidator()).thenReturn(validatorMock);
when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
@@ -474,12 +396,12 @@ public class TestJdbcRepository {
* find jobs for a given connector
*/
@Test
- public void testConnectorUpgradeHandlerFindJobsForConnectorError() {
+ public void testConnectorConfigUpgradeHandlerWithFindJobsForConnectorError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getConfigValidator()).thenReturn(validatorMock);
when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
@@ -508,12 +430,12 @@ public class TestJdbcRepository {
* delete job inputs for a given connector
*/
@Test
- public void testConnectorUpgradeHandlerDeleteJobInputsError() {
+ public void testConnectorConfigUpgradeHandlerWithDeleteJobInputsError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getConfigValidator()).thenReturn(validatorMock);
when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
@@ -545,12 +467,12 @@ public class TestJdbcRepository {
* delete link inputs for a given connector
*/
@Test
- public void testConnectorUpgradeHandlerDeleteLinkInputsError() {
+ public void testConnectorConfigUpgradeHandlerWithDeleteLinkInputsError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getConfigValidator()).thenReturn(validatorMock);
when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
@@ -584,12 +506,12 @@ public class TestJdbcRepository {
* update the connector entity
*/
@Test
- public void testConnectorUpgradeHandlerUpdateConnectorError() {
+ public void testConnectorConfigUpgradeHandlerWithUpdateConnectorError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getConfigValidator()).thenReturn(validatorMock);
when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
@@ -625,17 +547,17 @@ public class TestJdbcRepository {
* update the link entity
*/
@Test
- public void testConnectorUpgradeHandlerUpdateLinkError() {
+ public void testConnectorConfigUpgradeHandlerWithUpdateLinkError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
- when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
- when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(validatorMock.validateConfigForLink(any(MLink.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateConfigForJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(sqconnector.getConfigValidator()).thenReturn(validatorMock);
when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
- when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
+ when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationGroup.class);
+ when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(DriverConfiguration.class);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1,1), link(2,1));
@@ -674,17 +596,17 @@ public class TestJdbcRepository {
* update the job entity
*/
@Test
- public void testConnectorUpgradeHandlerUpdateJobError() {
+ public void testConnectorConfigUpgradeHandlerWithUpdateJobError() {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
- when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
- when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(validatorMock.validateConfigForLink(any(MLink.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateConfigForJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(sqconnector.getConfigValidator()).thenReturn(validatorMock);
when(sqconnector.getRepositoryUpgrader()).thenReturn(upgraderMock);
- when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
+ when(sqconnector.getLinkConfigurationClass()).thenReturn(EmptyConfigurationGroup.class);
+ when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(DriverConfiguration.class);
when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1,1), link(2,1));
@@ -724,54 +646,23 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * find links for driverConfig
- */
- @Test
- public void testDriverConfigUpgradeHandlerFindLinksError() {
- MDriverConfig newDriverConfig = driverConfig();
-
- when(driverMock.getValidator()).thenReturn(validatorMock);
- when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
-
- SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "find links error.");
- doThrow(exception).when(repoHandlerMock).findLinks(any(Connection.class));
-
- try {
- repoSpy.upgradeDriverConfig(newDriverConfig);
- } catch (SqoopException ex) {
- assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
- verifyNoMoreInteractions(repoHandlerMock);
- return ;
- }
-
- fail("Should throw out an exception with message: " + exception.getMessage());
- }
-
- /**
- * Test the exception handling procedure when the database handler fails to
* find jobs for driverConfig
*/
@Test
- public void testDriverConfigUpgradeHandlerFindJobsError() {
- MDriverConfig newDriverConfig = driverConfig();
+ public void testDriverConfigUpgradeHandlerWithFindJobsError() {
+ MDriver newDriverConfig = driver();
when(driverMock.getValidator()).thenReturn(validatorMock);
when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- List<MLink> linkList = links(link(1,1), link(2,1));
- doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
-
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"find jobs error.");
doThrow(exception).when(repoHandlerMock).findJobs(any(Connection.class));
try {
- repoSpy.upgradeDriverConfig(newDriverConfig);
+ repoSpy.upgradeDriver(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
@@ -785,15 +676,13 @@ public class TestJdbcRepository {
* delete job inputs for driverConfig upgrade
*/
@Test
- public void testDriverConfigUpgradeHandlerDeleteJobInputsError() {
- MDriverConfig newDriverConfig = driverConfig();
+ public void testDriverConfigUpgradeHandlerWithDeleteJobInputsError() {
+ MDriver newDriverConfig = driver();
when(driverMock.getValidator()).thenReturn(validatorMock);
when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
@@ -801,10 +690,9 @@ public class TestJdbcRepository {
doThrow(exception).when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
try {
- repoSpy.upgradeDriverConfig(newDriverConfig);
+ repoSpy.upgradeDriver(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verify(repoHandlerMock, times(1)).deleteJobInputs(anyLong(), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
@@ -816,71 +704,31 @@ public class TestJdbcRepository {
/**
* Test the exception handling procedure when the database handler fails to
- * delete link inputs for driverConfig upgrade
- */
- @Test
- public void testDriverConfigUpgradeHandlerDeleteConnectionInputsError() {
- MDriverConfig newDriverConfig = driverConfig();
-
- when(driverMock.getValidator()).thenReturn(validatorMock);
- when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
-
- List<MLink> linkList = links(link(1,1), link(2,1));
- List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
- doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
- doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
-
- SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "delete link inputs error.");
- doThrow(exception).when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
-
- try {
- repoSpy.upgradeDriverConfig(newDriverConfig);
- } catch (SqoopException ex) {
- assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
- verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).deleteLinkInputs(anyLong(), any(Connection.class));
- verifyNoMoreInteractions(repoHandlerMock);
- return ;
- }
-
- fail("Should throw out an exception with message: " + exception.getMessage());
- }
-
- /**
- * Test the exception handling procedure when the database handler fails to
* update the driverConfig entity
*/
@Test
- public void testDriverConfigUpgradeHandlerUpdateDriverConfigError() {
- MDriverConfig newDriverConfig = driverConfig();
+ public void testDriverConfigUpgradeHandlerWithUpdateDriverConfigError() {
+ MDriver newDriverConfig = driver();
when(driverMock.getValidator()).thenReturn(validatorMock);
when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update driverConfig entity error.");
- doThrow(exception).when(repoHandlerMock).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).updateDriver(any(MDriver.class), any(Connection.class));
try {
- repoSpy.upgradeDriverConfig(newDriverConfig);
+ repoSpy.upgradeDriver(newDriverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateDriver(any(MDriver.class), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -888,93 +736,38 @@ public class TestJdbcRepository {
fail("Should throw out an exception with message: " + exception.getMessage());
}
- /**
- * Test the exception handling procedure when the database handler fails to
- * update the link entity
- */
- @Test
- public void testDriverConfigUpgradeHandlerUpdateConnectionError() {
- MDriverConfig newDriverConfig = driverConfig();
-
- when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
- when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
- when(driverMock.getValidator()).thenReturn(validatorMock);
- when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- when(driverMock.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(driverMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
-
- List<MLink> linkList = links(link(1,1), link(2,1));
- List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
- doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
- doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
- doReturn(true).when(repoHandlerMock).existsLink(anyLong(), any(Connection.class));
-
- SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
- "update link error.");
- doThrow(exception).when(repoHandlerMock).updateLink(any(MLink.class), any(Connection.class));
-
- try {
- repoSpy.upgradeDriverConfig(newDriverConfig);
- } catch (SqoopException ex) {
- assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
- verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
- verify(repoHandlerMock, times(1)).existsLink(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateLink(any(MLink.class), any(Connection.class));
- verifyNoMoreInteractions(repoHandlerMock);
- return ;
- }
-
- fail("Should throw out an exception with message: " + exception.getMessage());
- }
/**
* Test the exception handling procedure when the database handler fails to
* update the job entity
*/
@Test
- public void testDriverConfigUpgradeHandlerUpdateJobError() {
- MDriverConfig driverConfig = driverConfig();
+ public void testDriverConfigUpgradeHandlerWithUpdateJobError() {
+ MDriver driverConfig = driver();
- when(validatorMock.validateLink(any(MLink.class))).thenReturn(validRepoMock);
- when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateConfigForLink(any(MLink.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateConfigForJob(any(MJob.class))).thenReturn(validRepoMock);
when(driverMock.getValidator()).thenReturn(validatorMock);
when(driverMock.getDriverConfigRepositoryUpgrader()).thenReturn(upgraderMock);
- when(driverMock.getLinkConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(driverMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(driverMock.getDriverConfigurationGroupClass()).thenReturn(EmptyConfigurationGroup.class);
- List<MLink> linkList = links(link(1,1), link(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(linkList).when(repoHandlerMock).findLinks(any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).deleteLinkInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
- doReturn(true).when(repoHandlerMock).existsLink(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateDriver(any(MDriver.class), any(Connection.class));
doReturn(true).when(repoHandlerMock).existsJob(anyLong(), any(Connection.class));
- doNothing().when(repoHandlerMock).updateLink(any(MLink.class), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update job error.");
doThrow(exception).when(repoHandlerMock).updateJob(any(MJob.class), any(Connection.class));
try {
- repoSpy.upgradeDriverConfig(driverConfig);
+ repoSpy.upgradeDriver(driverConfig);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandlerMock, times(1)).findLinks(any(Connection.class));
verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).deleteLinkInputs(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(1)).updateDriverConfig(any(MDriverConfig.class), any(Connection.class));
- verify(repoHandlerMock, times(2)).existsLink(anyLong(), any(Connection.class));
- verify(repoHandlerMock, times(2)).updateLink(any(MLink.class), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateDriver(any(MDriver.class), any(Connection.class));
verify(repoHandlerMock, times(1)).existsJob(anyLong(), any(Connection.class));
verify(repoHandlerMock, times(1)).updateJob(any(MJob.class), any(Connection.class));
verifyNoMoreInteractions(repoHandlerMock);
@@ -986,9 +779,9 @@ public class TestJdbcRepository {
private MConnector connector(long connectorId, String version) {
MConnector connector = new MConnector("A" + connectorId, "A" + connectorId, version + connectorId,
- new MConnectionForms(new LinkedList<MForm>()),
- new MJobForms(FormUtils.toForms(JobConfiguration.class)),
- new MJobForms(FormUtils.toForms(JobConfiguration.class)));
+ new MLinkConfig(new LinkedList<MConfig>()),
+ new MFromConfig(ConfigUtils.toConfigs(FromJobConfigurationGroup.class)),
+ new MToConfig(ConfigUtils.toConfigs(ToJobConfigurationGroup.class)));
connector.setPersistenceId(connectorId);
return connector;
}
@@ -997,34 +790,30 @@ public class TestJdbcRepository {
return connector(connectoId, "1.0");
}
- private MDriverConfig driverConfig() {
- MDriverConfig driverConfig = new MDriverConfig(
- new MConnectionForms(new LinkedList<MForm>()),
- new MJobForms(FormUtils.toForms(JobConfiguration.class)),
- Driver.CURRENT_DRIVER_VERSION);
- driverConfig.setPersistenceId(1);
- return driverConfig;
+ private MDriver driver() {
+ MDriver driver = new MDriver(new MDriverConfig(new LinkedList<MConfig>()),
+ DriverBean.CURRENT_DRIVER_VERSION);
+ driver.setPersistenceId(1);
+ return driver;
}
- private MDriverConfig anotherDriverConfig() {
- MDriverConfig driverConfig = new MDriverConfig(null, null,
- Driver.CURRENT_DRIVER_VERSION);
- driverConfig.setPersistenceId(1);
- return driverConfig;
+ private MDriver anotherDriver() {
+ MDriver driver = new MDriver(null, DriverBean.CURRENT_DRIVER_VERSION);
+ driver.setPersistenceId(1);
+ return driver;
}
private MLink link(long linkId, long connectorId) {
- MLink link = new MLink(connectorId, new MConnectionForms(new LinkedList<MForm>()),
- new MConnectionForms(new LinkedList<MForm>()));
+ MLink link = new MLink(connectorId, new MLinkConfig(new LinkedList<MConfig>()));
link.setPersistenceId(linkId);
return link;
}
private MJob job(long id, long fromConnectorId, long toConnectorId, long fromLinkId, long toLinkId) {
MJob job = new MJob(fromConnectorId, toConnectorId, fromLinkId, toLinkId,
- new MJobForms(new LinkedList<MForm>()),
- new MJobForms(new LinkedList<MForm>()),
- new MJobForms(new LinkedList<MForm>()));
+ new MFromConfig(new LinkedList<MConfig>()),
+ new MToConfig(new LinkedList<MConfig>()),
+ new MDriverConfig(new LinkedList<MConfig>()));
job.setPersistenceId(id);
return job;
}
@@ -1042,6 +831,12 @@ public class TestJdbcRepository {
}
@ConfigurationClass
- public static class EmptyConfigurationClass {
+ public static class EmptyConfigurationGroup {
+ }
+ @ConfigurationClass
+ public static class FromJobConfigurationGroup {
+ }
+ @ConfigurationClass
+ public static class ToJobConfigurationGroup {
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/Data.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/Data.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/Data.java
index 83c670c..5423b7b 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/Data.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/io/Data.java
@@ -526,4 +526,4 @@ public class Data implements WritableComparable<Data> {
String replacement = String.valueOf(stringDelimiter);
return string.replaceAll(regex, replacement);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
index b533837..0fa07f7 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
@@ -25,7 +25,7 @@ import org.apache.log4j.PropertyConfigurator;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.json.util.SchemaSerialization;
-import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.utils.ClassUtils;
import org.json.simple.JSONObject;
@@ -40,47 +40,35 @@ import java.util.Properties;
*/
public final class ConfigurationUtils {
- private static final String JOB_CONFIG_CLASS_FROM_CONNECTOR_CONNECTION = JobConstants.PREFIX_JOB_CONFIG + "config.class.connector.from.connection";
+ private static final String MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK = JobConstants.PREFIX_JOB_CONFIG + "config.class.connector.from.link";
- private static final String JOB_CONFIG_CLASS_TO_CONNECTOR_CONNECTION = JobConstants.PREFIX_JOB_CONFIG + "config.class.connector.to.connection";
+ private static final String MR_JOB_CONFIG_CLASS_TO_CONNECTOR_LINK = JobConstants.PREFIX_JOB_CONFIG + "config.class.connector.to.link";
- private static final String JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB = JobConstants.PREFIX_JOB_CONFIG + "config.class.connector.from.job";
+ private static final String MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB = JobConstants.PREFIX_JOB_CONFIG + "config.class.connector.from.job";
- private static final String JOB_CONFIG_CLASS_TO_CONNECTOR_JOB = JobConstants.PREFIX_JOB_CONFIG + "config.class.connector.to.job";
+ private static final String MR_JOB_CONFIG_CLASS_TO_CONNECTOR_JOB = JobConstants.PREFIX_JOB_CONFIG + "config.class.connector.to.job";
- private static final String JOB_CONFIG_CLASS_FROM_FRAMEWORK_CONNECTION = JobConstants.PREFIX_JOB_CONFIG + "config.class.framework.from.connection";
+ private static final String MR_JOB_CONFIG_DRIVER_CONFIG_CLASS = JobConstants.PREFIX_JOB_CONFIG + "config.class.driver";
- private static final String JOB_CONFIG_CLASS_TO_FRAMEWORK_CONNECTION = JobConstants.PREFIX_JOB_CONFIG + "config.class.framework.to.connection";
+ private static final String MR_JOB_CONFIG_FROM_CONNECTOR_LINK = JobConstants.PREFIX_JOB_CONFIG + "config.connector.from.link";
- private static final String JOB_CONFIG_CLASS_FRAMEWORK_JOB = JobConstants.PREFIX_JOB_CONFIG + "config.class.framework.job";
+ private static final Text MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY = new Text(MR_JOB_CONFIG_FROM_CONNECTOR_LINK);
- private static final String JOB_CONFIG_FROM_CONNECTOR_CONNECTION = JobConstants.PREFIX_JOB_CONFIG + "config.connector.from.connection";
+ private static final String MR_JOB_CONFIG_TO_CONNECTOR_LINK = JobConstants.PREFIX_JOB_CONFIG + "config.connector.to.link";
- private static final Text JOB_CONFIG_FROM_CONNECTOR_CONNECTION_KEY = new Text(JOB_CONFIG_FROM_CONNECTOR_CONNECTION);
+ private static final Text MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY = new Text(MR_JOB_CONFIG_TO_CONNECTOR_LINK);
- private static final String JOB_CONFIG_TO_CONNECTOR_CONNECTION = JobConstants.PREFIX_JOB_CONFIG + "config.connector.to.connection";
+ private static final String MR_JOB_CONFIG_FROM_JOB_CONFIG = JobConstants.PREFIX_JOB_CONFIG + "config.connector.from.job";
- private static final Text JOB_CONFIG_TO_CONNECTOR_CONNECTION_KEY = new Text(JOB_CONFIG_TO_CONNECTOR_CONNECTION);
+ private static final Text MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY = new Text(MR_JOB_CONFIG_FROM_JOB_CONFIG);
- private static final String JOB_CONFIG_FROM_CONNECTOR_JOB = JobConstants.PREFIX_JOB_CONFIG + "config.connector.from.job";
+ private static final String MR_JOB_CONFIG_TO_JOB_CONFIG = JobConstants.PREFIX_JOB_CONFIG + "config.connector.to.job";
- private static final Text JOB_CONFIG_FROM_CONNECTOR_JOB_KEY = new Text(JOB_CONFIG_FROM_CONNECTOR_JOB);
+ private static final Text MR_JOB_CONFIG_TO_JOB_CONFIG_KEY = new Text(MR_JOB_CONFIG_TO_JOB_CONFIG);
- private static final String JOB_CONFIG_TO_CONNECTOR_JOB = JobConstants.PREFIX_JOB_CONFIG + "config.connector.to.job";
+ private static final String MR_JOB_CONFIG_DRIVER_CONFIG = JobConstants.PREFIX_JOB_CONFIG + "config.driver";
- private static final Text JOB_CONFIG_TO_CONNECTOR_JOB_KEY = new Text(JOB_CONFIG_TO_CONNECTOR_JOB);
-
- private static final String JOB_CONFIG_FROM_FRAMEWORK_CONNECTION = JobConstants.PREFIX_JOB_CONFIG + "config.framework.from.connection";
-
- private static final Text JOB_CONFIG_FROM_FRAMEWORK_CONNECTION_KEY = new Text(JOB_CONFIG_FROM_FRAMEWORK_CONNECTION);
-
- private static final String JOB_CONFIG_TO_FRAMEWORK_CONNECTION = JobConstants.PREFIX_JOB_CONFIG + "config.framework.from.connection";
-
- private static final Text JOB_CONFIG_TO_FRAMEWORK_CONNECTION_KEY = new Text(JOB_CONFIG_TO_FRAMEWORK_CONNECTION);
-
- private static final String JOB_CONFIG_FRAMEWORK_JOB = JobConstants.PREFIX_JOB_CONFIG + "config.framework.job";
-
- private static final Text JOB_CONFIG_FRAMEWORK_JOB_KEY = new Text(JOB_CONFIG_FRAMEWORK_JOB);
+ private static final Text MR_JOB_CONFIG_DRIVER_CONFIG_KEY = new Text(MR_JOB_CONFIG_DRIVER_CONFIG);
private static final String SCHEMA_FROM = JobConstants.PREFIX_JOB_CONFIG + "schema.connector.from";
@@ -92,21 +80,21 @@ public final class ConfigurationUtils {
/**
- * Persist Connector configuration object for connection.
+ * Persist Connector configuration object for link.
*
* @param job MapReduce job object
* @param obj Configuration object
*/
- public static void setConnectorConnectionConfig(Direction type, Job job, Object obj) {
+ public static void setConnectorLinkConfig(Direction type, Job job, Object obj) {
switch (type) {
case FROM:
- job.getConfiguration().set(JOB_CONFIG_CLASS_FROM_CONNECTOR_CONNECTION, obj.getClass().getName());
- job.getCredentials().addSecretKey(JOB_CONFIG_FROM_CONNECTOR_CONNECTION_KEY, FormUtils.toJson(obj).getBytes());
+ job.getConfiguration().set(MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK, obj.getClass().getName());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes());
break;
case TO:
- job.getConfiguration().set(JOB_CONFIG_CLASS_TO_CONNECTOR_CONNECTION, obj.getClass().getName());
- job.getCredentials().addSecretKey(JOB_CONFIG_TO_CONNECTOR_CONNECTION_KEY, FormUtils.toJson(obj).getBytes());
+ job.getConfiguration().set(MR_JOB_CONFIG_CLASS_TO_CONNECTOR_LINK, obj.getClass().getName());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes());
break;
}
}
@@ -120,46 +108,27 @@ public final class ConfigurationUtils {
public static void setConnectorJobConfig(Direction type, Job job, Object obj) {
switch (type) {
case FROM:
- job.getConfiguration().set(JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, obj.getClass().getName());
- job.getCredentials().addSecretKey(JOB_CONFIG_FROM_CONNECTOR_JOB_KEY, FormUtils.toJson(obj).getBytes());
+ job.getConfiguration().set(MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, obj.getClass().getName());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
break;
case TO:
- job.getConfiguration().set(JOB_CONFIG_CLASS_TO_CONNECTOR_JOB, obj.getClass().getName());
- job.getCredentials().addSecretKey(JOB_CONFIG_TO_CONNECTOR_JOB_KEY, FormUtils.toJson(obj).getBytes());
+ job.getConfiguration().set(MR_JOB_CONFIG_CLASS_TO_CONNECTOR_JOB, obj.getClass().getName());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
break;
}
}
- /**
- * Persist Framework configuration object for connection.
- *
- * @param job MapReduce job object
- * @param obj Configuration object
- */
- public static void setFrameworkConnectionConfig(Direction type, Job job, Object obj) {
- switch (type) {
- case FROM:
- job.getConfiguration().set(JOB_CONFIG_CLASS_FROM_FRAMEWORK_CONNECTION, obj.getClass().getName());
- job.getCredentials().addSecretKey(JOB_CONFIG_FROM_FRAMEWORK_CONNECTION_KEY, FormUtils.toJson(obj).getBytes());
- break;
-
- case TO:
- job.getConfiguration().set(JOB_CONFIG_CLASS_TO_FRAMEWORK_CONNECTION, obj.getClass().getName());
- job.getCredentials().addSecretKey(JOB_CONFIG_TO_FRAMEWORK_CONNECTION_KEY, FormUtils.toJson(obj).getBytes());
- break;
- }
- }
/**
- * Persist Framework configuration object for job.
+ * Persist driver configuration object for job.
*
* @param job MapReduce job object
* @param obj Configuration object
*/
- public static void setFrameworkJobConfig(Job job, Object obj) {
- job.getConfiguration().set(JOB_CONFIG_CLASS_FRAMEWORK_JOB, obj.getClass().getName());
- job.getCredentials().addSecretKey(JOB_CONFIG_FRAMEWORK_JOB_KEY, FormUtils.toJson(obj).getBytes());
+ public static void setDriverConfig(Job job, Object obj) {
+ job.getConfiguration().set(MR_JOB_CONFIG_DRIVER_CONFIG_CLASS, obj.getClass().getName());
+ job.getCredentials().addSecretKey(MR_JOB_CONFIG_DRIVER_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
}
/**
@@ -185,17 +154,16 @@ public final class ConfigurationUtils {
/**
* Retrieve Connector configuration object for connection.
- *
* @param configuration MapReduce configuration object
* @return Configuration object
*/
public static Object getConnectorConnectionConfig(Direction type, Configuration configuration) {
switch (type) {
case FROM:
- return loadConfiguration((JobConf) configuration, JOB_CONFIG_CLASS_FROM_CONNECTOR_CONNECTION, JOB_CONFIG_FROM_CONNECTOR_CONNECTION_KEY);
+ return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK, MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY);
case TO:
- return loadConfiguration((JobConf) configuration, JOB_CONFIG_CLASS_TO_CONNECTOR_CONNECTION, JOB_CONFIG_TO_CONNECTOR_CONNECTION_KEY);
+ return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_TO_CONNECTOR_LINK, MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY);
}
return null;
@@ -210,28 +178,10 @@ public final class ConfigurationUtils {
public static Object getConnectorJobConfig(Direction type, Configuration configuration) {
switch (type) {
case FROM:
- return loadConfiguration((JobConf) configuration, JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, JOB_CONFIG_FROM_CONNECTOR_JOB_KEY);
-
- case TO:
- return loadConfiguration((JobConf) configuration, JOB_CONFIG_CLASS_TO_CONNECTOR_JOB, JOB_CONFIG_TO_CONNECTOR_JOB_KEY);
- }
-
- return null;
- }
-
- /**
- * Retrieve Framework configuration object for connection.
- *
- * @param configuration MapReduce configuration object
- * @return Configuration object
- */
- public static Object getFrameworkConnectionConfig(Direction type, Configuration configuration) {
- switch (type) {
- case FROM:
- return loadConfiguration((JobConf) configuration, JOB_CONFIG_CLASS_FROM_FRAMEWORK_CONNECTION, JOB_CONFIG_FROM_FRAMEWORK_CONNECTION_KEY);
+ return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY);
case TO:
- return loadConfiguration((JobConf) configuration, JOB_CONFIG_CLASS_TO_FRAMEWORK_CONNECTION, JOB_CONFIG_TO_FRAMEWORK_CONNECTION_KEY);
+ return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_TO_CONNECTOR_JOB, MR_JOB_CONFIG_TO_JOB_CONFIG_KEY);
}
return null;
@@ -243,8 +193,8 @@ public final class ConfigurationUtils {
* @param configuration MapReduce configuration object
* @return Configuration object
*/
- public static Object getFrameworkJobConfig(Configuration configuration) {
- return loadConfiguration((JobConf) configuration, JOB_CONFIG_CLASS_FRAMEWORK_JOB, JOB_CONFIG_FRAMEWORK_JOB_KEY);
+ public static Object getDriverConfig(Configuration configuration) {
+ return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_DRIVER_CONFIG_CLASS, MR_JOB_CONFIG_DRIVER_CONFIG_KEY);
}
@@ -303,7 +253,7 @@ public final class ConfigurationUtils {
String json = new String(configuration.getCredentials().getSecretKey(valueProperty));
// Fill it with JSON data
- FormUtils.fillValues(json, object);
+ ConfigUtils.fillValues(json, object);
// And give it back
return object;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
index aecde40..8d2a1da 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java
@@ -37,7 +37,7 @@ public class SqoopDestroyerExecutor {
/**
* Execute destroyer.
*
- * @param success True if the job execution was successfull
+ * @param success True if the job execution was successful
* @param configuration Configuration object to get destroyer class with context
* and configuration objects.
* @param direction The direction of the Destroyer to execute.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
index 1952cbb..0d14fc7 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
@@ -19,8 +19,6 @@ package org.apache.sqoop.job;
import java.io.IOException;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -33,6 +31,7 @@ import org.apache.sqoop.job.mr.SqoopInputFormat;
import org.apache.sqoop.job.mr.SqoopMapper;
import org.apache.sqoop.job.mr.SqoopNullOutputFormat;
import org.apache.sqoop.job.mr.SqoopSplit;
+import org.junit.Assert;
public class JobUtils {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
index 032cc11..e3b68e2 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
@@ -253,7 +253,6 @@ public class TestMapReduce {
public static class DummyLoader extends Loader {
private int index = START_PARTITION*NUMBER_OF_ROWS_PER_PARTITION;
private Data expected = new Data();
- private CSVIntermediateDataFormat actual = new CSVIntermediateDataFormat();
@Override
public void load(LoaderContext context, Object oc, Object oj) throws Exception{
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestConfigurationUtils.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestConfigurationUtils.java
index 1447e00..501e32c 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestConfigurationUtils.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/mr/TestConfigurationUtils.java
@@ -17,130 +17,118 @@
*/
package org.apache.sqoop.job.mr;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.Config;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
-import org.apache.sqoop.model.MJob;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Text;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
/**
- * Current tests are using mockito to propagate credentials from Job object
- * to JobConf object. This implementation was chosen because it's not clear
+ * Current tests are using mockito to propagate credentials from hadoop Job object
+ * to hadoop JobConf object. This implementation was chosen because it's not clear
* how MapReduce is converting one object to another.
*/
public class TestConfigurationUtils {
Job job;
- JobConf jobConf;
+ JobConf jobConfSpy;
@Before
public void setUp() throws Exception {
- setUpJob();
- setUpJobConf();
+ setUpHadoopJob();
+ setUpHadoopJobConf();
}
- public void setUpJob() throws Exception {
+ public void setUpHadoopJob() throws Exception {
job = new Job();
}
- public void setUpJobConf() throws Exception {
- jobConf = spy(new JobConf(job.getConfiguration()));
- when(jobConf.getCredentials()).thenReturn(job.getCredentials());
+ public void setUpHadoopJobConf() throws Exception {
+ jobConfSpy = spy(new JobConf(job.getConfiguration()));
+ when(jobConfSpy.getCredentials()).thenReturn(job.getCredentials());
}
@Test
- public void testConfigConnectorConnection() throws Exception {
- ConfigurationUtils.setConnectorConnectionConfig(Direction.FROM, job, getConfig());
- setUpJobConf();
- assertEquals(getConfig(), ConfigurationUtils.getConnectorConnectionConfig(Direction.FROM, jobConf));
-
- ConfigurationUtils.setConnectorConnectionConfig(Direction.TO, job, getConfig());
- setUpJobConf();
- assertEquals(getConfig(), ConfigurationUtils.getConnectorConnectionConfig(Direction.TO, jobConf));
+ public void testLinkConfiguration() throws Exception {
+ ConfigurationUtils.setConnectorLinkConfig(Direction.FROM, job, getConfig());
+ setUpHadoopJobConf();
+ assertEquals(getConfig(), ConfigurationUtils.getConnectorConnectionConfig(Direction.FROM, jobConfSpy));
+
+ ConfigurationUtils.setConnectorLinkConfig(Direction.TO, job, getConfig());
+ setUpHadoopJobConf();
+ assertEquals(getConfig(), ConfigurationUtils.getConnectorConnectionConfig(Direction.TO, jobConfSpy));
}
@Test
- public void testConfigConnectorJob() throws Exception {
+ public void testJobConfiguration() throws Exception {
ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, getConfig());
- setUpJobConf();
- assertEquals(getConfig(), ConfigurationUtils.getConnectorJobConfig(Direction.FROM, jobConf));
+ setUpHadoopJobConf();
+ assertEquals(getConfig(), ConfigurationUtils.getConnectorJobConfig(Direction.FROM, jobConfSpy));
ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, getConfig());
- setUpJobConf();
- assertEquals(getConfig(), ConfigurationUtils.getConnectorJobConfig(Direction.TO, jobConf));
- }
-
- @Test
- public void testConfigFrameworkConnection() throws Exception {
- ConfigurationUtils.setFrameworkConnectionConfig(Direction.FROM, job, getConfig());
- setUpJobConf();
- assertEquals(getConfig(), ConfigurationUtils.getFrameworkConnectionConfig(Direction.FROM, jobConf));
-
- ConfigurationUtils.setFrameworkConnectionConfig(Direction.TO, job, getConfig());
- setUpJobConf();
- assertEquals(getConfig(), ConfigurationUtils.getFrameworkConnectionConfig(Direction.TO, jobConf));
+ setUpHadoopJobConf();
+ assertEquals(getConfig(), ConfigurationUtils.getConnectorJobConfig(Direction.TO, jobConfSpy));
}
@Test
- public void testConfigFrameworkJob() throws Exception {
- ConfigurationUtils.setFrameworkJobConfig(job, getConfig());
- setUpJobConf();
- assertEquals(getConfig(), ConfigurationUtils.getFrameworkJobConfig(jobConf));
+ public void testDriverConfiguration() throws Exception {
+ ConfigurationUtils.setDriverConfig(job, getConfig());
+ setUpHadoopJobConf();
+ assertEquals(getConfig(), ConfigurationUtils.getDriverConfig(jobConfSpy));
}
@Test
public void testConnectorSchema() throws Exception {
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, getSchema("a"));
- assertEquals(getSchema("a"), ConfigurationUtils.getConnectorSchema(Direction.FROM, jobConf));
+ assertEquals(getSchema("a"), ConfigurationUtils.getConnectorSchema(Direction.FROM, jobConfSpy));
ConfigurationUtils.setConnectorSchema(Direction.TO, job, getSchema("b"));
- assertEquals(getSchema("b"), ConfigurationUtils.getConnectorSchema(Direction.TO, jobConf));
+ assertEquals(getSchema("b"), ConfigurationUtils.getConnectorSchema(Direction.TO, jobConfSpy));
}
@Test
public void testConnectorSchemaNull() throws Exception {
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, null);
- assertNull(ConfigurationUtils.getConnectorSchema(Direction.FROM, jobConf));
+ assertNull(ConfigurationUtils.getConnectorSchema(Direction.FROM, jobConfSpy));
ConfigurationUtils.setConnectorSchema(Direction.TO, job, null);
- assertNull(ConfigurationUtils.getConnectorSchema(Direction.FROM, jobConf));
+ assertNull(ConfigurationUtils.getConnectorSchema(Direction.FROM, jobConfSpy));
}
private Schema getSchema(String name) {
return new Schema(name).addColumn(new Text("c1"));
}
- private Config getConfig() {
- Config c = new Config();
- c.f.A = "This is secret text!";
+ private TestConfiguration getConfig() {
+ TestConfiguration c = new TestConfiguration();
+ c.c.A = "This is secret text!";
return c;
}
- @FormClass
- public static class F {
+ @ConfigClass
+ public static class C {
@Input String A;
@Override
public boolean equals(Object o) {
if (this == o) return true;
- if (!(o instanceof F)) return false;
+ if (!(o instanceof C)) return false;
- F f = (F) o;
+ C c = (C) o;
- if (A != null ? !A.equals(f.A) : f.A != null) return false;
+ if (A != null ? !A.equals(c.A) : c.A != null) return false;
return true;
}
@@ -152,21 +140,21 @@ public class TestConfigurationUtils {
}
@ConfigurationClass
- public static class Config {
- @Form F f;
+ public static class TestConfiguration {
+ @Config C c;
- public Config() {
- f = new F();
+ public TestConfiguration() {
+ c = new C();
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
- if (!(o instanceof Config)) return false;
+ if (!(o instanceof TestConfiguration)) return false;
- Config config = (Config) o;
+ TestConfiguration config = (TestConfiguration) o;
- if (f != null ? !f.equals(config.f) : config.f != null)
+ if (c != null ? !c.equals(config.c) : config.c != null)
return false;
return true;
@@ -174,7 +162,7 @@ public class TestConfigurationUtils {
@Override
public int hashCode() {
- return f != null ? f.hashCode() : 0;
+ return c != null ? c.hashCode() : 0;
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
index 61dc3b4..cc31d06 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
@@ -1,7 +1,7 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional inconfigation
+ * distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
[20/52] [abbrv] git commit: SQOOP-1497: Sqoop2: Entity Nomenclature
Revisited
Posted by ab...@apache.org.
SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/049994a0
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/049994a0
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/049994a0
Branch: refs/heads/SQOOP-1367
Commit: 049994a0217164b703e754978a85a6c17fcb720a
Parents: b04e796
Author: Veena Basavaraj <ve...@gmail.com>
Authored: Tue Sep 23 15:28:24 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:18 2014 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/client/SqoopClient.java | 280 ++++----
.../sqoop/client/request/ConnectionRequest.java | 91 ---
.../sqoop/client/request/ConnectorRequest.java | 47 --
.../request/ConnectorResourceRequest.java | 47 ++
.../request/DriverConfigResourceRequest.java | 42 ++
.../sqoop/client/request/FrameworkRequest.java | 42 --
.../apache/sqoop/client/request/JobRequest.java | 91 ---
.../client/request/JobResourceRequest.java | 91 +++
.../client/request/LinkResourceRequest.java | 89 +++
.../apache/sqoop/client/request/Request.java | 110 ---
.../sqoop/client/request/ResourceRequest.java | 113 +++
.../sqoop/client/request/SqoopRequests.java | 149 ----
.../client/request/SqoopResourceRequests.java | 149 ++++
.../sqoop/client/request/SubmissionRequest.java | 80 ---
.../request/SubmissionResourceRequest.java | 80 +++
.../sqoop/client/request/VersionRequest.java | 35 -
.../client/request/VersionResourceRequest.java | 35 +
.../apache/sqoop/client/TestSqoopClient.java | 105 ++-
.../org/apache/sqoop/common/SqoopException.java | 2 +-
.../org/apache/sqoop/etl/io/DataReader.java | 10 +-
.../org/apache/sqoop/job/etl/ActorContext.java | 77 --
.../apache/sqoop/job/etl/DestroyerContext.java | 2 +-
.../apache/sqoop/job/etl/ExtractorContext.java | 2 +-
.../sqoop/job/etl/InitializerContext.java | 2 +-
.../org/apache/sqoop/job/etl/LoaderContext.java | 2 +-
.../sqoop/job/etl/PartitionerContext.java | 2 +-
.../sqoop/job/etl/TransferableContext.java | 76 ++
.../org/apache/sqoop/json/ConnectionBean.java | 185 -----
.../sqoop/json/ConnectionValidationBean.java | 143 ----
.../org/apache/sqoop/json/ConnectorBean.java | 37 +-
.../org/apache/sqoop/json/DriverConfigBean.java | 92 +++
.../org/apache/sqoop/json/FrameworkBean.java | 102 ---
.../java/org/apache/sqoop/json/JobBean.java | 63 +-
.../java/org/apache/sqoop/json/LinkBean.java | 184 +++++
.../apache/sqoop/json/LinkValidationBean.java | 143 ++++
.../sqoop/json/util/FormSerialization.java | 2 +-
.../json/util/ResourceBundleSerialization.java | 13 +-
.../main/java/org/apache/sqoop/model/Input.java | 2 +-
.../org/apache/sqoop/model/MConnection.java | 130 ----
.../org/apache/sqoop/model/MDriverConfig.java | 97 +++
.../java/org/apache/sqoop/model/MFramework.java | 99 ---
.../java/org/apache/sqoop/model/MInput.java | 2 +-
.../main/java/org/apache/sqoop/model/MJob.java | 34 +-
.../main/java/org/apache/sqoop/model/MLink.java | 129 ++++
.../apache/sqoop/json/TestConnectionBean.java | 138 ----
.../apache/sqoop/json/TestDriverConfigBean.java | 64 ++
.../apache/sqoop/json/TestFrameworkBean.java | 64 --
.../java/org/apache/sqoop/json/TestJobBean.java | 4 +-
.../org/apache/sqoop/json/TestLinkBean.java | 138 ++++
.../java/org/apache/sqoop/json/TestUtil.java | 23 +-
.../apache/sqoop/json/TestValidationBean.java | 12 +-
.../sqoop/model/TestMAccountableEntity.java | 24 +-
.../org/apache/sqoop/model/TestMConnection.java | 122 ----
.../apache/sqoop/model/TestMDriverConfig.java | 47 ++
.../org/apache/sqoop/model/TestMFramework.java | 47 --
.../java/org/apache/sqoop/model/TestMLink.java | 122 ++++
.../connector/jdbc/GenericJdbcConnector.java | 12 +-
.../jdbc/GenericJdbcConnectorConstants.java | 2 +-
.../GenericJdbcConnectorMetadataUpgrader.java | 83 ---
.../jdbc/GenericJdbcConnectorUpgrader.java | 83 +++
.../connector/jdbc/GenericJdbcExtractor.java | 15 +-
.../jdbc/GenericJdbcFromDestroyer.java | 6 +-
.../jdbc/GenericJdbcFromInitializer.java | 60 +-
.../sqoop/connector/jdbc/GenericJdbcLoader.java | 14 +-
.../connector/jdbc/GenericJdbcPartitioner.java | 8 +-
.../connector/jdbc/GenericJdbcToDestroyer.java | 22 +-
.../jdbc/GenericJdbcToInitializer.java | 52 +-
.../connector/jdbc/GenericJdbcValidator.java | 66 +-
.../configuration/ConnectionConfiguration.java | 34 -
.../jdbc/configuration/ConnectionForm.java | 64 --
.../jdbc/configuration/FromJobConfig.java | 68 ++
.../configuration/FromJobConfiguration.java | 4 +-
.../jdbc/configuration/FromTableForm.java | 68 --
.../jdbc/configuration/LinkConfig.java | 64 ++
.../jdbc/configuration/LinkConfiguration.java | 34 +
.../jdbc/configuration/ToJobConfig.java | 55 ++
.../jdbc/configuration/ToJobConfiguration.java | 4 +-
.../jdbc/configuration/ToTableForm.java | 55 --
.../generic-jdbc-connector-config.properties | 127 ++++
.../generic-jdbc-connector-resources.properties | 127 ----
.../sqoop/connector/jdbc/TestExtractor.java | 14 +-
.../connector/jdbc/TestFromInitializer.java | 116 +--
.../apache/sqoop/connector/jdbc/TestLoader.java | 8 +-
.../sqoop/connector/jdbc/TestPartitioner.java | 36 +-
.../sqoop/connector/jdbc/TestToInitializer.java | 132 ++--
.../connector/hdfs/HdfsConfigUpgrader.java | 83 +++
.../sqoop/connector/hdfs/HdfsConnector.java | 16 +-
.../sqoop/connector/hdfs/HdfsConstants.java | 2 +-
.../sqoop/connector/hdfs/HdfsExtractor.java | 6 +-
.../sqoop/connector/hdfs/HdfsInitializer.java | 9 +-
.../apache/sqoop/connector/hdfs/HdfsLoader.java | 41 +-
.../connector/hdfs/HdfsMetadataUpgrader.java | 83 ---
.../sqoop/connector/hdfs/HdfsPartitioner.java | 12 +-
.../sqoop/connector/hdfs/HdfsValidator.java | 31 +-
.../configuration/ConnectionConfiguration.java | 31 -
.../hdfs/configuration/ConnectionForm.java | 29 -
.../hdfs/configuration/FromJobConfig.java | 30 +
.../configuration/FromJobConfiguration.java | 5 +-
.../connector/hdfs/configuration/InputForm.java | 30 -
.../hdfs/configuration/LinkConfig.java | 29 +
.../hdfs/configuration/LinkConfiguration.java | 31 +
.../hdfs/configuration/OutputCompression.java | 33 -
.../hdfs/configuration/OutputForm.java | 36 -
.../hdfs/configuration/OutputFormat.java | 33 -
.../hdfs/configuration/ToCompression.java | 33 +
.../connector/hdfs/configuration/ToFormat.java | 33 +
.../hdfs/configuration/ToJobConfig.java | 36 +
.../hdfs/configuration/ToJobConfiguration.java | 4 +-
.../hdfs/hdfsWriter/HdfsSequenceWriter.java | 2 +-
.../resources/hdfs-connector-config.properties | 58 ++
.../hdfs-connector-resources.properties | 58 --
.../sqoop/connector/hdfs/TestExtractor.java | 14 +-
.../apache/sqoop/connector/hdfs/TestLoader.java | 34 +-
.../sqoop/connector/hdfs/TestPartitioner.java | 14 +-
.../connector/idf/IntermediateDataFormat.java | 2 +-
.../sqoop/connector/ConnectorHandler.java | 2 +-
.../sqoop/core/ConfigurationConstants.java | 4 +-
.../java/org/apache/sqoop/core/SqoopServer.java | 8 +-
.../java/org/apache/sqoop/driver/Driver.java | 179 +++++
.../sqoop/driver/DriverConfigUpgrader.java | 79 ++
.../apache/sqoop/driver/DriverConstants.java | 68 ++
.../org/apache/sqoop/driver/DriverError.java | 66 ++
.../apache/sqoop/driver/DriverValidator.java | 54 ++
.../apache/sqoop/driver/ExecutionEngine.java | 60 ++
.../org/apache/sqoop/driver/JobManager.java | 712 +++++++++++++++++++
.../org/apache/sqoop/driver/JobRequest.java | 356 ++++++++++
.../apache/sqoop/driver/SubmissionEngine.java | 115 +++
.../driver/configuration/JobConfiguration.java | 34 +
.../driver/configuration/LinkConfiguration.java | 28 +
.../driver/configuration/ThrottlingForm.java | 32 +
.../apache/sqoop/framework/ExecutionEngine.java | 60 --
.../sqoop/framework/FrameworkConstants.java | 69 --
.../apache/sqoop/framework/FrameworkError.java | 66 --
.../sqoop/framework/FrameworkManager.java | 178 -----
.../framework/FrameworkMetadataUpgrader.java | 79 --
.../sqoop/framework/FrameworkValidator.java | 54 --
.../org/apache/sqoop/framework/JobManager.java | 710 ------------------
.../org/apache/sqoop/framework/JobRequest.java | 356 ----------
.../sqoop/framework/SubmissionEngine.java | 115 ---
.../configuration/ConnectionConfiguration.java | 30 -
.../configuration/JobConfiguration.java | 31 -
.../framework/configuration/SecurityForm.java | 29 -
.../framework/configuration/ThrottlingForm.java | 32 -
.../apache/sqoop/repository/JdbcRepository.java | 106 ++-
.../sqoop/repository/JdbcRepositoryHandler.java | 190 +++--
.../org/apache/sqoop/repository/Repository.java | 221 +++---
.../sqoop/repository/RepositoryError.java | 44 +-
.../src/main/resources/driver-config.properties | 30 +
.../resources/framework-resources.properties | 71 --
.../core/MockInvalidConfigurationProvider.java | 7 -
.../apache/sqoop/core/TestConfiguration.java | 165 -----
.../sqoop/core/TestSqoopConfiguration.java | 165 +++++
.../java/org/apache/sqoop/core/TestUtils.java | 2 -
.../sqoop/driver/TestDriverConfigUpgrader.java | 171 +++++
.../org/apache/sqoop/driver/TestJobManager.java | 190 +++++
.../org/apache/sqoop/driver/TestJobRequest.java | 70 ++
.../TestFrameworkMetadataUpgrader.java | 170 -----
.../sqoop/framework/TestFrameworkValidator.java | 156 ----
.../apache/sqoop/framework/TestJobManager.java | 173 -----
.../apache/sqoop/framework/TestJobRequest.java | 71 --
.../sqoop/repository/TestJdbcRepository.java | 559 ++++++++-------
.../sqoop/execution/mapreduce/MRJobRequest.java | 4 +-
.../mapreduce/MapreduceExecutionEngine.java | 13 +-
.../java/org/apache/sqoop/job/JobConstants.java | 4 +-
.../job/mr/SqoopOutputFormatLoadExecutor.java | 4 +-
.../repository/derby/DerbyRepoConstants.java | 2 +-
.../sqoop/repository/derby/DerbyRepoError.java | 122 ++--
.../derby/DerbyRepositoryHandler.java | 316 ++++----
.../repository/derby/DerbySchemaQuery.java | 2 +-
.../sqoop/repository/derby/DerbyTestCase.java | 81 ++-
.../derby/TestConnectionHandling.java | 245 -------
.../repository/derby/TestConnectorHandling.java | 18 +-
.../derby/TestDriverConfigHandling.java | 126 ++++
.../repository/derby/TestFrameworkHandling.java | 129 ----
.../sqoop/repository/derby/TestInputTypes.java | 34 +-
.../sqoop/repository/derby/TestInternals.java | 20 +-
.../sqoop/repository/derby/TestJobHandling.java | 74 +-
.../repository/derby/TestLinkHandling.java | 244 +++++++
.../derby/TestSubmissionHandling.java | 66 +-
.../sqoop/handler/ConnectionRequestHandler.java | 274 -------
.../handler/DriverConfigRequestHandler.java | 50 ++
.../sqoop/handler/FrameworkRequestHandler.java | 53 --
.../apache/sqoop/handler/JobRequestHandler.java | 14 +-
.../sqoop/handler/LinkRequestHandler.java | 271 +++++++
.../sqoop/handler/SubmissionRequestHandler.java | 3 +-
.../sqoop/server/v1/ConnectionServlet.java | 56 --
.../sqoop/server/v1/DriverConfigServlet.java | 41 ++
.../sqoop/server/v1/FrameworkServlet.java | 40 --
.../org/apache/sqoop/server/v1/JobServlet.java | 13 +-
.../org/apache/sqoop/server/v1/LinkServlet.java | 57 ++
.../sqoop/server/v1/SubmissionServlet.java | 11 +-
server/src/main/webapp/WEB-INF/web.xml | 21 +-
.../org/apache/sqoop/shell/CloneCommand.java | 2 +-
.../sqoop/shell/CloneConnectionFunction.java | 109 ---
.../apache/sqoop/shell/CloneJobFunction.java | 14 +-
.../apache/sqoop/shell/CloneLinkFunction.java | 109 +++
.../org/apache/sqoop/shell/CreateCommand.java | 2 +-
.../sqoop/shell/CreateConnectionFunction.java | 108 ---
.../apache/sqoop/shell/CreateJobFunction.java | 24 +-
.../apache/sqoop/shell/CreateLinkFunction.java | 108 +++
.../org/apache/sqoop/shell/DeleteCommand.java | 2 +-
.../sqoop/shell/DeleteConnectionFunction.java | 47 --
.../apache/sqoop/shell/DeleteJobFunction.java | 2 +-
.../apache/sqoop/shell/DeleteLinkFunction.java | 47 ++
.../org/apache/sqoop/shell/DisableCommand.java | 2 +-
.../sqoop/shell/DisableConnectionFunction.java | 47 --
.../apache/sqoop/shell/DisableLinkFunction.java | 47 ++
.../org/apache/sqoop/shell/EnableCommand.java | 2 +-
.../sqoop/shell/EnableConnectionFunction.java | 47 --
.../apache/sqoop/shell/EnableLinkFunction.java | 47 ++
.../org/apache/sqoop/shell/ShowCommand.java | 4 +-
.../sqoop/shell/ShowConnectionFunction.java | 128 ----
.../sqoop/shell/ShowConnectorFunction.java | 2 +-
.../sqoop/shell/ShowDriverConfigFunction.java | 60 ++
.../sqoop/shell/ShowFrameworkFunction.java | 60 --
.../org/apache/sqoop/shell/ShowJobFunction.java | 10 +-
.../apache/sqoop/shell/ShowLinkFunction.java | 128 ++++
.../apache/sqoop/shell/ShowVersionFunction.java | 8 +-
.../org/apache/sqoop/shell/UpdateCommand.java | 2 +-
.../sqoop/shell/UpdateConnectionFunction.java | 107 ---
.../apache/sqoop/shell/UpdateJobFunction.java | 10 +-
.../apache/sqoop/shell/UpdateLinkFunction.java | 107 +++
.../org/apache/sqoop/shell/core/Constants.java | 112 +--
.../utils/ConnectionDynamicFormOptions.java | 44 --
.../apache/sqoop/shell/utils/FormDisplayer.java | 20 +-
.../apache/sqoop/shell/utils/FormFiller.java | 66 +-
.../shell/utils/LinkDynamicFormOptions.java | 43 ++
.../sqoop/shell/utils/SubmissionDisplayer.java | 4 +-
.../main/resources/shell-resource.properties | 66 +-
.../sqoop/connector/spi/MetadataUpgrader.java | 46 --
.../sqoop/connector/spi/RepositoryUpgrader.java | 51 ++
.../sqoop/connector/spi/SqoopConnector.java | 12 +-
.../org/apache/sqoop/job/etl/Destroyer.java | 6 +-
.../org/apache/sqoop/job/etl/Extractor.java | 8 +-
.../java/org/apache/sqoop/job/etl/From.java | 4 +-
.../org/apache/sqoop/job/etl/Initializer.java | 14 +-
.../java/org/apache/sqoop/job/etl/Loader.java | 6 +-
.../org/apache/sqoop/job/etl/Partitioner.java | 6 +-
.../main/java/org/apache/sqoop/job/etl/To.java | 4 +-
.../org/apache/sqoop/validation/Validator.java | 10 +-
.../mapreduce/MapreduceSubmissionEngine.java | 36 +-
.../apache/sqoop/test/db/DatabaseProvider.java | 14 +-
.../sqoop/test/db/DatabaseProviderFactory.java | 2 +-
.../sqoop/test/testcases/ConnectorTestCase.java | 61 +-
.../sqoop/test/testcases/TomcatTestCase.java | 14 +-
.../jdbc/generic/FromHDFSToRDBMSTest.java | 32 +-
.../jdbc/generic/FromRDBMSToHDFSTest.java | 63 +-
.../connector/jdbc/generic/PartitionerTest.java | 37 +-
.../jdbc/generic/TableStagedRDBMSTest.java | 32 +-
.../SubmissionWithDisabledModelObjectsTest.java | 50 +-
.../sqoop/integration/server/VersionTest.java | 8 +-
.../apache/sqoop/tools/tool/UpgradeTool.java | 19 +-
252 files changed, 8493 insertions(+), 8768 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
index 0e45a24..d7e6768 100644
--- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
+++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
@@ -17,16 +17,16 @@
*/
package org.apache.sqoop.client;
-import org.apache.sqoop.client.request.SqoopRequests;
+import org.apache.sqoop.client.request.SqoopResourceRequests;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.json.ConnectorBean;
-import org.apache.sqoop.json.FrameworkBean;
+import org.apache.sqoop.json.DriverConfigBean;
import org.apache.sqoop.json.ValidationResultBean;
import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.validation.Status;
@@ -45,7 +45,7 @@ import java.util.ResourceBundle;
* implementation is not thread safe.
*
* SqoopClient is keeping cache of objects that are unlikely to be changed
- * (Resources, Connector structures). Volatile structures (Connections, Jobs)
+ * (Resources, Connector structures). Volatile structures (Links, Jobs)
* are not cached.
*/
public class SqoopClient {
@@ -53,32 +53,29 @@ public class SqoopClient {
/**
* Underlying request object to fetch data from Sqoop server.
*/
- private SqoopRequests requests;
+ private SqoopResourceRequests resourceRequests;
/**
* True if user retrieved all connectors at once.
*/
- private boolean allConnectors;
-
+ private boolean isAllConnectors;
/**
- * All cached bundles for all connectors.
+ * All cached connectors.
*/
- private Map<Long, ResourceBundle> bundles;
-
+ private Map<Long, MConnector> connectors;
/**
- * Cached framework bundle.
+ * All cached bundles for all connectors.
*/
- private ResourceBundle frameworkBundle;
+ private Map<Long, ResourceBundle> connectorConfigBundles;
/**
- * All cached connectors.
+ * Cached driverConfig.
*/
- private Map<Long, MConnector> connectors;
-
+ private MDriverConfig driverConfig;
/**
- * Cached framework.
+ * Cached driverConfig bundle.
*/
- private MFramework framework;
+ private ResourceBundle driverConfigBundle;
/**
* Status flags used when updating the submission callback status
@@ -90,7 +87,7 @@ public class SqoopClient {
}
public SqoopClient(String serverUrl) {
- requests = new SqoopRequests();
+ resourceRequests = new SqoopResourceRequests();
setServerUrl(serverUrl);
}
@@ -102,7 +99,7 @@ public class SqoopClient {
* @param serverUrl Server URL
*/
public void setServerUrl(String serverUrl) {
- requests.setServerUrl(serverUrl);
+ resourceRequests.setServerUrl(serverUrl);
clearCache();
}
@@ -111,8 +108,8 @@ public class SqoopClient {
*
* @param requests SqoopRequests object
*/
- public void setSqoopRequests(SqoopRequests requests) {
- this.requests = requests;
+ public void setSqoopRequests(SqoopResourceRequests requests) {
+ this.resourceRequests = requests;
clearCache();
}
@@ -120,11 +117,11 @@ public class SqoopClient {
* Clear internal cache.
*/
public void clearCache() {
- bundles = new HashMap<Long, ResourceBundle>();
- frameworkBundle = null;
+ connectorConfigBundles = new HashMap<Long, ResourceBundle>();
+ driverConfigBundle = null;
connectors = new HashMap<Long, MConnector>();
- framework = null;
- allConnectors = false;
+ driverConfig = null;
+ isAllConnectors = false;
}
/**
@@ -155,7 +152,7 @@ public class SqoopClient {
// If the connector wasn't in cache and we have all connectors,
// it simply do not exists.
- if(allConnectors) return null;
+ if(isAllConnectors) return null;
// Retrieve all connectors from server
getConnectors();
@@ -186,9 +183,9 @@ public class SqoopClient {
* @param cid Connector id
*/
private void retrieveConnector(long cid) {
- ConnectorBean request = requests.readConnector(cid);
+ ConnectorBean request = resourceRequests.readConnector(cid);
connectors.put(cid, request.getConnectors().get(0));
- bundles.put(cid, request.getResourceBundles().get(cid));
+ connectorConfigBundles.put(cid, request.getResourceBundles().get(cid));
}
/**
@@ -197,16 +194,16 @@ public class SqoopClient {
* @return
*/
public Collection<MConnector> getConnectors() {
- if(allConnectors) {
+ if(isAllConnectors) {
return connectors.values();
}
- ConnectorBean bean = requests.readConnector(null);
- allConnectors = true;
+ ConnectorBean bean = resourceRequests.readConnector(null);
+ isAllConnectors = true;
for(MConnector connector : bean.getConnectors()) {
connectors.put(connector.getPersistenceId(), connector);
}
- bundles = bean.getResourceBundles();
+ connectorConfigBundles = bean.getResourceBundles();
return connectors.values();
}
@@ -214,173 +211,171 @@ public class SqoopClient {
/**
* Get resource bundle for given connector.
*
- * @param cid Connector id.
+ * @param connectorId Connector id.
* @return
*/
- public ResourceBundle getResourceBundle(long cid) {
- if(bundles.containsKey(cid)) {
- return bundles.get(cid);
+ public ResourceBundle getConnectorConfigResourceBundle(long connectorId) {
+ if(connectorConfigBundles.containsKey(connectorId)) {
+ return connectorConfigBundles.get(connectorId);
}
- retrieveConnector(cid);
- return bundles.get(cid);
+ retrieveConnector(connectorId);
+ return connectorConfigBundles.get(connectorId);
}
/**
- * Return framework metadata.
+ * Return driver config.
*
* @return
*/
- public MFramework getFramework() {
- if(framework != null) {
- return framework.clone(false);
+ public MDriverConfig getDriverConfig() {
+ if(driverConfig != null) {
+ return driverConfig.clone(false);
}
-
- retrieveFramework();
- return framework.clone(false);
+ retrieveAndCacheDriverConfig();
+ return driverConfig.clone(false);
}
/**
- * Retrieve framework structure and cache it.
+ * Retrieve driverConfig and cache it.
*/
- private void retrieveFramework() {
- FrameworkBean request = requests.readFramework();
- framework = request.getFramework();
- frameworkBundle = request.getResourceBundle();
+ private void retrieveAndCacheDriverConfig() {
+ DriverConfigBean driverConfigBean = resourceRequests.readDriverConfig();
+ driverConfig = driverConfigBean.getDriverConfig();
+ driverConfigBundle = driverConfigBean.getResourceBundle();
}
/**
- * Return framework bundle.
+ * Return driverConfig bundle.
*
* @return
*/
- public ResourceBundle getFrameworkResourceBundle() {
- if(frameworkBundle != null) {
- return frameworkBundle;
+ public ResourceBundle getDriverConfigBundle() {
+ if(driverConfigBundle != null) {
+ return driverConfigBundle;
}
-
- retrieveFramework();
- return frameworkBundle;
+ retrieveAndCacheDriverConfig();
+ return driverConfigBundle;
}
/**
- * Create new connection object for given connector.
+ * Create new link object for given connector id
*
- * @param cid Connector id
+ * @param connectorId Connector id
* @return
*/
- public MConnection newConnection(long cid) {
- return new MConnection(
- cid,
- getConnector(cid).getConnectionForms(),
- getFramework().getConnectionForms()
+ public MLink createLink(long connectorId) {
+ return new MLink(
+ connectorId,
+ getConnector(connectorId).getConnectionForms(),
+ getDriverConfig().getConnectionForms()
);
}
/**
- * Create new connection object for given connector.
+ * Create new link object for given connector name
*
* @param connectorName Connector name
* @return
*/
- public MConnection newConnection(String connectorName) {
+ public MLink createLink(String connectorName) {
MConnector connector = getConnector(connectorName);
if(connector == null) {
throw new SqoopException(ClientError.CLIENT_0003, connectorName);
}
- return newConnection(connector.getPersistenceId());
+ return createLink(connector.getPersistenceId());
}
/**
- * Retrieve connection with given id.
+ * Retrieve link for given id.
*
- * @param xid Connnection id
+ * @param linkId Link id
* @return
*/
- public MConnection getConnection(long xid) {
- return requests.readConnection(xid).getConnections().get(0);
+ public MLink getLink(long linkId) {
+ return resourceRequests.readLink(linkId).getLinks().get(0);
}
/**
- * Retrieve list of all connections.
+ * Retrieve list of all links.
*
* @return
*/
- public List<MConnection> getConnections() {
- return requests.readConnection(null).getConnections();
+ public List<MLink> getLinks() {
+ return resourceRequests.readLink(null).getLinks();
}
/**
- * Create the connection on server.
+ * Create the link and save to the repository
*
- * @param connection Connection that should be created
+ * @param link link that should be created
* @return
*/
- public Status createConnection(MConnection connection) {
- return applyValidations(requests.createConnection(connection), connection);
+ public Status saveLink(MLink link) {
+ return applyLinkValidations(resourceRequests.saveLink(link), link);
}
/**
- * Update connection on the server.
+ * Update link on the server.
*
- * @param connection Connection that should be updated
+ * @param link link that should be updated
* @return
*/
- public Status updateConnection(MConnection connection) {
- return applyValidations(requests.updateConnection(connection), connection);
+ public Status updateLink(MLink link) {
+ return applyLinkValidations(resourceRequests.updateLink(link), link);
}
/**
- * Enable/disable connection with given id
+ * Enable/disable link with given id
*
- * @param xid Connection id
+ * @param linkId link id
* @param enabled Enable or disable
*/
- public void enableConnection(long xid, boolean enabled) {
- requests.enableConnection(xid, enabled);
+ public void enableLink(long linkId, boolean enabled) {
+ resourceRequests.enableLink(linkId, enabled);
}
/**
- * Delete connection with given id.
+ * Delete link with given id.
*
- * @param xid Connection id
+ * @param linkId link id
*/
- public void deleteConnection(long xid) {
- requests.deleteConnection(xid);
+ public void deleteLink(long linkId) {
+ resourceRequests.deleteLink(linkId);
}
/**
- * Create new job the for given connections.
+ * Create new job the for given links.
*
- * @param fromXid From Connection id
- * @param toXid To Connection id
+ * @param fromLinkId From link id
+ * @param toLinkId To link id
* @return
*/
- public MJob newJob(long fromXid, long toXid) {
- MConnection fromConnection = getConnection(fromXid);
- MConnection toConnection = getConnection(toXid);
+ public MJob createJob(long fromLinkId, long toLinkId) {
+ MLink fromLink = getLink(fromLinkId);
+ MLink toLink = getLink(toLinkId);
return new MJob(
- fromConnection.getConnectorId(),
- toConnection.getConnectorId(),
- fromConnection.getPersistenceId(),
- toConnection.getPersistenceId(),
- getConnector(fromConnection.getConnectorId()).getJobForms(Direction.FROM),
- getConnector(toConnection.getConnectorId()).getJobForms(Direction.TO),
- getFramework().getJobForms()
+ fromLink.getConnectorId(),
+ toLink.getConnectorId(),
+ fromLink.getPersistenceId(),
+ toLink.getPersistenceId(),
+ getConnector(fromLink.getConnectorId()).getJobForms(Direction.FROM),
+ getConnector(toLink.getConnectorId()).getJobForms(Direction.TO),
+ getDriverConfig().getJobForms()
);
}
/**
* Retrieve job for given id.
*
- * @param jid Job id
+ * @param jobId Job id
* @return
*/
- public MJob getJob(long jid) {
- return requests.readJob(jid).getJobs().get(0);
+ public MJob getJob(long jobId) {
+ return resourceRequests.readJob(jobId).getJobs().get(0);
}
/**
@@ -389,17 +384,17 @@ public class SqoopClient {
* @return
*/
public List<MJob> getJobs() {
- return requests.readJob(null).getJobs();
+ return resourceRequests.readJob(null).getJobs();
}
/**
- * Create job on server.
+ * Create job on server and save to the repository
*
* @param job Job that should be created
* @return
*/
- public Status createJob(MJob job) {
- return applyValidations(requests.createJob(job), job);
+ public Status saveJob(MJob job) {
+ return applyJobValidations(resourceRequests.saveJob(job), job);
}
/**
@@ -408,7 +403,7 @@ public class SqoopClient {
* @return
*/
public Status updateJob(MJob job) {
- return applyValidations(requests.updateJob(job), job);
+ return applyJobValidations(resourceRequests.updateJob(job), job);
}
/**
@@ -418,44 +413,45 @@ public class SqoopClient {
* @param enabled Enable or disable
*/
public void enableJob(long jid, boolean enabled) {
- requests.enableJob(jid, enabled);
+ resourceRequests.enableJob(jid, enabled);
}
/**
* Delete job with given id.
*
- * @param jid Job id
+ * @param jobId Job id
*/
- public void deleteJob(long jid) {
- requests.deleteJob(jid);
+ public void deleteJob(long jobId) {
+ resourceRequests.deleteJob(jobId);
}
/**
* Start job with given id.
*
- * @param jid Job id
+ * @param jobId Job id
* @return
*/
- public MSubmission startSubmission(long jid) {
- return requests.createSubmission(jid).getSubmissions().get(0);
+ public MSubmission startSubmission(long jobId) {
+ return resourceRequests.createSubmission(jobId).getSubmissions().get(0);
}
/**
* Method used for synchronous job submission.
* Pass null to callback parameter if submission status is not required and after completion
* job execution returns MSubmission which contains final status of submission.
- * @param jid - Job ID
+ * @param jobId - Job ID
* @param callback - User may set null if submission status is not required, else callback methods invoked
* @param pollTime - Server poll time
* @return MSubmission - Final status of job submission
* @throws InterruptedException
*/
- public MSubmission startSubmission(long jid, SubmissionCallback callback, long pollTime) throws InterruptedException {
+ public MSubmission startSubmission(long jobId, SubmissionCallback callback, long pollTime)
+ throws InterruptedException {
if(pollTime <= 0) {
throw new SqoopException(ClientError.CLIENT_0002);
}
boolean first = true;
- MSubmission submission = requests.createSubmission(jid).getSubmissions().get(0);
+ MSubmission submission = resourceRequests.createSubmission(jobId).getSubmissions().get(0);
while(submission.getStatus().isRunning()) {
if(first) {
submissionCallback(callback, submission, SubmissionStatus.SUBMITTED);
@@ -464,7 +460,7 @@ public class SqoopClient {
submissionCallback(callback, submission, SubmissionStatus.UPDATED);
}
Thread.sleep(pollTime);
- submission = getSubmissionStatus(jid);
+ submission = getSubmissionStatus(jobId);
}
submissionCallback(callback, submission, SubmissionStatus.FINISHED);
return submission;
@@ -477,9 +473,9 @@ public class SqoopClient {
* @param submission
* @param status
*/
- private void submissionCallback(SubmissionCallback callback,
- MSubmission submission, SubmissionStatus status) {
- if(callback == null) {
+ private void submissionCallback(SubmissionCallback callback, MSubmission submission,
+ SubmissionStatus status) {
+ if (callback == null) {
return;
}
switch (status) {
@@ -501,7 +497,7 @@ public class SqoopClient {
* @return
*/
public MSubmission stopSubmission(long jid) {
- return requests.deleteSubmission(jid).getSubmissions().get(0);
+ return resourceRequests.deleteSubmission(jid).getSubmissions().get(0);
}
/**
@@ -511,7 +507,7 @@ public class SqoopClient {
* @return
*/
public MSubmission getSubmissionStatus(long jid) {
- return requests.readSubmission(jid).getSubmissions().get(0);
+ return resourceRequests.readSubmission(jid).getSubmissions().get(0);
}
/**
@@ -520,46 +516,46 @@ public class SqoopClient {
* @return
*/
public List<MSubmission> getSubmissions() {
- return requests.readHistory(null).getSubmissions();
+ return resourceRequests.readHistory(null).getSubmissions();
}
/**
* Retrieve list of submissions for given jobId.
*
- * @param jid Job id
+ * @param jobId Job id
* @return
*/
- public List<MSubmission> getSubmissionsForJob(long jid) {
- return requests.readHistory(jid).getSubmissions();
+ public List<MSubmission> getSubmissionsForJob(long jobId) {
+ return resourceRequests.readHistory(jobId).getSubmissions();
}
- private Status applyValidations(ValidationResultBean bean, MConnection connection) {
+ private Status applyLinkValidations(ValidationResultBean bean, MLink link) {
ValidationResult connector = bean.getValidationResults()[0];
- ValidationResult framework = bean.getValidationResults()[1];
+ ValidationResult driverConfig = bean.getValidationResults()[1];
// Apply validation results
- FormUtils.applyValidation(connection.getConnectorPart().getForms(), connector);
- FormUtils.applyValidation(connection.getFrameworkPart().getForms(), framework);
+ FormUtils.applyValidation(link.getConnectorPart().getForms(), connector);
+ FormUtils.applyValidation(link.getFrameworkPart().getForms(), driverConfig);
Long id = bean.getId();
if(id != null) {
- connection.setPersistenceId(id);
+ link.setPersistenceId(id);
}
- return Status.getWorstStatus(connector.getStatus(), framework.getStatus());
+ return Status.getWorstStatus(connector.getStatus(), driverConfig.getStatus());
}
- private Status applyValidations(ValidationResultBean bean, MJob job) {
+ private Status applyJobValidations(ValidationResultBean bean, MJob job) {
ValidationResult fromConnector = bean.getValidationResults()[0];
ValidationResult toConnector = bean.getValidationResults()[1];
- ValidationResult framework = bean.getValidationResults()[2];
+ ValidationResult driverConfig = bean.getValidationResults()[2];
// Apply validation results
// @TODO(Abe): From/To validation.
FormUtils.applyValidation(
job.getConnectorPart(Direction.FROM).getForms(),
fromConnector);
- FormUtils.applyValidation(job.getFrameworkPart().getForms(), framework);
+ FormUtils.applyValidation(job.getFrameworkPart().getForms(), driverConfig);
FormUtils.applyValidation(
job.getConnectorPart(Direction.TO).getForms(),
toConnector);
@@ -569,6 +565,6 @@ public class SqoopClient {
job.setPersistenceId(id);
}
- return Status.getWorstStatus(fromConnector.getStatus(), framework.getStatus(), toConnector.getStatus());
+ return Status.getWorstStatus(fromConnector.getStatus(), driverConfig.getStatus(), toConnector.getStatus());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/ConnectionRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/ConnectionRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ConnectionRequest.java
deleted file mode 100644
index 8436fc5..0000000
--- a/client/src/main/java/org/apache/sqoop/client/request/ConnectionRequest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.client.request;
-
-import org.apache.sqoop.json.ConnectionBean;
-import org.apache.sqoop.json.ValidationResultBean;
-import org.apache.sqoop.model.MConnection;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-/**
- * Provide CRUD semantics over RESTfull HTTP API for connections. All operations
- * are normally supported.
- */
-public class ConnectionRequest extends Request {
-
- public static final String RESOURCE = "v1/connection/";
-
- private static final String ENABLE = "/enable";
- private static final String DISABLE = "/disable";
-
- public ConnectionBean read(String serverUrl, Long xid) {
- String response;
- if (xid == null) {
- response = super.get(serverUrl + RESOURCE + "all");
- } else {
- response = super.get(serverUrl + RESOURCE + xid);
- }
- JSONObject jsonObject = (JSONObject)JSONValue.parse(response);
-
- ConnectionBean connectionBean = new ConnectionBean();
- connectionBean.restore(jsonObject);
-
- return connectionBean;
- }
-
- public ValidationResultBean create(String serverUrl, MConnection connection) {
- ConnectionBean connectionBean = new ConnectionBean(connection);
-
- // Extract all form inputs including sensitive inputs
- JSONObject connectionJson = connectionBean.extract(false);
-
- String response = super.post(serverUrl + RESOURCE, connectionJson.toJSONString());
-
- ValidationResultBean validationBean = new ValidationResultBean();
- validationBean.restore((JSONObject) JSONValue.parse(response));
-
- return validationBean;
- }
-
- public ValidationResultBean update(String serverUrl, MConnection connection) {
- ConnectionBean connectionBean = new ConnectionBean(connection);
-
- // Extract all form inputs including sensitive inputs
- JSONObject connectionJson = connectionBean.extract(false);
-
- String response = super.put(serverUrl + RESOURCE + connection.getPersistenceId(), connectionJson.toJSONString());
-
- ValidationResultBean validationBean = new ValidationResultBean();
- validationBean.restore((JSONObject) JSONValue.parse(response));
-
- return validationBean;
- }
-
- public void delete(String serverUrl, Long id) {
- super.delete(serverUrl + RESOURCE + id);
- }
-
- public void enable(String serverUrl, Long id, Boolean enabled) {
- if (enabled) {
- super.put(serverUrl + RESOURCE + id + ENABLE, null);
- } else {
- super.put(serverUrl + RESOURCE + id + DISABLE, null);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/ConnectorRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/ConnectorRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ConnectorRequest.java
deleted file mode 100644
index 876099c..0000000
--- a/client/src/main/java/org/apache/sqoop/client/request/ConnectorRequest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.client.request;
-
-import org.apache.sqoop.json.ConnectorBean;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-/**
- * Provide cRud semantics over RESTfull HTTP API for connectors. Only read
- * is supported as creation, update and delete might be done only directly on
- * server side.
- */
-public class ConnectorRequest extends Request
-{
- public static final String RESOURCE = "v1/connector/";
-
- public ConnectorBean read(String serverUrl, Long cid) {
- String response;
- if (cid == null) {
- response = super.get(serverUrl + RESOURCE + "all");
- } else {
- response = super.get(serverUrl + RESOURCE + cid);
- }
- JSONObject jsonObject = (JSONObject)JSONValue.parse(response);
-
- ConnectorBean connectorBean = new ConnectorBean();
- connectorBean.restore(jsonObject);
-
- return connectorBean;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
new file mode 100644
index 0000000..2855de6
--- /dev/null
+++ b/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client.request;
+
+import org.apache.sqoop.json.ConnectorBean;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+/**
+ * Provide cRud semantics over RESTfull HTTP API for connectors. Only read
+ * is supported as creation, update and delete might be done only directly on
+ * server side.
+ */
+public class ConnectorResourceRequest extends ResourceRequest
+{
+ public static final String RESOURCE = "v1/connector/";
+
+ public ConnectorBean read(String serverUrl, Long cid) {
+ String response;
+ if (cid == null) {
+ response = super.get(serverUrl + RESOURCE + "all");
+ } else {
+ response = super.get(serverUrl + RESOURCE + cid);
+ }
+ JSONObject jsonObject = (JSONObject)JSONValue.parse(response);
+
+ ConnectorBean connectorBean = new ConnectorBean();
+ connectorBean.restore(jsonObject);
+
+ return connectorBean;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java
new file mode 100644
index 0000000..7b2ddc2
--- /dev/null
+++ b/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client.request;
+
+import org.apache.sqoop.json.DriverConfigBean;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+/**
+ * Provide cRud semantics over RESTfull HTTP API for driverConfig. Only read
+ * is supported as creation, update and delete is not allowed.
+ */
+public class DriverConfigResourceRequest extends ResourceRequest {
+
+ public static final String RESOURCE = "v1/config/driver";
+
+ public DriverConfigBean read(String serverUrl) {
+ String response = null;
+ response = super.get(serverUrl + RESOURCE);
+ JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
+
+ DriverConfigBean driverConfigBean = new DriverConfigBean();
+ driverConfigBean.restore(jsonObject);
+
+ return driverConfigBean;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/FrameworkRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/FrameworkRequest.java b/client/src/main/java/org/apache/sqoop/client/request/FrameworkRequest.java
deleted file mode 100644
index 06f9006..0000000
--- a/client/src/main/java/org/apache/sqoop/client/request/FrameworkRequest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.client.request;
-
-import org.apache.sqoop.json.FrameworkBean;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-/**
- * Provide cRud semantics over RESTfull HTTP API for framework. Only read
- * is supported as creation, update and delete is not allowed.
- */
-public class FrameworkRequest extends Request {
-
- public static final String RESOURCE = "v1/framework";
-
- public FrameworkBean read(String serverUrl) {
- String response = null;
- response = super.get(serverUrl + RESOURCE);
- JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
-
- FrameworkBean frameworkBean = new FrameworkBean();
- frameworkBean.restore(jsonObject);
-
- return frameworkBean;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/JobRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/JobRequest.java b/client/src/main/java/org/apache/sqoop/client/request/JobRequest.java
deleted file mode 100644
index e520883..0000000
--- a/client/src/main/java/org/apache/sqoop/client/request/JobRequest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.client.request;
-
-import org.apache.sqoop.json.JobBean;
-import org.apache.sqoop.json.ValidationResultBean;
-import org.apache.sqoop.model.MJob;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-/**
- * Provide CRUD semantics over RESTfull HTTP API for jobs. All operations
- * are normally supported.
- */
-public class JobRequest extends Request {
-
- public static final String RESOURCE = "v1/job/";
-
- private static final String ENABLE = "/enable";
- private static final String DISABLE = "/disable";
-
- public JobBean read(String serverUrl, Long xid) {
- String response;
- if (xid == null) {
- response = super.get(serverUrl + RESOURCE + "all");
- } else {
- response = super.get(serverUrl + RESOURCE + xid);
- }
- JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
-
- JobBean jobBean = new JobBean();
- jobBean.restore(jsonObject);
-
- return jobBean;
- }
-
- public ValidationResultBean create(String serverUrl, MJob job) {
- JobBean jobBean = new JobBean(job);
-
- // Extract all form inputs including sensitive inputs
- JSONObject jobJson = jobBean.extract(false);
-
- String response = super.post(serverUrl + RESOURCE, jobJson.toJSONString());
-
- ValidationResultBean validationBean = new ValidationResultBean();
- validationBean.restore((JSONObject) JSONValue.parse(response));
-
- return validationBean;
- }
-
- public ValidationResultBean update(String serverUrl, MJob job) {
- JobBean jobBean = new JobBean(job);
-
- // Extract all form inputs including sensitive inputs
- JSONObject jobJson = jobBean.extract(false);
-
- String response = super.put(serverUrl + RESOURCE + job.getPersistenceId(), jobJson.toJSONString());
-
- ValidationResultBean validationBean = new ValidationResultBean();
- validationBean.restore((JSONObject) JSONValue.parse(response));
-
- return validationBean;
- }
-
- public void delete(String serverUrl, Long id) {
- super.delete(serverUrl + RESOURCE + id);
- }
-
- public void enable(String serverUrl, Long id, Boolean enabled) {
- if (enabled) {
- super.put(serverUrl + RESOURCE + id + ENABLE, null);
- } else {
- super.put(serverUrl + RESOURCE + id + DISABLE, null);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
new file mode 100644
index 0000000..fd858ec
--- /dev/null
+++ b/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client.request;
+
+import org.apache.sqoop.json.JobBean;
+import org.apache.sqoop.json.ValidationResultBean;
+import org.apache.sqoop.model.MJob;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+/**
+ * Provide CRUD semantics over RESTfull HTTP API for jobs. All operations
+ * are normally supported.
+ */
+public class JobResourceRequest extends ResourceRequest {
+
+ public static final String RESOURCE = "v1/job/";
+
+ private static final String ENABLE = "/enable";
+ private static final String DISABLE = "/disable";
+
+ public JobBean read(String serverUrl, Long xid) {
+ String response;
+ if (xid == null) {
+ response = super.get(serverUrl + RESOURCE + "all");
+ } else {
+ response = super.get(serverUrl + RESOURCE + xid);
+ }
+ JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
+
+ JobBean jobBean = new JobBean();
+ jobBean.restore(jsonObject);
+
+ return jobBean;
+ }
+
+ public ValidationResultBean create(String serverUrl, MJob job) {
+ JobBean jobBean = new JobBean(job);
+
+ // Extract all form inputs including sensitive inputs
+ JSONObject jobJson = jobBean.extract(false);
+
+ String response = super.post(serverUrl + RESOURCE, jobJson.toJSONString());
+
+ ValidationResultBean validationBean = new ValidationResultBean();
+ validationBean.restore((JSONObject) JSONValue.parse(response));
+
+ return validationBean;
+ }
+
+ public ValidationResultBean update(String serverUrl, MJob job) {
+ JobBean jobBean = new JobBean(job);
+
+ // Extract all form inputs including sensitive inputs
+ JSONObject jobJson = jobBean.extract(false);
+
+ String response = super.put(serverUrl + RESOURCE + job.getPersistenceId(), jobJson.toJSONString());
+
+ ValidationResultBean validationBean = new ValidationResultBean();
+ validationBean.restore((JSONObject) JSONValue.parse(response));
+
+ return validationBean;
+ }
+
+ public void delete(String serverUrl, Long id) {
+ super.delete(serverUrl + RESOURCE + id);
+ }
+
+ public void enable(String serverUrl, Long id, Boolean enabled) {
+ if (enabled) {
+ super.put(serverUrl + RESOURCE + id + ENABLE, null);
+ } else {
+ super.put(serverUrl + RESOURCE + id + DISABLE, null);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java
new file mode 100644
index 0000000..2b784c8
--- /dev/null
+++ b/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client.request;
+
+import org.apache.sqoop.json.LinkBean;
+import org.apache.sqoop.json.ValidationResultBean;
+import org.apache.sqoop.model.MLink;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+/**
+ * Provide CRUD semantics over RESTfull HTTP API for links. All operations
+ * are normally supported.
+ */
+public class LinkResourceRequest extends ResourceRequest {
+
+ public static final String RESOURCE = "v1/link/";
+
+ private static final String ENABLE = "/enable";
+ private static final String DISABLE = "/disable";
+
+ public LinkBean read(String serverUrl, Long xid) {
+ String response;
+ if (xid == null) {
+ response = super.get(serverUrl + RESOURCE + "all");
+ } else {
+ response = super.get(serverUrl + RESOURCE + xid);
+ }
+ JSONObject jsonObject = (JSONObject)JSONValue.parse(response);
+ LinkBean linkBean = new LinkBean();
+ linkBean.restore(jsonObject);
+ return linkBean;
+ }
+
+ public ValidationResultBean create(String serverUrl, MLink link) {
+ LinkBean linkBean = new LinkBean(link);
+
+ // Extract all form inputs including sensitive inputs
+ JSONObject linkJson = linkBean.extract(false);
+
+ String response = super.post(serverUrl + RESOURCE, linkJson.toJSONString());
+
+ ValidationResultBean validationBean = new ValidationResultBean();
+ validationBean.restore((JSONObject) JSONValue.parse(response));
+
+ return validationBean;
+ }
+
+ public ValidationResultBean update(String serverUrl, MLink link) {
+ LinkBean linkBean = new LinkBean(link);
+
+ // Extract all form inputs including sensitive inputs
+ JSONObject linkJson = linkBean.extract(false);
+
+ String response = super.put(serverUrl + RESOURCE + link.getPersistenceId(), linkJson.toJSONString());
+
+ ValidationResultBean validationBean = new ValidationResultBean();
+ validationBean.restore((JSONObject) JSONValue.parse(response));
+
+ return validationBean;
+ }
+
+ public void delete(String serverUrl, Long id) {
+ super.delete(serverUrl + RESOURCE + id);
+ }
+
+ public void enable(String serverUrl, Long id, Boolean enabled) {
+ if (enabled) {
+ super.put(serverUrl + RESOURCE + id + ENABLE, null);
+ } else {
+ super.put(serverUrl + RESOURCE + id + DISABLE, null);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/Request.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/Request.java b/client/src/main/java/org/apache/sqoop/client/request/Request.java
deleted file mode 100644
index d9229f5..0000000
--- a/client/src/main/java/org/apache/sqoop/client/request/Request.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.client.request;
-
-import javax.ws.rs.core.MediaType;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientRequest;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.WebResource.Builder;
-import com.sun.jersey.api.client.filter.ClientFilter;
-
-import org.apache.sqoop.client.ClientError;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.common.SqoopProtocolConstants;
-import org.apache.sqoop.json.ThrowableBean;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-import java.util.Locale;
-
-public class Request
-{
- private static ServerExceptionFilter serverExceptionFilter;
-
- static {
- serverExceptionFilter = new ServerExceptionFilter();
- }
-
- protected Builder getBuilder(String url) {
- Client client = Client.create();
- WebResource resource = client.resource(url);
-
- // Provide filter that will rebuild exception that is sent from server
- resource.addFilter(serverExceptionFilter);
-
- return resource
- // Provide name of user executing request.
- .header(SqoopProtocolConstants.HEADER_SQOOP_USERNAME, System.getProperty("user.name"))
- // Sqoop is using JSON for data transfers
- .accept(MediaType.APPLICATION_JSON_TYPE)
- // Transfer client locale to return client specific data
- .acceptLanguage(Locale.getDefault());
- }
-
- public String get(String url) {
- return getBuilder(url).get(String.class);
- }
-
- public String post(String url, String data) {
- return getBuilder(url).post(String.class, data);
- }
-
- public String put(String url, String data) {
- return getBuilder(url).put(String.class, data);
- }
-
- public String delete(String url) {
- return getBuilder(url).delete(String.class);
- }
-
- /**
- * Client filter to intercepting exceptions sent by sqoop server and
- * recreating them on client side. Current implementation will create new
- * instance of SqoopException and will attach original error code and message.
- */
- private static class ServerExceptionFilter extends ClientFilter {
- @Override
- public ClientResponse handle(ClientRequest cr) {
- ClientResponse resp = getNext().handle(cr);
-
- // Special handling for 500 internal server error in case that server
- // has sent us it's exception correctly. We're using default route
- // for all other 500 occurrences.
- if(resp.getClientResponseStatus()
- == ClientResponse.Status.INTERNAL_SERVER_ERROR) {
-
- if(resp.getHeaders().containsKey(
- SqoopProtocolConstants.HEADER_SQOOP_INTERNAL_ERROR_CODE)) {
-
- ThrowableBean ex = new ThrowableBean();
-
- String responseText = resp.getEntity(String.class);
- JSONObject json = (JSONObject) JSONValue.parse(responseText);
- ex.restore(json);
-
- throw new SqoopException(ClientError.CLIENT_0001, ex.getThrowable());
- }
- }
-
- return resp;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
new file mode 100644
index 0000000..c84a83e
--- /dev/null
+++ b/client/src/main/java/org/apache/sqoop/client/request/ResourceRequest.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client.request;
+
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientRequest;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.api.client.filter.ClientFilter;
+
+import org.apache.sqoop.client.ClientError;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.common.SqoopProtocolConstants;
+import org.apache.sqoop.json.ThrowableBean;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+import java.util.Locale;
+/**
+ * Represents the sqoop REST resource requests
+ *
+ */
+public class ResourceRequest
+{
+ private static ServerExceptionFilter serverExceptionFilter;
+
+ static {
+ serverExceptionFilter = new ServerExceptionFilter();
+ }
+
+ protected Builder getBuilder(String url) {
+ Client client = Client.create();
+ WebResource resource = client.resource(url);
+
+ // Provide filter that will rebuild exception that is sent from server
+ resource.addFilter(serverExceptionFilter);
+
+ return resource
+ // Provide name of user executing request.
+ .header(SqoopProtocolConstants.HEADER_SQOOP_USERNAME, System.getProperty("user.name"))
+ // Sqoop is using JSON for data transfers
+ .accept(MediaType.APPLICATION_JSON_TYPE)
+ // Transfer client locale to return client specific data
+ .acceptLanguage(Locale.getDefault());
+ }
+
+ public String get(String url) {
+ return getBuilder(url).get(String.class);
+ }
+
+ public String post(String url, String data) {
+ return getBuilder(url).post(String.class, data);
+ }
+
+ public String put(String url, String data) {
+ return getBuilder(url).put(String.class, data);
+ }
+
+ public String delete(String url) {
+ return getBuilder(url).delete(String.class);
+ }
+
+ /**
+ * Client filter to intercepting exceptions sent by sqoop server and
+ * recreating them on client side. Current implementation will create new
+ * instance of SqoopException and will attach original error code and message.
+ */
+ private static class ServerExceptionFilter extends ClientFilter {
+ @Override
+ public ClientResponse handle(ClientRequest cr) {
+ ClientResponse resp = getNext().handle(cr);
+
+ // Special handling for 500 internal server error in case that server
+ // has sent us it's exception correctly. We're using default route
+ // for all other 500 occurrences.
+ if(resp.getClientResponseStatus()
+ == ClientResponse.Status.INTERNAL_SERVER_ERROR) {
+
+ if(resp.getHeaders().containsKey(
+ SqoopProtocolConstants.HEADER_SQOOP_INTERNAL_ERROR_CODE)) {
+
+ ThrowableBean ex = new ThrowableBean();
+
+ String responseText = resp.getEntity(String.class);
+ JSONObject json = (JSONObject) JSONValue.parse(responseText);
+ ex.restore(json);
+
+ throw new SqoopException(ClientError.CLIENT_0001, ex.getThrowable());
+ }
+ }
+
+ return resp;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/SqoopRequests.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/SqoopRequests.java b/client/src/main/java/org/apache/sqoop/client/request/SqoopRequests.java
deleted file mode 100644
index 0d13304..0000000
--- a/client/src/main/java/org/apache/sqoop/client/request/SqoopRequests.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.client.request;
-
-import org.apache.sqoop.json.ConnectionBean;
-import org.apache.sqoop.json.ConnectorBean;
-import org.apache.sqoop.json.FrameworkBean;
-import org.apache.sqoop.json.JobBean;
-import org.apache.sqoop.json.SubmissionBean;
-import org.apache.sqoop.json.ValidationResultBean;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MJob;
-
-/**
- * Unified class for all request objects.
- */
-public class SqoopRequests {
-
- private String serverUrl;
-
- private FrameworkRequest frameworkRequest;
- private ConnectorRequest connectorRequest;
- private ConnectionRequest connectionRequest;
- private JobRequest jobRequest;
- private SubmissionRequest submissionRequest;
-
- public void setServerUrl(String serverUrl) {
- this.serverUrl = serverUrl;
- }
-
- public FrameworkRequest getFrameworkRequest() {
- if (frameworkRequest == null) {
- frameworkRequest = new FrameworkRequest();
- }
-
- return frameworkRequest;
- }
-
- public ConnectorRequest getConnectorRequest() {
- if (connectorRequest == null) {
- connectorRequest = new ConnectorRequest();
- }
-
- return connectorRequest;
- }
-
- public ConnectionRequest getConnectionRequest() {
- if (connectionRequest == null) {
- connectionRequest = new ConnectionRequest();
- }
-
- return connectionRequest;
- }
-
- public JobRequest getJobRequest() {
- if (jobRequest == null) {
- jobRequest = new JobRequest();
- }
-
- return jobRequest;
- }
-
- public SubmissionRequest getSubmissionRequest() {
- if (submissionRequest == null) {
- submissionRequest = new SubmissionRequest();
- }
-
- return submissionRequest;
- }
-
- public FrameworkBean readFramework() {
- return getFrameworkRequest().read(serverUrl);
- }
-
- public ConnectorBean readConnector(Long cid) {
- return getConnectorRequest().read(serverUrl, cid);
- }
-
- public ValidationResultBean createConnection(MConnection connection) {
- return getConnectionRequest().create(serverUrl, connection);
- }
-
- public ConnectionBean readConnection(Long connectionId) {
- return getConnectionRequest().read(serverUrl, connectionId);
- }
-
- public ValidationResultBean updateConnection(MConnection connection) {
- return getConnectionRequest().update(serverUrl, connection);
- }
-
- public void enableConnection(Long xid, Boolean enabled) {
- getConnectionRequest().enable(serverUrl, xid, enabled);
- }
-
- public void deleteConnection(Long xid) {
- getConnectionRequest().delete(serverUrl, xid);
- }
-
- public ValidationResultBean createJob(MJob job) {
- return getJobRequest().create(serverUrl, job);
- }
-
- public JobBean readJob(Long jobId) {
- return getJobRequest().read(serverUrl, jobId);
- }
-
- public ValidationResultBean updateJob(MJob job) {
- return getJobRequest().update(serverUrl, job);
- }
-
- public void enableJob(Long jid, Boolean enabled) {
- getJobRequest().enable(serverUrl, jid, enabled);
- }
-
- public void deleteJob(Long jid) {
- getJobRequest().delete(serverUrl, jid);
- }
-
- public SubmissionBean readHistory(Long jid) {
- return getSubmissionRequest().readHistory(serverUrl, jid);
- }
-
- public SubmissionBean readSubmission(Long jid) {
- return getSubmissionRequest().read(serverUrl, jid);
- }
-
- public SubmissionBean createSubmission(Long jid) {
- return getSubmissionRequest().create(serverUrl, jid);
- }
-
- public SubmissionBean deleteSubmission(Long jid) {
- return getSubmissionRequest().delete(serverUrl, jid);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java b/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
new file mode 100644
index 0000000..5401eca
--- /dev/null
+++ b/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client.request;
+
+import org.apache.sqoop.json.LinkBean;
+import org.apache.sqoop.json.ConnectorBean;
+import org.apache.sqoop.json.DriverConfigBean;
+import org.apache.sqoop.json.JobBean;
+import org.apache.sqoop.json.SubmissionBean;
+import org.apache.sqoop.json.ValidationResultBean;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MJob;
+
+/**
+ * Unified class for all request objects.
+ */
+public class SqoopResourceRequests {
+
+ private String serverUrl;
+
+ private DriverConfigResourceRequest driverConfigRequest;
+ private ConnectorResourceRequest connectorRequest;
+ private LinkResourceRequest linkRequest;
+ private JobResourceRequest jobRequest;
+ private SubmissionResourceRequest submissionRequest;
+
+ public void setServerUrl(String serverUrl) {
+ this.serverUrl = serverUrl;
+ }
+
+ public DriverConfigResourceRequest getDriverConfigResourceRequest() {
+ if (driverConfigRequest == null) {
+ driverConfigRequest = new DriverConfigResourceRequest();
+ }
+
+ return driverConfigRequest;
+ }
+
+ public ConnectorResourceRequest getConnectorResourceRequest() {
+ if (connectorRequest == null) {
+ connectorRequest = new ConnectorResourceRequest();
+ }
+
+ return connectorRequest;
+ }
+
+ public LinkResourceRequest getLinkResourceRequest() {
+ if (linkRequest == null) {
+ linkRequest = new LinkResourceRequest();
+ }
+
+ return linkRequest;
+ }
+
+ public JobResourceRequest getJobResourceRequest() {
+ if (jobRequest == null) {
+ jobRequest = new JobResourceRequest();
+ }
+
+ return jobRequest;
+ }
+
+ public SubmissionResourceRequest getSubmissionResourceRequest() {
+ if (submissionRequest == null) {
+ submissionRequest = new SubmissionResourceRequest();
+ }
+
+ return submissionRequest;
+ }
+
+ public DriverConfigBean readDriverConfig() {
+ return getDriverConfigResourceRequest().read(serverUrl);
+ }
+
+ public ConnectorBean readConnector(Long cid) {
+ return getConnectorResourceRequest().read(serverUrl, cid);
+ }
+
+ public ValidationResultBean saveLink(MLink link) {
+ return getLinkResourceRequest().create(serverUrl, link);
+ }
+
+ public LinkBean readLink(Long linkId) {
+ return getLinkResourceRequest().read(serverUrl, linkId);
+ }
+
+ public ValidationResultBean updateLink(MLink link) {
+ return getLinkResourceRequest().update(serverUrl, link);
+ }
+
+ public void enableLink(Long lid, Boolean enabled) {
+ getLinkResourceRequest().enable(serverUrl, lid, enabled);
+ }
+
+ public void deleteLink(Long lid) {
+ getLinkResourceRequest().delete(serverUrl, lid);
+ }
+
+ public ValidationResultBean saveJob(MJob job) {
+ return getJobResourceRequest().create(serverUrl, job);
+ }
+
+ public JobBean readJob(Long jobId) {
+ return getJobResourceRequest().read(serverUrl, jobId);
+ }
+
+ public ValidationResultBean updateJob(MJob job) {
+ return getJobResourceRequest().update(serverUrl, job);
+ }
+
+ public void enableJob(Long jid, Boolean enabled) {
+ getJobResourceRequest().enable(serverUrl, jid, enabled);
+ }
+
+ public void deleteJob(Long jid) {
+ getJobResourceRequest().delete(serverUrl, jid);
+ }
+
+ public SubmissionBean readHistory(Long jid) {
+ return getSubmissionResourceRequest().readHistory(serverUrl, jid);
+ }
+
+ public SubmissionBean readSubmission(Long jid) {
+ return getSubmissionResourceRequest().read(serverUrl, jid);
+ }
+
+ public SubmissionBean createSubmission(Long jid) {
+ return getSubmissionResourceRequest().create(serverUrl, jid);
+ }
+
+ public SubmissionBean deleteSubmission(Long jid) {
+ return getSubmissionResourceRequest().delete(serverUrl, jid);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/SubmissionRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/SubmissionRequest.java b/client/src/main/java/org/apache/sqoop/client/request/SubmissionRequest.java
deleted file mode 100644
index bafb30f..0000000
--- a/client/src/main/java/org/apache/sqoop/client/request/SubmissionRequest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.client.request;
-
-import org.apache.sqoop.json.SubmissionBean;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-/**
- * Provide CRD semantics over RESTfull HTTP API for submissions. Please note
- * that "update" is not supported as client can't update submission status.
- */
-public class SubmissionRequest extends Request {
-
- public static final String RESOURCE = "v1/submission/";
-
- public static final String ACTION = RESOURCE + "action/";
-
- public static final String HISTORY = RESOURCE + "history/";
-
- public SubmissionBean readHistory(String serverUrl, Long jid) {
- String response;
- if (jid == null) {
- response = super.get(serverUrl + HISTORY + "all");
- } else {
- response = super.get(serverUrl + HISTORY + jid);
- }
-
- JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
-
- SubmissionBean submissionBean = new SubmissionBean();
- submissionBean.restore(jsonObject);
-
- return submissionBean;
- }
-
- public SubmissionBean read(String serverUrl, Long jid) {
- String response = super.get(serverUrl + ACTION + jid);
-
- JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
-
- SubmissionBean submissionBean = new SubmissionBean();
- submissionBean.restore(jsonObject);
-
- return submissionBean;
- }
-
- public SubmissionBean create(String serverUrl, Long jid) {
- String response = super.post(serverUrl + ACTION + jid, null);
-
- SubmissionBean submissionBean = new SubmissionBean();
- submissionBean.restore((JSONObject) JSONValue.parse(response));
-
- return submissionBean;
- }
-
- public SubmissionBean delete(String serverUrl, Long id) {
- String response = super.delete(serverUrl + ACTION + id);
-
- SubmissionBean submissionBean = new SubmissionBean();
- submissionBean.restore((JSONObject) JSONValue.parse(response));
-
- return submissionBean;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java
new file mode 100644
index 0000000..5055783
--- /dev/null
+++ b/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client.request;
+
+import org.apache.sqoop.json.SubmissionBean;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+/**
+ * Provide CRD semantics over RESTfull HTTP API for submissions. Please note
+ * that "update" is not supported as client can't update submission status.
+ */
+public class SubmissionResourceRequest extends ResourceRequest {
+
+ public static final String RESOURCE = "v1/submission/";
+
+ public static final String ACTION = RESOURCE + "action/";
+
+ public static final String HISTORY = RESOURCE + "history/";
+
+ public SubmissionBean readHistory(String serverUrl, Long jid) {
+ String response;
+ if (jid == null) {
+ response = super.get(serverUrl + HISTORY + "all");
+ } else {
+ response = super.get(serverUrl + HISTORY + jid);
+ }
+
+ JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
+
+ SubmissionBean submissionBean = new SubmissionBean();
+ submissionBean.restore(jsonObject);
+
+ return submissionBean;
+ }
+
+ public SubmissionBean read(String serverUrl, Long jid) {
+ String response = super.get(serverUrl + ACTION + jid);
+
+ JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
+
+ SubmissionBean submissionBean = new SubmissionBean();
+ submissionBean.restore(jsonObject);
+
+ return submissionBean;
+ }
+
+ public SubmissionBean create(String serverUrl, Long jid) {
+ String response = super.post(serverUrl + ACTION + jid, null);
+
+ SubmissionBean submissionBean = new SubmissionBean();
+ submissionBean.restore((JSONObject) JSONValue.parse(response));
+
+ return submissionBean;
+ }
+
+ public SubmissionBean delete(String serverUrl, Long id) {
+ String response = super.delete(serverUrl + ACTION + id);
+
+ SubmissionBean submissionBean = new SubmissionBean();
+ submissionBean.restore((JSONObject) JSONValue.parse(response));
+
+ return submissionBean;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/VersionRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/VersionRequest.java b/client/src/main/java/org/apache/sqoop/client/request/VersionRequest.java
deleted file mode 100644
index 2b236ba..0000000
--- a/client/src/main/java/org/apache/sqoop/client/request/VersionRequest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.client.request;
-
-import org.apache.sqoop.json.VersionBean;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-public class VersionRequest extends Request
-{
- public VersionBean doGet(String serverUrl) {
- String response = super.get(serverUrl + "version");
- JSONObject jsonObject = (JSONObject)JSONValue.parse(response);
-
- VersionBean versionBean = new VersionBean();
- versionBean.restore(jsonObject);
-
- return versionBean;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/main/java/org/apache/sqoop/client/request/VersionResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/VersionResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/VersionResourceRequest.java
new file mode 100644
index 0000000..cdd2a9b
--- /dev/null
+++ b/client/src/main/java/org/apache/sqoop/client/request/VersionResourceRequest.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client.request;
+
+import org.apache.sqoop.json.VersionBean;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+public class VersionResourceRequest extends ResourceRequest
+{
+ public VersionBean read(String serverUrl) {
+ String response = super.get(serverUrl + "version");
+ JSONObject jsonObject = (JSONObject)JSONValue.parse(response);
+
+ VersionBean versionBean = new VersionBean();
+ versionBean.restore(jsonObject);
+
+ return versionBean;
+ }
+}
[48/52] [abbrv] git commit: SQOOP-1498: Sqoop2: Repository Object
refactoring (objects prefixed with M)
Posted by ab...@apache.org.
SQOOP-1498: Sqoop2: Repository Object refactoring (objects prefixed with M)
(Veena Basavaraj via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/8362c73c
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/8362c73c
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/8362c73c
Branch: refs/heads/SQOOP-1367
Commit: 8362c73cc01313d2510a659bf296efd727bb1d4e
Parents: e540668
Author: Abraham Elmahrek <ab...@elmahrek.com>
Authored: Thu Oct 2 23:29:50 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 18:22:51 2014 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/client/SqoopClient.java | 119 +--
.../request/ConnectorResourceRequest.java | 4 +-
.../request/DriverConfigResourceRequest.java | 15 +-
.../client/request/JobResourceRequest.java | 26 +-
.../client/request/LinkResourceRequest.java | 10 +-
.../client/request/SqoopResourceRequests.java | 4 +-
.../apache/sqoop/client/TestSqoopClient.java | 73 +-
.../org/apache/sqoop/json/ConnectorBean.java | 129 +--
.../java/org/apache/sqoop/json/DriverBean.java | 90 ++
.../org/apache/sqoop/json/DriverConfigBean.java | 92 --
.../java/org/apache/sqoop/json/JobBean.java | 77 +-
.../apache/sqoop/json/JobValidationBean.java | 83 +-
.../java/org/apache/sqoop/json/LinkBean.java | 96 +-
.../apache/sqoop/json/LinkValidationBean.java | 56 +-
.../org/apache/sqoop/json/ThrowableBean.java | 2 +-
.../apache/sqoop/json/ValidationResultBean.java | 18 +-
.../sqoop/json/util/ConfigSerialization.java | 226 +++++
.../sqoop/json/util/FormSerialization.java | 223 -----
.../json/util/ResourceBundleSerialization.java | 2 -
.../java/org/apache/sqoop/model/Config.java | 34 +
.../org/apache/sqoop/model/ConfigClass.java | 45 +
.../org/apache/sqoop/model/ConfigUtils.java | 637 +++++++++++++
.../apache/sqoop/model/ConfigurationClass.java | 7 +-
.../main/java/org/apache/sqoop/model/Form.java | 35 -
.../java/org/apache/sqoop/model/FormClass.java | 45 -
.../java/org/apache/sqoop/model/FormUtils.java | 634 -------------
.../java/org/apache/sqoop/model/MConfig.java | 117 +++
.../org/apache/sqoop/model/MConfigList.java | 124 +++
.../org/apache/sqoop/model/MConfigType.java | 34 +
.../apache/sqoop/model/MConnectionForms.java | 54 --
.../java/org/apache/sqoop/model/MConnector.java | 124 ++-
.../java/org/apache/sqoop/model/MDriver.java | 82 ++
.../org/apache/sqoop/model/MDriverConfig.java | 65 +-
.../main/java/org/apache/sqoop/model/MForm.java | 117 ---
.../java/org/apache/sqoop/model/MFormList.java | 124 ---
.../java/org/apache/sqoop/model/MFormType.java | 34 -
.../org/apache/sqoop/model/MFromConfig.java | 64 ++
.../main/java/org/apache/sqoop/model/MJob.java | 102 +-
.../java/org/apache/sqoop/model/MJobForms.java | 55 --
.../main/java/org/apache/sqoop/model/MLink.java | 72 +-
.../org/apache/sqoop/model/MLinkConfig.java | 54 ++
.../java/org/apache/sqoop/model/MToConfig.java | 64 ++
.../java/org/apache/sqoop/model/ModelError.java | 6 +-
.../java/org/apache/sqoop/utils/ClassUtils.java | 6 +-
.../sqoop/validation/ConfigValidationError.java | 52 +
.../validation/ConfigValidationResult.java | 98 ++
.../validation/ConfigValidationRunner.java | 177 ++++
.../sqoop/validation/ConfigValidator.java | 228 +++++
.../org/apache/sqoop/validation/Message.java | 2 +-
.../org/apache/sqoop/validation/Validation.java | 228 -----
.../sqoop/validation/ValidationError.java | 52 -
.../sqoop/validation/ValidationResult.java | 98 --
.../sqoop/validation/ValidationRunner.java | 176 ----
.../org/apache/sqoop/json/ConfigTestUtil.java | 217 +++++
.../apache/sqoop/json/TestConnectorBean.java | 37 +-
.../org/apache/sqoop/json/TestDriverBean.java | 62 ++
.../apache/sqoop/json/TestDriverConfigBean.java | 62 --
.../java/org/apache/sqoop/json/TestJobBean.java | 40 +-
.../org/apache/sqoop/json/TestLinkBean.java | 56 +-
.../java/org/apache/sqoop/json/TestUtil.java | 153 ---
.../apache/sqoop/json/TestValidationBean.java | 83 +-
.../sqoop/json/TestValidationResultBean.java | 24 +-
.../json/util/TestConfigSerialization.java | 157 ++++
.../sqoop/json/util/TestFormSerialization.java | 157 ----
.../org/apache/sqoop/model/TestConfigUtils.java | 290 ++++++
.../org/apache/sqoop/model/TestFormUtils.java | 295 ------
.../sqoop/model/TestMAccountableEntity.java | 9 +-
.../org/apache/sqoop/model/TestMConfig.java | 86 ++
.../org/apache/sqoop/model/TestMConfigList.java | 55 ++
.../sqoop/model/TestMConnectionForms.java | 48 -
.../org/apache/sqoop/model/TestMConnector.java | 144 +--
.../org/apache/sqoop/model/TestMDriver.java | 40 +
.../apache/sqoop/model/TestMDriverConfig.java | 47 -
.../java/org/apache/sqoop/model/TestMForm.java | 89 --
.../org/apache/sqoop/model/TestMFormList.java | 58 --
.../java/org/apache/sqoop/model/TestMJob.java | 110 ++-
.../org/apache/sqoop/model/TestMJobConfig.java | 42 +
.../org/apache/sqoop/model/TestMJobForms.java | 46 -
.../java/org/apache/sqoop/model/TestMLink.java | 74 +-
.../org/apache/sqoop/model/TestMLinkConfig.java | 45 +
.../apache/sqoop/validation/TestValidation.java | 70 +-
.../sqoop/validation/TestValidationRunner.java | 98 +-
.../validators/TestClassAvailable.java | 12 +-
.../connector/jdbc/GenericJdbcConnector.java | 2 +-
.../jdbc/GenericJdbcConnectorUpgrader.java | 54 +-
.../connector/jdbc/GenericJdbcExtractor.java | 12 +-
.../jdbc/GenericJdbcFromDestroyer.java | 2 +-
.../jdbc/GenericJdbcFromInitializer.java | 36 +-
.../sqoop/connector/jdbc/GenericJdbcLoader.java | 10 +-
.../connector/jdbc/GenericJdbcPartitioner.java | 5 +-
.../connector/jdbc/GenericJdbcToDestroyer.java | 18 +-
.../jdbc/GenericJdbcToInitializer.java | 32 +-
.../connector/jdbc/GenericJdbcValidator.java | 30 +-
.../jdbc/configuration/FromJobConfig.java | 14 +-
.../configuration/FromJobConfiguration.java | 4 +-
.../jdbc/configuration/LinkConfig.java | 6 +-
.../jdbc/configuration/LinkConfiguration.java | 6 +-
.../jdbc/configuration/ToJobConfig.java | 16 +-
.../jdbc/configuration/ToJobConfiguration.java | 4 +-
.../generic-jdbc-connector-config.properties | 24 +-
.../connector/jdbc/GenericJdbcExecutorTest.java | 3 -
.../sqoop/connector/jdbc/TestExtractor.java | 24 +-
.../connector/jdbc/TestFromInitializer.java | 158 ++--
.../apache/sqoop/connector/jdbc/TestLoader.java | 8 +-
.../sqoop/connector/jdbc/TestPartitioner.java | 98 +-
.../sqoop/connector/jdbc/TestToInitializer.java | 188 ++--
.../connector/hdfs/HdfsConfigUpgrader.java | 50 +-
.../sqoop/connector/hdfs/HdfsConnector.java | 2 +-
.../sqoop/connector/hdfs/HdfsExtractor.java | 4 +-
.../sqoop/connector/hdfs/HdfsInitializer.java | 6 +-
.../apache/sqoop/connector/hdfs/HdfsLoader.java | 14 +-
.../sqoop/connector/hdfs/HdfsPartitioner.java | 6 +-
.../sqoop/connector/hdfs/HdfsValidator.java | 49 +-
.../hdfs/configuration/FromJobConfig.java | 4 +-
.../configuration/FromJobConfiguration.java | 4 +-
.../hdfs/configuration/LinkConfig.java | 6 +-
.../hdfs/configuration/LinkConfiguration.java | 8 +-
.../hdfs/configuration/ToJobConfig.java | 4 +-
.../hdfs/configuration/ToJobConfiguration.java | 4 +-
.../resources/hdfs-connector-config.properties | 8 +-
.../idf/TestCSVIntermediateDataFormat.java | 14 +-
.../sqoop/connector/ConnectorHandler.java | 28 +-
.../sqoop/connector/ConnectorManager.java | 2 +-
.../java/org/apache/sqoop/driver/Driver.java | 47 +-
.../sqoop/driver/DriverConfigUpgrader.java | 48 +-
.../sqoop/driver/DriverConfigValidator.java | 46 +
.../apache/sqoop/driver/DriverValidator.java | 54 --
.../org/apache/sqoop/driver/JobManager.java | 72 +-
.../org/apache/sqoop/driver/JobRequest.java | 63 +-
.../configuration/DriverConfiguration.java | 34 +
.../driver/configuration/JobConfiguration.java | 34 -
.../driver/configuration/LinkConfiguration.java | 28 -
.../driver/configuration/ThrottlingConfig.java | 32 +
.../driver/configuration/ThrottlingForm.java | 32 -
.../apache/sqoop/repository/JdbcRepository.java | 46 +-
.../sqoop/repository/JdbcRepositoryHandler.java | 14 +-
.../org/apache/sqoop/repository/Repository.java | 254 +++--
.../sqoop/repository/RepositoryManager.java | 2 +-
.../src/main/resources/driver-config.properties | 12 +-
.../sqoop/driver/TestDriverConfigUpgrader.java | 88 +-
.../org/apache/sqoop/driver/TestJobManager.java | 18 +-
.../sqoop/repository/TestJdbcRepository.java | 495 +++-------
.../main/java/org/apache/sqoop/job/io/Data.java | 2 +-
.../apache/sqoop/job/mr/ConfigurationUtils.java | 124 +--
.../sqoop/job/mr/SqoopDestroyerExecutor.java | 2 +-
.../java/org/apache/sqoop/job/JobUtils.java | 3 +-
.../org/apache/sqoop/job/TestMapReduce.java | 1 -
.../sqoop/job/mr/TestConfigurationUtils.java | 118 ++-
.../sqoop/repository/derby/DerbyRepoError.java | 2 +-
.../derby/DerbyRepositoryHandler.java | 777 +++++++--------
.../repository/derby/DerbySchemaConstants.java | 108 +--
.../repository/derby/DerbySchemaQuery.java | 640 ++++++-------
.../sqoop/repository/derby/DerbyTestCase.java | 287 +++---
.../repository/derby/TestConnectorHandling.java | 6 +-
.../derby/TestDriverConfigHandling.java | 135 ---
.../repository/derby/TestDriverHandling.java | 135 +++
.../sqoop/repository/derby/TestInputTypes.java | 22 +-
.../sqoop/repository/derby/TestJobHandling.java | 216 ++---
.../repository/derby/TestLinkHandling.java | 118 +--
.../derby/TestSubmissionHandling.java | 2 +-
.../sqoop/handler/ConnectorRequestHandler.java | 2 +-
.../handler/DriverConfigRequestHandler.java | 11 +-
.../apache/sqoop/handler/JobRequestHandler.java | 95 +-
.../sqoop/handler/LinkRequestHandler.java | 158 ++--
.../apache/sqoop/shell/CloneJobFunction.java | 18 +-
.../apache/sqoop/shell/CloneLinkFunction.java | 17 +-
.../apache/sqoop/shell/CreateJobFunction.java | 26 +-
.../apache/sqoop/shell/CreateLinkFunction.java | 22 +-
.../org/apache/sqoop/shell/ShowCommand.java | 2 +-
.../sqoop/shell/ShowConnectorFunction.java | 4 +-
.../sqoop/shell/ShowDriverConfigFunction.java | 60 --
.../apache/sqoop/shell/ShowDriverFunction.java | 55 ++
.../org/apache/sqoop/shell/ShowJobFunction.java | 13 +-
.../apache/sqoop/shell/ShowLinkFunction.java | 28 +-
.../org/apache/sqoop/shell/SqoopFunction.java | 4 +-
.../apache/sqoop/shell/UpdateJobFunction.java | 25 +-
.../apache/sqoop/shell/UpdateLinkFunction.java | 23 +-
.../org/apache/sqoop/shell/core/Constants.java | 64 +-
.../sqoop/shell/utils/ConfigDisplayer.java | 258 +++++
.../apache/sqoop/shell/utils/ConfigFiller.java | 911 ++++++++++++++++++
.../apache/sqoop/shell/utils/ConfigOptions.java | 117 +++
.../sqoop/shell/utils/DynamicConfigOptions.java | 35 +
.../sqoop/shell/utils/DynamicFormOptions.java | 35 -
.../apache/sqoop/shell/utils/FormDisplayer.java | 267 ------
.../apache/sqoop/shell/utils/FormFiller.java | 939 -------------------
.../apache/sqoop/shell/utils/FormOptions.java | 117 ---
.../shell/utils/JobDynamicConfigOptions.java | 48 +
.../shell/utils/JobDynamicFormOptions.java | 48 -
.../shell/utils/LinkDynamicConfigOptions.java | 39 +
.../shell/utils/LinkDynamicFormOptions.java | 43 -
.../main/resources/shell-resource.properties | 34 +-
.../sqoop/connector/spi/RepositoryUpgrader.java | 28 +-
.../sqoop/connector/spi/SqoopConnector.java | 10 +-
.../org/apache/sqoop/job/etl/Destroyer.java | 2 +-
.../org/apache/sqoop/job/etl/Extractor.java | 4 +-
.../org/apache/sqoop/job/etl/Initializer.java | 40 +-
.../java/org/apache/sqoop/job/etl/Loader.java | 10 +-
.../org/apache/sqoop/job/etl/Partitioner.java | 10 +-
.../org/apache/sqoop/validation/Validator.java | 12 +-
.../mapreduce/MapreduceSubmissionEngine.java | 18 +-
.../minicluster/TomcatSqoopMiniCluster.java | 13 +-
.../sqoop/test/testcases/ConnectorTestCase.java | 53 +-
.../sqoop/test/testcases/TomcatTestCase.java | 4 +-
.../jdbc/generic/FromHDFSToRDBMSTest.java | 19 +-
.../jdbc/generic/FromRDBMSToHDFSTest.java | 39 +-
.../connector/jdbc/generic/PartitionerTest.java | 25 +-
.../jdbc/generic/TableStagedRDBMSTest.java | 26 +-
.../SubmissionWithDisabledModelObjectsTest.java | 24 +-
208 files changed, 8198 insertions(+), 8638 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
index d7e6768..bffb599 100644
--- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
+++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
@@ -17,26 +17,27 @@
*/
package org.apache.sqoop.client;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.ResourceBundle;
+
import org.apache.sqoop.client.request.SqoopResourceRequests;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.json.ConnectorBean;
-import org.apache.sqoop.json.DriverConfigBean;
+import org.apache.sqoop.json.DriverBean;
import org.apache.sqoop.json.ValidationResultBean;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.validation.ConfigValidationResult;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.ValidationResult;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.ResourceBundle;
/**
* Sqoop client API.
@@ -69,9 +70,9 @@ public class SqoopClient {
private Map<Long, ResourceBundle> connectorConfigBundles;
/**
- * Cached driverConfig.
+ * Cached driver.
*/
- private MDriverConfig driverConfig;
+ private MDriver mDriver;
/**
* Cached driverConfig bundle.
*/
@@ -120,7 +121,7 @@ public class SqoopClient {
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
driverConfigBundle = null;
connectors = new HashMap<Long, MConnector>();
- driverConfig = null;
+ mDriver = null;
isAllConnectors = false;
}
@@ -214,11 +215,10 @@ public class SqoopClient {
* @param connectorId Connector id.
* @return
*/
- public ResourceBundle getConnectorConfigResourceBundle(long connectorId) {
+ public ResourceBundle getConnectorConfigBundle(long connectorId) {
if(connectorConfigBundles.containsKey(connectorId)) {
return connectorConfigBundles.get(connectorId);
}
-
retrieveConnector(connectorId);
return connectorConfigBundles.get(connectorId);
}
@@ -229,33 +229,46 @@ public class SqoopClient {
* @return
*/
public MDriverConfig getDriverConfig() {
- if(driverConfig != null) {
- return driverConfig.clone(false);
+ if (mDriver != null) {
+ return mDriver.clone(false).getDriverConfig();
}
- retrieveAndCacheDriverConfig();
- return driverConfig.clone(false);
-
+ retrieveAndCacheDriver();
+ return mDriver.clone(false).getDriverConfig();
+ }
+
+ /**
+ * Return driver.
+ *
+ * @return
+ */
+ public MDriver getDriver() {
+ if (mDriver != null) {
+ return mDriver.clone(false);
+ }
+ retrieveAndCacheDriver();
+ return mDriver.clone(false);
+
}
/**
* Retrieve driverConfig and cache it.
*/
- private void retrieveAndCacheDriverConfig() {
- DriverConfigBean driverConfigBean = resourceRequests.readDriverConfig();
- driverConfig = driverConfigBean.getDriverConfig();
- driverConfigBundle = driverConfigBean.getResourceBundle();
+ private void retrieveAndCacheDriver() {
+ DriverBean driverBean = resourceRequests.readDriver();
+ mDriver = driverBean.getDriver();
+ driverConfigBundle = driverBean.getDriverConfigResourceBundle();
}
/**
* Return driverConfig bundle.
- *
+ *xx
* @return
*/
public ResourceBundle getDriverConfigBundle() {
if(driverConfigBundle != null) {
return driverConfigBundle;
}
- retrieveAndCacheDriverConfig();
+ retrieveAndCacheDriver();
return driverConfigBundle;
}
@@ -266,11 +279,7 @@ public class SqoopClient {
* @return
*/
public MLink createLink(long connectorId) {
- return new MLink(
- connectorId,
- getConnector(connectorId).getConnectionForms(),
- getDriverConfig().getConnectionForms()
- );
+ return new MLink(connectorId, getConnector(connectorId).getLinkConfig());
}
/**
@@ -281,10 +290,9 @@ public class SqoopClient {
*/
public MLink createLink(String connectorName) {
MConnector connector = getConnector(connectorName);
- if(connector == null) {
+ if (connector == null) {
throw new SqoopException(ClientError.CLIENT_0003, connectorName);
}
-
return createLink(connector.getPersistenceId());
}
@@ -362,9 +370,9 @@ public class SqoopClient {
toLink.getConnectorId(),
fromLink.getPersistenceId(),
toLink.getPersistenceId(),
- getConnector(fromLink.getConnectorId()).getJobForms(Direction.FROM),
- getConnector(toLink.getConnectorId()).getJobForms(Direction.TO),
- getDriverConfig().getJobForms()
+ getConnector(fromLink.getConnectorId()).getFromConfig(),
+ getConnector(toLink.getConnectorId()).getToConfig(),
+ getDriverConfig()
);
}
@@ -530,41 +538,36 @@ public class SqoopClient {
}
private Status applyLinkValidations(ValidationResultBean bean, MLink link) {
- ValidationResult connector = bean.getValidationResults()[0];
- ValidationResult driverConfig = bean.getValidationResults()[1];
-
+ ConfigValidationResult linkConfig = bean.getValidationResults()[0];
// Apply validation results
- FormUtils.applyValidation(link.getConnectorPart().getForms(), connector);
- FormUtils.applyValidation(link.getFrameworkPart().getForms(), driverConfig);
-
+ ConfigUtils.applyValidation(link.getConnectorLinkConfig().getConfigs(), linkConfig);
Long id = bean.getId();
if(id != null) {
link.setPersistenceId(id);
}
-
- return Status.getWorstStatus(connector.getStatus(), driverConfig.getStatus());
+ return Status.getWorstStatus(linkConfig.getStatus());
}
private Status applyJobValidations(ValidationResultBean bean, MJob job) {
- ValidationResult fromConnector = bean.getValidationResults()[0];
- ValidationResult toConnector = bean.getValidationResults()[1];
- ValidationResult driverConfig = bean.getValidationResults()[2];
-
- // Apply validation results
- // @TODO(Abe): From/To validation.
- FormUtils.applyValidation(
- job.getConnectorPart(Direction.FROM).getForms(),
- fromConnector);
- FormUtils.applyValidation(job.getFrameworkPart().getForms(), driverConfig);
- FormUtils.applyValidation(
- job.getConnectorPart(Direction.TO).getForms(),
- toConnector);
+ ConfigValidationResult fromConfig = bean.getValidationResults()[0];
+ ConfigValidationResult toConfig = bean.getValidationResults()[1];
+ // TODO(VB): fix this as part of SQOOP 1509
+ //ConfigValidationResult driverConfig = bean.getValidationResults()[2];
+
+ ConfigUtils.applyValidation(
+ job.getJobConfig(Direction.FROM).getConfigs(),
+ fromConfig);
+ ConfigUtils.applyValidation(
+ job.getJobConfig(Direction.TO).getConfigs(),
+ toConfig);
+ //ConfigUtils.applyValidation(job.getDriverConfig().getSelf().getConfigs(), driverConfig);
Long id = bean.getId();
if(id != null) {
job.setPersistenceId(id);
}
- return Status.getWorstStatus(fromConnector.getStatus(), driverConfig.getStatus(), toConnector.getStatus());
+ return Status.getWorstStatus(fromConfig.getStatus(), toConfig.getStatus());
+ // driverConfig.getStatus());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
index 2855de6..e2aae98 100644
--- a/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
+++ b/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
@@ -22,7 +22,7 @@ import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
/**
- * Provide cRud semantics over RESTfull HTTP API for connectors. Only read
+ * Provide Read semantics over RESTfull HTTP API for connectors. Only read
* is supported as creation, update and delete might be done only directly on
* server side.
*/
@@ -38,10 +38,8 @@ public class ConnectorResourceRequest extends ResourceRequest
response = super.get(serverUrl + RESOURCE + cid);
}
JSONObject jsonObject = (JSONObject)JSONValue.parse(response);
-
ConnectorBean connectorBean = new ConnectorBean();
connectorBean.restore(jsonObject);
-
return connectorBean;
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java
index 7b2ddc2..f6066fc 100644
--- a/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java
+++ b/client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java
@@ -17,26 +17,23 @@
*/
package org.apache.sqoop.client.request;
-import org.apache.sqoop.json.DriverConfigBean;
+import org.apache.sqoop.json.DriverBean;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
/**
- * Provide cRud semantics over RESTfull HTTP API for driverConfig. Only read
- * is supported as creation, update and delete is not allowed.
+ * Provide CRUD semantics over RESTfull HTTP API for driverConfig
*/
public class DriverConfigResourceRequest extends ResourceRequest {
public static final String RESOURCE = "v1/config/driver";
- public DriverConfigBean read(String serverUrl) {
+ public DriverBean read(String serverUrl) {
String response = null;
response = super.get(serverUrl + RESOURCE);
JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
-
- DriverConfigBean driverConfigBean = new DriverConfigBean();
- driverConfigBean.restore(jsonObject);
-
- return driverConfigBean;
+ DriverBean driverBean = new DriverBean();
+ driverBean.restore(jsonObject);
+ return driverBean;
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
index fd858ec..83c08b3 100644
--- a/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
+++ b/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
@@ -34,46 +34,36 @@ public class JobResourceRequest extends ResourceRequest {
private static final String ENABLE = "/enable";
private static final String DISABLE = "/disable";
- public JobBean read(String serverUrl, Long xid) {
+ public JobBean read(String serverUrl, Long linkId) {
String response;
- if (xid == null) {
+ if (linkId == null) {
response = super.get(serverUrl + RESOURCE + "all");
} else {
- response = super.get(serverUrl + RESOURCE + xid);
+ response = super.get(serverUrl + RESOURCE + linkId);
}
JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
-
JobBean jobBean = new JobBean();
jobBean.restore(jsonObject);
-
return jobBean;
}
public ValidationResultBean create(String serverUrl, MJob job) {
JobBean jobBean = new JobBean(job);
-
- // Extract all form inputs including sensitive inputs
+ // Extract all config inputs including sensitive inputs
JSONObject jobJson = jobBean.extract(false);
-
String response = super.post(serverUrl + RESOURCE, jobJson.toJSONString());
-
- ValidationResultBean validationBean = new ValidationResultBean();
- validationBean.restore((JSONObject) JSONValue.parse(response));
-
- return validationBean;
+ ValidationResultBean validationResultBean = new ValidationResultBean();
+ validationResultBean.restore((JSONObject) JSONValue.parse(response));
+ return validationResultBean;
}
public ValidationResultBean update(String serverUrl, MJob job) {
JobBean jobBean = new JobBean(job);
-
- // Extract all form inputs including sensitive inputs
+ // Extract all config inputs including sensitive inputs
JSONObject jobJson = jobBean.extract(false);
-
String response = super.put(serverUrl + RESOURCE + job.getPersistenceId(), jobJson.toJSONString());
-
ValidationResultBean validationBean = new ValidationResultBean();
validationBean.restore((JSONObject) JSONValue.parse(response));
-
return validationBean;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java
index 2b784c8..64e5cb1 100644
--- a/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java
+++ b/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java
@@ -50,28 +50,22 @@ public class LinkResourceRequest extends ResourceRequest {
public ValidationResultBean create(String serverUrl, MLink link) {
LinkBean linkBean = new LinkBean(link);
- // Extract all form inputs including sensitive inputs
+ // Extract all config inputs including sensitive inputs
JSONObject linkJson = linkBean.extract(false);
-
String response = super.post(serverUrl + RESOURCE, linkJson.toJSONString());
-
ValidationResultBean validationBean = new ValidationResultBean();
validationBean.restore((JSONObject) JSONValue.parse(response));
-
return validationBean;
}
public ValidationResultBean update(String serverUrl, MLink link) {
LinkBean linkBean = new LinkBean(link);
- // Extract all form inputs including sensitive inputs
+ // Extract all config inputs including sensitive inputs
JSONObject linkJson = linkBean.extract(false);
-
String response = super.put(serverUrl + RESOURCE + link.getPersistenceId(), linkJson.toJSONString());
-
ValidationResultBean validationBean = new ValidationResultBean();
validationBean.restore((JSONObject) JSONValue.parse(response));
-
return validationBean;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java b/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
index 5401eca..a4dee75 100644
--- a/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
+++ b/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.client.request;
import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.ConnectorBean;
-import org.apache.sqoop.json.DriverConfigBean;
+import org.apache.sqoop.json.DriverBean;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.SubmissionBean;
import org.apache.sqoop.json.ValidationResultBean;
@@ -83,7 +83,7 @@ public class SqoopResourceRequests {
return submissionRequest;
}
- public DriverConfigBean readDriverConfig() {
+ public DriverBean readDriver() {
return getDriverConfigResourceRequest().read(serverUrl);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
----------------------------------------------------------------------
diff --git a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
index 54ea3d2..18b6132 100644
--- a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
+++ b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
@@ -17,28 +17,36 @@
*/
package org.apache.sqoop.client;
-import org.apache.sqoop.client.request.SqoopResourceRequests;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.json.ConnectorBean;
-import org.apache.sqoop.json.DriverConfigBean;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MJobForms;
-import org.apache.sqoop.utils.MapResourceBundle;
-import org.junit.Before;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.*;
+import org.apache.sqoop.client.request.SqoopResourceRequests;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.json.ConnectorBean;
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriver;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MToConfig;
+import org.apache.sqoop.utils.MapResourceBundle;
+import org.junit.Before;
+import org.junit.Test;
public class TestSqoopClient {
@@ -62,7 +70,7 @@ public class TestSqoopClient {
MConnector connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
- client.getConnectorConfigResourceBundle(1L);
+ client.getConnectorConfigBundle(1L);
verify(resourceRequests, times(1)).readConnector(1L);
}
@@ -74,7 +82,7 @@ public class TestSqoopClient {
assertEquals(1, connector.getPersistenceId());
assertEquals("A1", connector.getUniqueName());
- client.getConnectorConfigResourceBundle(1L);
+ client.getConnectorConfigBundle(1L);
verify(resourceRequests, times(0)).readConnector(1L);
verify(resourceRequests, times(1)).readConnector(null);
@@ -87,7 +95,7 @@ public class TestSqoopClient {
@Test
public void testGetConnectorBundle() {
when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
- client.getConnectorConfigResourceBundle(1L);
+ client.getConnectorConfigBundle(1L);
MConnector connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
@@ -101,12 +109,12 @@ public class TestSqoopClient {
*/
@Test
public void testGetDriverConfig() {
- when(resourceRequests.readDriverConfig()).thenReturn(driverConfigBean(driverConfig()));
+ when(resourceRequests.readDriver()).thenReturn(driverBean(driver()));
client.getDriverConfig();
client.getDriverConfigBundle();
- verify(resourceRequests, times(1)).readDriverConfig();
+ verify(resourceRequests, times(1)).readDriver();
}
/**
@@ -115,12 +123,12 @@ public class TestSqoopClient {
*/
@Test
public void testGetDriverConfigBundle() {
- when(resourceRequests.readDriverConfig()).thenReturn(driverConfigBean(driverConfig()));
+ when(resourceRequests.readDriver()).thenReturn(driverBean(driver()));
client.getDriverConfigBundle();
client.getDriverConfig();
- verify(resourceRequests, times(1)).readDriverConfig();
+ verify(resourceRequests, times(1)).readDriver();
}
/**
@@ -135,12 +143,12 @@ public class TestSqoopClient {
Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size());
- client.getConnectorConfigResourceBundle(1);
+ client.getConnectorConfigBundle(1);
connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
connector = client.getConnector(2);
- client.getConnectorConfigResourceBundle(2);
+ client.getConnectorConfigBundle(2);
assertEquals(2, connector.getPersistenceId());
connectors = client.getConnectors();
@@ -173,11 +181,11 @@ public class TestSqoopClient {
when(resourceRequests.readConnector(1L)).thenReturn(bean);
when(resourceRequests.readConnector(2L)).thenReturn(bean);
- client.getConnectorConfigResourceBundle(1);
+ client.getConnectorConfigBundle(1);
client.getConnector(1);
client.getConnector(2);
- client.getConnectorConfigResourceBundle(2);
+ client.getConnectorConfigBundle(2);
Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size());
@@ -207,21 +215,20 @@ public class TestSqoopClient {
}
return new ConnectorBean(connectorList, bundles);
}
- private DriverConfigBean driverConfigBean(MDriverConfig driverConfig) {
- return new DriverConfigBean(driverConfig, new MapResourceBundle(null));
+ private DriverBean driverBean(MDriver driver) {
+ return new DriverBean(driver, new MapResourceBundle(null));
}
private MConnector connector(long id) {
MConnector connector = new MConnector("A" + id, "A" + id, "1.0" + id,
- new MConnectionForms(null), new MJobForms(null), new MJobForms(null));
+ new MLinkConfig(null), new MFromConfig(null), new MToConfig(null));
connector.setPersistenceId(id);
return connector;
}
- private MDriverConfig driverConfig() {
- MDriverConfig driverConfig = new MDriverConfig(new MConnectionForms(null),
- new MJobForms(null), "1");
- driverConfig.setPersistenceId(1);
- return driverConfig;
+ private MDriver driver() {
+ MDriver driver = new MDriver(new MDriverConfig(new LinkedList<MConfig>()), "1");
+ driver.setPersistenceId(1);
+ return driver;
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
index d726a14..d12b6b0 100644
--- a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
@@ -17,15 +17,15 @@
*/
package org.apache.sqoop.json;
-import static org.apache.sqoop.json.util.FormSerialization.ALL;
-import static org.apache.sqoop.json.util.FormSerialization.CLASS;
-import static org.apache.sqoop.json.util.FormSerialization.CON_FORMS;
-import static org.apache.sqoop.json.util.FormSerialization.ID;
-import static org.apache.sqoop.json.util.FormSerialization.JOB_FORMS;
-import static org.apache.sqoop.json.util.FormSerialization.NAME;
-import static org.apache.sqoop.json.util.FormSerialization.VERSION;
-import static org.apache.sqoop.json.util.FormSerialization.extractForms;
-import static org.apache.sqoop.json.util.FormSerialization.restoreForms;
+import static org.apache.sqoop.json.util.ConfigSerialization.ALL;
+import static org.apache.sqoop.json.util.ConfigSerialization.CLASS;
+import static org.apache.sqoop.json.util.ConfigSerialization.ID;
+import static org.apache.sqoop.json.util.ConfigSerialization.CONNECTOR_JOB_CONFIG;
+import static org.apache.sqoop.json.util.ConfigSerialization.CONNECTOR_LINK_CONFIG;
+import static org.apache.sqoop.json.util.ConfigSerialization.NAME;
+import static org.apache.sqoop.json.util.ConfigSerialization.VERSION;
+import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList;
+import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONNECTOR_CONFIGS;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle;
@@ -38,24 +38,28 @@ import java.util.ResourceBundle;
import java.util.Set;
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MToConfig;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
+/**
+ * Json representation of the connector object
+ *
+ */
public class ConnectorBean implements JsonBean {
private List<MConnector> connectors;
- private Map<Long, ResourceBundle> bundles;
+ private Map<Long, ResourceBundle> connectorConfigBundles;
// for "extract"
- public ConnectorBean(List<MConnector> connectors,
- Map<Long, ResourceBundle> bundles) {
+ public ConnectorBean(List<MConnector> connectors, Map<Long, ResourceBundle> bundles) {
this.connectors = connectors;
- this.bundles = bundles;
+ this.connectorConfigBundles = bundles;
}
// for "restore"
@@ -67,46 +71,46 @@ public class ConnectorBean implements JsonBean {
}
public Map<Long, ResourceBundle> getResourceBundles() {
- return bundles;
+ return connectorConfigBundles;
}
@SuppressWarnings("unchecked")
@Override
public JSONObject extract(boolean skipSensitive) {
- JSONArray array = new JSONArray();
+ JSONArray connectorArray = new JSONArray();
for (MConnector connector : connectors) {
- JSONObject object = new JSONObject();
-
- object.put(ID, connector.getPersistenceId());
- object.put(NAME, connector.getUniqueName());
- object.put(CLASS, connector.getClassName());
- object.put(VERSION, connector.getVersion());
-
- object.put(CON_FORMS, extractForms(connector.getConnectionForms().getForms(), skipSensitive));
- object.put(JOB_FORMS, new JSONObject());
- if (connector.getJobForms(Direction.FROM) != null) {
- ((JSONObject)object.get(JOB_FORMS)).put(
- Direction.FROM, extractForms(connector.getJobForms(Direction.FROM).getForms(), skipSensitive));
+ JSONObject connectorJsonObject = new JSONObject();
+
+ connectorJsonObject.put(ID, connector.getPersistenceId());
+ connectorJsonObject.put(NAME, connector.getUniqueName());
+ connectorJsonObject.put(CLASS, connector.getClassName());
+ connectorJsonObject.put(VERSION, connector.getVersion());
+ connectorJsonObject.put(CONNECTOR_LINK_CONFIG,
+ extractConfigList(connector.getLinkConfig().getConfigs(), skipSensitive));
+
+ connectorJsonObject.put(CONNECTOR_JOB_CONFIG, new JSONObject());
+ // add sub fields to the job config for from and to
+ if (connector.getFromConfig() != null) {
+ ((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(Direction.FROM,
+ extractConfigList(connector.getFromConfig().getConfigs(), skipSensitive));
}
-
- if (connector.getJobForms(Direction.TO) != null) {
- ((JSONObject)object.get(JOB_FORMS)).put(
- Direction.TO, extractForms(connector.getJobForms(Direction.TO).getForms(), skipSensitive));
+ if (connector.getToConfig() != null) {
+ ((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(Direction.TO,
+ extractConfigList(connector.getToConfig().getConfigs(), skipSensitive));
}
- array.add(object);
+ connectorArray.add(connectorJsonObject);
}
JSONObject all = new JSONObject();
- all.put(ALL, array);
+ all.put(ALL, connectorArray);
- if(bundles != null && !bundles.isEmpty()) {
+ if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
JSONObject jsonBundles = new JSONObject();
- for(Map.Entry<Long, ResourceBundle> entry : bundles.entrySet()) {
- jsonBundles.put(entry.getKey().toString(),
- extractResourceBundle(entry.getValue()));
+ for (Map.Entry<Long, ResourceBundle> entry : connectorConfigBundles.entrySet()) {
+ jsonBundles.put(entry.getKey().toString(), extractResourceBundle(entry.getValue()));
}
all.put(CONNECTOR_CONFIGS, jsonBundles);
}
@@ -129,35 +133,42 @@ public class ConnectorBean implements JsonBean {
String className = (String) object.get(CLASS);
String version = (String) object.get(VERSION);
- MJobForms fromJob = null;
- MJobForms toJob = null;
- List<MForm> connForms = restoreForms((JSONArray) object.get(CON_FORMS));
- JSONObject jobJson = (JSONObject) object.get(JOB_FORMS);
- JSONArray fromJobJson = (JSONArray)jobJson.get(Direction.FROM.name());
- JSONArray toJobJson = (JSONArray)jobJson.get(Direction.TO.name());
- if (fromJobJson != null) {
- List<MForm> fromJobForms = restoreForms(fromJobJson);
- fromJob = new MJobForms(fromJobForms);
+ List<MConfig> linkConfigs = restoreConfigList((JSONArray) object.get(CONNECTOR_LINK_CONFIG));
+
+ // parent that encapsualtes both the from/to configs
+ JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG);
+ JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name());
+ JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name());
+
+ MFromConfig fromConfig = null;
+ MToConfig toConfig = null;
+ if (fromJobConfigJson != null) {
+
+ List<MConfig> fromJobConfig = restoreConfigList(fromJobConfigJson);
+ fromConfig = new MFromConfig(fromJobConfig);
+
}
- if (toJobJson != null) {
- List<MForm> toJobForms = restoreForms(toJobJson);
- toJob = new MJobForms(toJobForms);
+ if (toJobConfigJson != null) {
+ List<MConfig> toJobConfig = restoreConfigList(toJobConfigJson);
+ toConfig = new MToConfig(toJobConfig);
}
- MConnectionForms connection = new MConnectionForms(connForms);
- MConnector connector = new MConnector(uniqueName, className, version,
- connection, fromJob, toJob);
+
+ MLinkConfig linkConfig = new MLinkConfig(linkConfigs);
+ MConnector connector = new MConnector(uniqueName, className, version, linkConfig, fromConfig,
+ toConfig);
+
connector.setPersistenceId(connectorId);
connectors.add(connector);
}
- if(jsonObject.containsKey(CONNECTOR_CONFIGS)) {
- bundles = new HashMap<Long, ResourceBundle>();
+ if (jsonObject.containsKey(CONNECTOR_CONFIGS)) {
+ connectorConfigBundles = new HashMap<Long, ResourceBundle>();
JSONObject jsonBundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS);
Set<Map.Entry<String, JSONObject>> entrySet = jsonBundles.entrySet();
for (Map.Entry<String, JSONObject> entry : entrySet) {
- bundles.put(Long.parseLong(entry.getKey()),
- restoreResourceBundle(entry.getValue()));
+ connectorConfigBundles.put(Long.parseLong(entry.getKey()),
+ restoreResourceBundle(entry.getValue()));
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/DriverBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/DriverBean.java b/common/src/main/java/org/apache/sqoop/json/DriverBean.java
new file mode 100644
index 0000000..90cdbef
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/json/DriverBean.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json;
+
+import static org.apache.sqoop.json.util.ConfigSerialization.DRIVER_CONFIG;
+import static org.apache.sqoop.json.util.ConfigSerialization.DRIVER_VERSION;
+import static org.apache.sqoop.json.util.ConfigSerialization.ID;
+import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList;
+import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONFIGS;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle;
+
+import java.util.List;
+import java.util.ResourceBundle;
+
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MDriver;
+import org.apache.sqoop.model.MDriverConfig;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+/**
+ * Json representation of the driver
+ *
+ */
+public class DriverBean implements JsonBean {
+
+ public static final String CURRENT_DRIVER_VERSION = "1";
+
+ private MDriver driver;
+
+ private ResourceBundle bundle;
+
+ // for "extract"
+ public DriverBean(MDriver driver, ResourceBundle bundle) {
+ this.driver = driver;
+ this.bundle = bundle;
+ }
+
+ // for "restore"
+ public DriverBean() {
+ }
+
+ public MDriver getDriver() {
+ return driver;
+ }
+
+ public ResourceBundle getDriverConfigResourceBundle() {
+ return bundle;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public JSONObject extract(boolean skipSensitive) {
+ JSONArray configs =
+ extractConfigList(driver.getDriverConfig().getConfigs(), skipSensitive);
+
+ JSONObject result = new JSONObject();
+ result.put(ID, driver.getPersistenceId());
+ result.put(DRIVER_VERSION, driver.getVersion());
+ result.put(DRIVER_CONFIG, configs);
+ result.put(CONFIGS, extractResourceBundle(bundle));
+ return result;
+ }
+
+ @Override
+ public void restore(JSONObject jsonObject) {
+ long id = (Long) jsonObject.get(ID);
+ String driverVersion = (String) jsonObject.get(DRIVER_VERSION);
+ List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_CONFIG));
+ driver = new MDriver(new MDriverConfig(driverConfig), driverVersion);
+ driver.setPersistenceId(id);
+ bundle = restoreResourceBundle((JSONObject) jsonObject.get(CONFIGS));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java b/common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java
deleted file mode 100644
index a2cc8b8..0000000
--- a/common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json;
-
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MJobForms;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-
-import java.util.List;
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.json.util.FormSerialization.*;
-import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
-
-public class DriverConfigBean implements JsonBean {
-
- private MDriverConfig driverConfig;
-
- private ResourceBundle bundle;
-
- // for "extract"
- public DriverConfigBean(MDriverConfig driverConfig, ResourceBundle bundle) {
- this.driverConfig = driverConfig;
- this.bundle = bundle;
- }
-
- // for "restore"
- public DriverConfigBean() {
- }
-
- public MDriverConfig getDriverConfig() {
- return driverConfig;
- }
-
- public ResourceBundle getResourceBundle() {
- return bundle;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public JSONObject extract(boolean skipSensitive) {
- // TODO(Abe): Add From/To connection forms.
- JSONArray conForms =
- extractForms(driverConfig.getConnectionForms().getForms(), skipSensitive);
- JSONArray jobForms = extractForms(driverConfig.getJobForms().getForms(), skipSensitive);
-
- JSONObject result = new JSONObject();
- result.put(ID, driverConfig.getPersistenceId());
- result.put(DRIVER_VERSION, driverConfig.getVersion());
- result.put(CON_FORMS, conForms);
- result.put(JOB_FORMS, jobForms);
- result.put(CONFIGS, extractResourceBundle(bundle));
- return result;
- }
-
- @Override
- public void restore(JSONObject jsonObject) {
- long id = (Long) jsonObject.get(ID);
- String driverVersion = (String) jsonObject.get(DRIVER_VERSION);
-
- List<MForm> connForms = restoreForms((JSONArray) jsonObject.get(CON_FORMS));
- List<MForm> jobForms = restoreForms((JSONArray) jsonObject.get(JOB_FORMS));
-
- // TODO(Abe): Get From/To connection forms.
- driverConfig = new MDriverConfig(
- new MConnectionForms(connForms),
- new MJobForms(jobForms),
- driverVersion);
- driverConfig.setPersistenceId(id);
-
- bundle = restoreResourceBundle((JSONObject) jsonObject.get(CONFIGS));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/JobBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/JobBean.java b/common/src/main/java/org/apache/sqoop/json/JobBean.java
index a3e6071..8f42edb 100644
--- a/common/src/main/java/org/apache/sqoop/json/JobBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/JobBean.java
@@ -17,12 +17,17 @@
*/
package org.apache.sqoop.json;
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
+import static org.apache.sqoop.json.util.ConfigSerialization.CREATION_DATE;
+import static org.apache.sqoop.json.util.ConfigSerialization.CREATION_USER;
+import static org.apache.sqoop.json.util.ConfigSerialization.ENABLED;
+import static org.apache.sqoop.json.util.ConfigSerialization.UPDATE_DATE;
+import static org.apache.sqoop.json.util.ConfigSerialization.UPDATE_USER;
+import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList;
+import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONNECTOR_CONFIGS;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.DRIVER_CONFIGS;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle;
import java.util.ArrayList;
import java.util.Date;
@@ -32,11 +37,18 @@ import java.util.Map;
import java.util.ResourceBundle;
import java.util.Set;
-import static org.apache.sqoop.json.util.FormSerialization.*;
-import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MDriver;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MToConfig;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
/**
- *
+ * Json representation of the job config
*/
public class JobBean implements JsonBean {
@@ -47,9 +59,9 @@ public class JobBean implements JsonBean {
private static final String TO_LINK_ID = "to-link-id";
private static final String FROM_CONNECTOR_ID = "from-connector-id";
private static final String TO_CONNECTOR_ID = "to-connector-id";
- private static final String FROM_CONNECTOR_PART = "from-connector";
- private static final String TO_CONNECTOR_PART = "to-connector";
- private static final String FRAMEWORK_PART = "framework";
+ private static final String FROM_CONFIG = "from-config";
+ private static final String TO_CONFIG = "to-config";
+ private static final String DRIVER_CONFIG = "driver-config";
// Required
private List<MJob> jobs;
@@ -114,18 +126,21 @@ public class JobBean implements JsonBean {
object.put(CREATION_DATE, job.getCreationDate().getTime());
object.put(UPDATE_USER, job.getLastUpdateUser());
object.put(UPDATE_DATE, job.getLastUpdateDate().getTime());
- object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
- object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
+ // job link associated connectors
object.put(FROM_CONNECTOR_ID, job.getConnectorId(Direction.FROM));
object.put(TO_CONNECTOR_ID, job.getConnectorId(Direction.TO));
- object.put(FROM_CONNECTOR_PART,
- extractForms(job.getConnectorPart(Direction.FROM).getForms(),skipSensitive));
- object.put(TO_CONNECTOR_PART,
- extractForms(job.getConnectorPart(Direction.TO).getForms(), skipSensitive));
- object.put(FRAMEWORK_PART,
- extractForms(job.getFrameworkPart().getForms(), skipSensitive));
-
- array.add(object);
+ // job associated links
+ object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
+ object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
+ // job configs
+ object.put(FROM_CONFIG, extractConfigList(job
+ .getJobConfig(Direction.FROM).getConfigs(), skipSensitive));
+ object.put(TO_CONFIG,
+ extractConfigList(job.getJobConfig(Direction.TO).getConfigs(), skipSensitive));
+ object.put(DRIVER_CONFIG,
+ extractConfigList(job.getDriverConfig().getConfigs(), skipSensitive));
+
+ array.add(object);
}
JSONObject all = new JSONObject();
@@ -160,22 +175,22 @@ public class JobBean implements JsonBean {
long toConnectorId = (Long) object.get(TO_CONNECTOR_ID);
long fromConnectionId = (Long) object.get(FROM_LINK_ID);
long toConnectionId = (Long) object.get(TO_LINK_ID);
- JSONArray fromConnectorPart = (JSONArray) object.get(FROM_CONNECTOR_PART);
- JSONArray toConnectorPart = (JSONArray) object.get(TO_CONNECTOR_PART);
- JSONArray frameworkPart = (JSONArray) object.get(FRAMEWORK_PART);
+ JSONArray fromConfigJson = (JSONArray) object.get(FROM_CONFIG);
+ JSONArray toConfigJson = (JSONArray) object.get(TO_CONFIG);
+ JSONArray driverConfigJson = (JSONArray) object.get(DRIVER_CONFIG);
- List<MForm> fromConnectorParts = restoreForms(fromConnectorPart);
- List<MForm> toConnectorParts = restoreForms(toConnectorPart);
- List<MForm> frameworkForms = restoreForms(frameworkPart);
+ List<MConfig> fromConfig = restoreConfigList(fromConfigJson);
+ List<MConfig> toConfig = restoreConfigList(toConfigJson);
+ List<MConfig> driverConfig = restoreConfigList(driverConfigJson);
MJob job = new MJob(
fromConnectorId,
toConnectorId,
fromConnectionId,
toConnectionId,
- new MJobForms(fromConnectorParts),
- new MJobForms(toConnectorParts),
- new MJobForms(frameworkForms)
+ new MFromConfig(fromConfig),
+ new MToConfig(toConfig),
+ new MDriverConfig(driverConfig)
);
job.setPersistenceId((Long) object.get(ID));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/JobValidationBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/JobValidationBean.java b/common/src/main/java/org/apache/sqoop/json/JobValidationBean.java
index 34c0a4b..9e78fe9 100644
--- a/common/src/main/java/org/apache/sqoop/json/JobValidationBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/JobValidationBean.java
@@ -21,41 +21,41 @@ import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.DirectionError;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
+import org.apache.sqoop.validation.ConfigValidator;
import org.json.simple.JSONObject;
import java.util.HashMap;
import java.util.Map;
/**
- * Bean for sending validations across network. This bean will move two
- * validation objects at one time - one for connector and second for framework
- * part of validated entity. Optionally validation bean can also transfer
+ * Bean for sending validations across network. This bean will send job validation results
+ * Optionally validation bean can also transfer
* created persistent id in case that new entity was created.
*/
public class JobValidationBean implements JsonBean {
private static final String ID = "id";
- private static final String FRAMEWORK = "framework";
- private static final String CONNECTOR = "connector";
+ private static final String JOB = "job";
private static final String FROM = "from";
private static final String TO = "to";
+ private static final String DRIVER = "driver";
+
private static final String STATUS = "status";
private static final String MESSAGE = "message";
private static final String MESSAGES = "messages";
private Long id;
- private Validation fromConnectorValidation;
- private Validation toConnectorValidation;
- private Validation frameworkValidation;
+ private ConfigValidator fromConfigValidation;
+ private ConfigValidator toConfigValidation;
+ private ConfigValidator driverConfigValidation;
// For "extract"
- public JobValidationBean(Validation fromConnector, Validation framework, Validation toConnector) {
+ public JobValidationBean(ConfigValidator fromConnector, ConfigValidator framework, ConfigValidator toConnector) {
this();
- this.fromConnectorValidation = fromConnector;
- this.toConnectorValidation = toConnector;
- this.frameworkValidation = framework;
+ this.fromConfigValidation = fromConnector;
+ this.toConfigValidation = toConnector;
+ this.driverConfigValidation = framework;
}
// For "restore"
@@ -63,21 +63,21 @@ public class JobValidationBean implements JsonBean {
id = null;
}
- public Validation getConnectorValidation(Direction type) {
+ public ConfigValidator getConnectorValidation(Direction type) {
switch(type) {
case FROM:
- return fromConnectorValidation;
+ return fromConfigValidation;
case TO:
- return toConnectorValidation;
+ return toConfigValidation;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
- public Validation getFrameworkValidation() {
- return frameworkValidation;
+ public ConfigValidator getFrameworkValidation() {
+ return driverConfigValidation;
}
public void setId(Long id) {
@@ -91,32 +91,30 @@ public class JobValidationBean implements JsonBean {
@SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) {
JSONObject object = new JSONObject();
- JSONObject connectorObject = new JSONObject();
+ JSONObject jobObject = new JSONObject();
// Optionally transfer id
if(id != null) {
object.put(ID, id);
}
- connectorObject.put(FROM, extractValidation(getConnectorValidation(Direction.FROM)));
- connectorObject.put(TO, extractValidation(getConnectorValidation(Direction.TO)));
-
- object.put(FRAMEWORK, extractValidation(frameworkValidation));
- object.put(CONNECTOR, connectorObject);
-
+ jobObject.put(FROM, extractValidation(getConnectorValidation(Direction.FROM)));
+ jobObject.put(TO, extractValidation(getConnectorValidation(Direction.TO)));
+ jobObject.put(DRIVER, extractValidation(driverConfigValidation));
+ object.put(JOB, jobObject);
return object;
}
@SuppressWarnings("unchecked")
- private JSONObject extractValidation(Validation validation) {
+ private JSONObject extractValidation(ConfigValidator validation) {
JSONObject object = new JSONObject();
object.put(STATUS, validation.getStatus().name());
JSONObject jsonMessages = new JSONObject();
- Map<Validation.FormInput, Validation.Message> messages = validation.getMessages();
+ Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages = validation.getMessages();
- for(Map.Entry<Validation.FormInput, Validation.Message> entry : messages.entrySet()) {
+ for(Map.Entry<ConfigValidator.ConfigInput, ConfigValidator.Message> entry : messages.entrySet()) {
JSONObject jsonEntry = new JSONObject();
jsonEntry.put(STATUS, entry.getValue().getStatus().name());
jsonEntry.put(MESSAGE, entry.getValue().getMessage());
@@ -133,20 +131,21 @@ public class JobValidationBean implements JsonBean {
// Optional and accepting NULLs
id = (Long) jsonObject.get(ID);
- JSONObject jsonConnectorObject = (JSONObject)jsonObject.get(CONNECTOR);
+ JSONObject jobJsonObject = (JSONObject)jsonObject.get(JOB);
- fromConnectorValidation = restoreValidation(
- (JSONObject)jsonConnectorObject.get(FROM));
- toConnectorValidation = restoreValidation(
- (JSONObject)jsonConnectorObject.get(TO));
- frameworkValidation = restoreValidation(
- (JSONObject)jsonObject.get(FRAMEWORK));
+ fromConfigValidation = restoreValidation(
+ (JSONObject)jobJsonObject.get(FROM));
+ toConfigValidation = restoreValidation(
+ (JSONObject)jobJsonObject.get(TO));
+ driverConfigValidation = restoreValidation(
+ (JSONObject)jobJsonObject.get(DRIVER));
}
- public Validation restoreValidation(JSONObject jsonObject) {
+ public ConfigValidator restoreValidation(JSONObject jsonObject) {
+
JSONObject jsonMessages = (JSONObject) jsonObject.get(MESSAGES);
- Map<Validation.FormInput, Validation.Message> messages
- = new HashMap<Validation.FormInput, Validation.Message>();
+ Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages
+ = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
for(Object key : jsonMessages.keySet()) {
JSONObject jsonMessage = (JSONObject) jsonMessages.get(key);
@@ -154,14 +153,14 @@ public class JobValidationBean implements JsonBean {
Status status = Status.valueOf((String) jsonMessage.get(STATUS));
String stringMessage = (String) jsonMessage.get(MESSAGE);
- Validation.Message message
- = new Validation.Message(status, stringMessage);
+ ConfigValidator.Message message
+ = new ConfigValidator.Message(status, stringMessage);
- messages.put(new Validation.FormInput((String)key), message);
+ messages.put(new ConfigValidator.ConfigInput((String)key), message);
}
Status status = Status.valueOf((String) jsonObject.get(STATUS));
- return new Validation(status, messages);
+ return new ConfigValidator(status, messages);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/LinkBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/LinkBean.java b/common/src/main/java/org/apache/sqoop/json/LinkBean.java
index 8981ea7..af0fc9d 100644
--- a/common/src/main/java/org/apache/sqoop/json/LinkBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/LinkBean.java
@@ -18,8 +18,8 @@
package org.apache.sqoop.json;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MConfig;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@@ -31,7 +31,7 @@ import java.util.Map;
import java.util.ResourceBundle;
import java.util.Set;
-import static org.apache.sqoop.json.util.FormSerialization.*;
+import static org.apache.sqoop.json.util.ConfigSerialization.*;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
/**
@@ -42,16 +42,14 @@ import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
*/
public class LinkBean implements JsonBean {
- private static final String CONNECTOR_ID = "connector-id";
- private static final String CONNECTOR_PART = "connector";
- private static final String FRAMEWORK_PART = "framework";
+ static final String CONNECTOR_ID = "connector-id";
+ static final String LINK_CONFIG = "link-config";
// Required
private List<MLink> links;
// Optional
- private Map<Long, ResourceBundle> connectorConfigBundles;
- private ResourceBundle driverConfigBundle;
+ private Map<Long, ResourceBundle> linkConfigBundles;
// For "extract"
public LinkBean(MLink link) {
@@ -67,72 +65,56 @@ public class LinkBean implements JsonBean {
// For "restore"
public LinkBean() {
- connectorConfigBundles = new HashMap<Long, ResourceBundle>();
- }
-
- public void setDriverConfigBundle(ResourceBundle driverConfigBundle) {
- this.driverConfigBundle = driverConfigBundle;
+ linkConfigBundles = new HashMap<Long, ResourceBundle>();
}
public void addConnectorConfigBundle(Long id, ResourceBundle connectorConfigBundle) {
- connectorConfigBundles.put(id, connectorConfigBundle);
+ linkConfigBundles.put(id, connectorConfigBundle);
}
- public boolean hasConnectorBundle(Long id) {
- return connectorConfigBundles.containsKey(id);
+ public boolean hasConnectorConfigBundle(Long id) {
+ return linkConfigBundles.containsKey(id);
}
public List<MLink> getLinks() {
return links;
}
- public ResourceBundle getConnectorBundle(Long id) {
- return connectorConfigBundles.get(id);
- }
-
- public ResourceBundle getFrameworkBundle() {
- return driverConfigBundle;
+ public ResourceBundle getConnectorConfigBundle(Long id) {
+ return linkConfigBundles.get(id);
}
@Override
@SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) {
- JSONArray array = new JSONArray();
+ JSONArray linkArray = new JSONArray();
for(MLink link : links) {
- JSONObject object = new JSONObject();
-
- object.put(ID, link.getPersistenceId());
- object.put(NAME, link.getName());
- object.put(ENABLED, link.getEnabled());
- object.put(CREATION_USER, link.getCreationUser());
- object.put(CREATION_DATE, link.getCreationDate().getTime());
- object.put(UPDATE_USER, link.getLastUpdateUser());
- object.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
- object.put(CONNECTOR_ID, link.getConnectorId());
- object.put(CONNECTOR_PART,
- extractForms(link.getConnectorPart().getForms(), skipSensitive));
- object.put(FRAMEWORK_PART,
- extractForms(link.getFrameworkPart().getForms(), skipSensitive));
-
- array.add(object);
+ JSONObject linkJsonObject = new JSONObject();
+
+ linkJsonObject.put(ID, link.getPersistenceId());
+ linkJsonObject.put(NAME, link.getName());
+ linkJsonObject.put(ENABLED, link.getEnabled());
+ linkJsonObject.put(CREATION_USER, link.getCreationUser());
+ linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime());
+ linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
+ linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
+ linkJsonObject.put(CONNECTOR_ID, link.getConnectorId());
+ linkJsonObject.put(LINK_CONFIG,
+ extractConfigList(link.getConnectorLinkConfig().getConfigs(), skipSensitive));
+
+ linkArray.add(linkJsonObject);
}
JSONObject all = new JSONObject();
- all.put(ALL, array);
-
- if(!connectorConfigBundles.isEmpty()) {
+ all.put(ALL, linkArray);
+ if (!linkConfigBundles.isEmpty()) {
JSONObject bundles = new JSONObject();
-
- for(Map.Entry<Long, ResourceBundle> entry : connectorConfigBundles.entrySet()) {
- bundles.put(entry.getKey().toString(),
- extractResourceBundle(entry.getValue()));
+ for (Map.Entry<Long, ResourceBundle> entry : linkConfigBundles.entrySet()) {
+ bundles.put(entry.getKey().toString(), extractResourceBundle(entry.getValue()));
}
all.put(CONNECTOR_CONFIGS, bundles);
}
- if(driverConfigBundle != null) {
- all.put(DRIVER_CONFIGS,extractResourceBundle(driverConfigBundle));
- }
return all;
}
@@ -147,15 +129,11 @@ public class LinkBean implements JsonBean {
JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(CONNECTOR_ID);
- JSONArray connectorPart = (JSONArray) object.get(CONNECTOR_PART);
- JSONArray frameworkPart = (JSONArray) object.get(FRAMEWORK_PART);
+ JSONArray connectorLinkConfig = (JSONArray) object.get(LINK_CONFIG);
- List<MForm> connectorForms = restoreForms(connectorPart);
- List<MForm> frameworkForms = restoreForms(frameworkPart);
+ List<MConfig> linkConfig = restoreConfigList(connectorLinkConfig);
- MLink link = new MLink(connectorId,
- new MConnectionForms(connectorForms),
- new MConnectionForms(frameworkForms));
+ MLink link = new MLink(connectorId, new MLinkConfig(linkConfig));
link.setPersistenceId((Long) object.get(ID));
link.setName((String) object.get(NAME));
@@ -172,13 +150,9 @@ public class LinkBean implements JsonBean {
JSONObject bundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS);
Set<Map.Entry<String, JSONObject>> entrySet = bundles.entrySet();
for (Map.Entry<String, JSONObject> entry : entrySet) {
- connectorConfigBundles.put(Long.parseLong(entry.getKey()),
+ linkConfigBundles.put(Long.parseLong(entry.getKey()),
restoreResourceBundle(entry.getValue()));
}
}
- if(jsonObject.containsKey(DRIVER_CONFIGS)) {
- driverConfigBundle = restoreResourceBundle(
- (JSONObject) jsonObject.get(DRIVER_CONFIGS));
- }
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java b/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java
index a6a6b65..682f63b 100644
--- a/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java
@@ -18,37 +18,32 @@
package org.apache.sqoop.json;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
+import org.apache.sqoop.validation.ConfigValidator;
import org.json.simple.JSONObject;
import java.util.HashMap;
import java.util.Map;
/**
- * Bean for sending validations across network. This bean will move two
- * validation objects at one time - one for connector and second for framework
- * part of validated entity. Optionally validation bean can also transfer
+ * Bean for sending validations across network.This bean will transfer link config
+ * validation results. Optionally validation bean can also transfer
* created persistent id in case that new entity was created.
*/
public class LinkValidationBean implements JsonBean {
private static final String ID = "id";
- private static final String FRAMEWORK = "framework";
- private static final String CONNECTOR = "connector";
private static final String STATUS = "status";
private static final String MESSAGE = "message";
private static final String MESSAGES = "messages";
private Long id;
- private Validation connectorValidation;
- private Validation frameworkValidation;
+ private ConfigValidator linkConfigValidation;
// For "extract"
- public LinkValidationBean(Validation connector, Validation framework) {
+ public LinkValidationBean(ConfigValidator linkConfigValidator) {
this();
- this.connectorValidation = connector;
- this.frameworkValidation = framework;
+ this.linkConfigValidation = linkConfigValidator;
}
// For "restore"
@@ -56,12 +51,8 @@ public class LinkValidationBean implements JsonBean {
id = null;
}
- public Validation getConnectorValidation() {
- return connectorValidation;
- }
-
- public Validation getFrameworkValidation() {
- return frameworkValidation;
+ public ConfigValidator getLinkConfigValidator() {
+ return linkConfigValidation;
}
public void setId(Long id) {
@@ -80,23 +71,20 @@ public class LinkValidationBean implements JsonBean {
if(id != null) {
object.put(ID, id);
}
-
- object.put(CONNECTOR, extractValidation(connectorValidation));
- object.put(FRAMEWORK, extractValidation(frameworkValidation));
-
+ object.put(LinkBean.LINK_CONFIG, extractValidation(linkConfigValidation));
return object;
}
@SuppressWarnings("unchecked")
- private JSONObject extractValidation(Validation validation) {
+ private JSONObject extractValidation(ConfigValidator validation) {
JSONObject object = new JSONObject();
object.put(STATUS, validation.getStatus().name());
JSONObject jsonMessages = new JSONObject();
- Map<Validation.FormInput, Validation.Message> messages = validation.getMessages();
+ Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages = validation.getMessages();
- for(Map.Entry<Validation.FormInput, Validation.Message> entry : messages.entrySet()) {
+ for(Map.Entry<ConfigValidator.ConfigInput, ConfigValidator.Message> entry : messages.entrySet()) {
JSONObject jsonEntry = new JSONObject();
jsonEntry.put(STATUS, entry.getValue().getStatus().name());
jsonEntry.put(MESSAGE, entry.getValue().getMessage());
@@ -113,16 +101,14 @@ public class LinkValidationBean implements JsonBean {
// Optional and accepting NULLs
id = (Long) jsonObject.get(ID);
- connectorValidation = restoreValidation(
- (JSONObject)jsonObject.get(CONNECTOR));
- frameworkValidation = restoreValidation(
- (JSONObject)jsonObject.get(FRAMEWORK));
+ linkConfigValidation = restoreValidation(
+ (JSONObject)jsonObject.get(LinkBean.LINK_CONFIG));
}
- public Validation restoreValidation(JSONObject jsonObject) {
+ public ConfigValidator restoreValidation(JSONObject jsonObject) {
JSONObject jsonMessages = (JSONObject) jsonObject.get(MESSAGES);
- Map<Validation.FormInput, Validation.Message> messages
- = new HashMap<Validation.FormInput, Validation.Message>();
+ Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages
+ = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
for(Object key : jsonMessages.keySet()) {
JSONObject jsonMessage = (JSONObject) jsonMessages.get(key);
@@ -130,14 +116,14 @@ public class LinkValidationBean implements JsonBean {
Status status = Status.valueOf((String) jsonMessage.get(STATUS));
String stringMessage = (String) jsonMessage.get(MESSAGE);
- Validation.Message message
- = new Validation.Message(status, stringMessage);
+ ConfigValidator.Message message
+ = new ConfigValidator.Message(status, stringMessage);
- messages.put(new Validation.FormInput((String)key), message);
+ messages.put(new ConfigValidator.ConfigInput((String)key), message);
}
Status status = Status.valueOf((String) jsonObject.get(STATUS));
- return new Validation(status, messages);
+ return new ConfigValidator(status, messages);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/ThrowableBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/ThrowableBean.java b/common/src/main/java/org/apache/sqoop/json/ThrowableBean.java
index fc63ed4..cd26020 100644
--- a/common/src/main/java/org/apache/sqoop/json/ThrowableBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/ThrowableBean.java
@@ -26,7 +26,7 @@ import java.util.LinkedList;
import java.util.List;
/**
- * Transfer throwable instance.
+ * Transfer throwable instance as a throwable bean.
*/
public class ThrowableBean implements JsonBean {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/ValidationResultBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/ValidationResultBean.java b/common/src/main/java/org/apache/sqoop/json/ValidationResultBean.java
index 89bc8db..4b3fc37 100644
--- a/common/src/main/java/org/apache/sqoop/json/ValidationResultBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/ValidationResultBean.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.json;
import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.ValidationResult;
+import org.apache.sqoop.validation.ConfigValidationResult;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@@ -38,18 +38,18 @@ public class ValidationResultBean implements JsonBean {
private static final String STATUS = "STATUS";
private static final String TEXT = "TEXT";
- private ValidationResult[] results;
+ private ConfigValidationResult[] results;
private Long id;
public ValidationResultBean() {
// Empty, for restore
}
- public ValidationResultBean(ValidationResult ... results) {
+ public ValidationResultBean(ConfigValidationResult ... results) {
this.results = results;
}
- public ValidationResult[] getValidationResults() {
+ public ConfigValidationResult[] getValidationResults() {
return results;
}
@@ -65,7 +65,7 @@ public class ValidationResultBean implements JsonBean {
public JSONObject extract(boolean skipSensitive) {
JSONArray array = new JSONArray();
- for(ValidationResult result : results) {
+ for(ConfigValidationResult result : results) {
JSONObject output = extractValidationResult(result);
array.add(output);
}
@@ -78,7 +78,7 @@ public class ValidationResultBean implements JsonBean {
return object;
}
- private JSONObject extractValidationResult(ValidationResult result) {
+ private JSONObject extractValidationResult(ConfigValidationResult result) {
JSONObject ret = new JSONObject();
for(Map.Entry<String, List<Message>> entry : result.getMessages().entrySet()) {
@@ -110,7 +110,7 @@ public class ValidationResultBean implements JsonBean {
@Override
public void restore(JSONObject jsonObject) {
JSONArray array = (JSONArray) jsonObject.get(ROOT);
- results = new ValidationResult[array.size()];
+ results = new ConfigValidationResult[array.size()];
int i = 0;
for(Object item : array) {
@@ -122,8 +122,8 @@ public class ValidationResultBean implements JsonBean {
}
}
- private ValidationResult restoreValidationResult(JSONObject item) {
- ValidationResult result = new ValidationResult();
+ private ConfigValidationResult restoreValidationResult(JSONObject item) {
+ ConfigValidationResult result = new ConfigValidationResult();
Set<Map.Entry<String, JSONArray>> entrySet = item.entrySet();
for(Map.Entry<String, JSONArray> entry : entrySet) {
[26/52] [abbrv] git commit: SQOOP-1378: Sqoop2: From/To: Refactor
schema
Posted by ab...@apache.org.
SQOOP-1378: Sqoop2: From/To: Refactor schema
This patch also changes the tools documentation.
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/2c20d920
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/2c20d920
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/2c20d920
Branch: refs/heads/SQOOP-1367
Commit: 2c20d920f4ab31ae97ba57952d17677146069b5c
Parents: d015362
Author: Gwen Shapira <cs...@gmail.com>
Authored: Thu Sep 25 15:39:43 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:59:24 2014 -0700
----------------------------------------------------------------------
.../apache/sqoop/job/etl/ExtractorContext.java | 13 +-
.../java/org/apache/sqoop/json/SchemaBean.java | 2 +-
.../org/apache/sqoop/json/SubmissionBean.java | 7 +-
.../sqoop/json/util/SchemaSerialization.java | 2 +-
.../org/apache/sqoop/model/MSubmission.java | 15 +-
.../java/org/apache/sqoop/schema/Schema.java | 11 +-
.../org/apache/sqoop/schema/SchemaError.java | 6 +
.../apache/sqoop/schema/SchemaMatchOption.java | 40 +++++
.../org/apache/sqoop/schema/type/Column.java | 6 +-
.../json/util/TestSchemaSerialization.java | 2 +-
.../sqoop/connector/jdbc/TestExtractor.java | 4 +-
.../sqoop/connector/hdfs/HdfsConnector.java | 5 +-
.../connector/hdfs/HdfsConnectorError.java | 3 +-
.../sqoop/connector/hdfs/HdfsInitializer.java | 2 +-
.../sqoop/connector/hdfs/TestExtractor.java | 4 +-
.../idf/CSVIntermediateDataFormat.java | 100 ++++++++----
.../connector/idf/IntermediateDataFormat.java | 16 +-
.../connector/idf/matcher/AbstractMatcher.java | 62 ++++++++
.../connector/idf/matcher/LocationMatcher.java | 82 ++++++++++
.../connector/idf/matcher/NameMatcher.java | 69 +++++++++
.../idf/TestCSVIntermediateDataFormat.java | 151 ++++++++++++++++++-
.../org/apache/sqoop/driver/JobManager.java | 15 +-
docs/src/site/sphinx/Tools.rst | 3 +-
.../apache/sqoop/job/mr/ConfigurationUtils.java | 53 ++++---
.../org/apache/sqoop/job/mr/SqoopMapper.java | 18 +--
.../job/mr/SqoopOutputFormatLoadExecutor.java | 15 +-
.../sqoop/shell/utils/SubmissionDisplayer.java | 4 +-
.../main/resources/shell-resource.properties | 5 +-
.../mapreduce/MapreduceSubmissionEngine.java | 7 +-
29 files changed, 584 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java b/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
index fd73890..3272b56 100644
--- a/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
+++ b/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
@@ -30,12 +30,10 @@ public class ExtractorContext extends TransferableContext {
private DataWriter writer;
- private Schema schema;
- public ExtractorContext(ImmutableContext context, DataWriter writer, Schema schema) {
+ public ExtractorContext(ImmutableContext context, DataWriter writer) {
super(context);
this.writer = writer;
- this.schema = schema;
}
/**
@@ -47,12 +45,5 @@ public class ExtractorContext extends TransferableContext {
return writer;
}
- /**
- * Return schema associated with this step.
- *
- * @return
- */
- public Schema getSchema() {
- return schema;
- }
+
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/json/SchemaBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/SchemaBean.java b/common/src/main/java/org/apache/sqoop/json/SchemaBean.java
index 468f7ee..f51fec8 100644
--- a/common/src/main/java/org/apache/sqoop/json/SchemaBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/SchemaBean.java
@@ -48,7 +48,7 @@ public class SchemaBean implements JsonBean {
@Override
public void restore(JSONObject jsonObject) {
- schema = SchemaSerialization.restoreSchemna(jsonObject);
+ schema = SchemaSerialization.restoreSchema(jsonObject);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java b/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
index 9b1ae74..4b80338 100644
--- a/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
@@ -32,7 +32,7 @@ import java.util.Map;
import java.util.Set;
import static org.apache.sqoop.json.util.SchemaSerialization.extractSchema;
-import static org.apache.sqoop.json.util.SchemaSerialization.restoreSchemna;
+import static org.apache.sqoop.json.util.SchemaSerialization.restoreSchema;
/**
*
@@ -188,11 +188,12 @@ public class SubmissionBean implements JsonBean {
if(object.containsKey(COUNTERS)) {
submission.setCounters(restoreCounters((JSONObject) object.get(COUNTERS)));
}
+
if(object.containsKey(FROM_SCHEMA)) {
- submission.setFromSchema(restoreSchemna((JSONObject) object.get(FROM_SCHEMA)));
+ submission.setFromSchema(restoreSchema((JSONObject) object.get(FROM_SCHEMA)));
}
if(object.containsKey(TO_SCHEMA)) {
- submission.setToSchema(restoreSchemna((JSONObject) object.get(TO_SCHEMA)));
+ submission.setToSchema(restoreSchema((JSONObject) object.get(TO_SCHEMA)));
}
this.submissions.add(submission);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java
index f6a9bbf..1e6da6d 100644
--- a/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java
+++ b/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java
@@ -79,7 +79,7 @@ public class SchemaSerialization {
return object;
}
- public static Schema restoreSchemna(JSONObject jsonObject) {
+ public static Schema restoreSchema(JSONObject jsonObject) {
String name = (String)jsonObject.get(NAME);
String note = (String)jsonObject.get(NOTE);
java.util.Date date = new java.util.Date((Long)jsonObject.get(CREATION_DATE));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/model/MSubmission.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MSubmission.java b/common/src/main/java/org/apache/sqoop/model/MSubmission.java
index ca21135..7290df5 100644
--- a/common/src/main/java/org/apache/sqoop/model/MSubmission.java
+++ b/common/src/main/java/org/apache/sqoop/model/MSubmission.java
@@ -102,17 +102,14 @@ public class MSubmission extends MAccountableEntity {
/**
* Schema for the FROM part of the job submission
*
- * This property is required.
+ * This property is required, but can be empty.
*/
Schema fromSchema;
/**
* Schema for the TO part of the job submission
- * Optional schema that reported by the underlying I/O implementation. Please
- * note that this property might be empty and in such case use the FROM schema
- * on the TO side.
*
- * This property is optional.
+ * This property is required, but can be empty.
*/
Schema toSchema;
@@ -224,16 +221,16 @@ public class MSubmission extends MAccountableEntity {
return fromSchema;
}
- public void setFromSchema(Schema connectorSchema) {
- this.fromSchema = connectorSchema;
+ public void setFromSchema(Schema fromSchema) {
+ this.fromSchema = fromSchema;
}
public Schema getToSchema() {
return toSchema;
}
- public void setToSchema(Schema hioSchema) {
- this.toSchema = hioSchema;
+ public void setToSchema(Schema toSchema) {
+ this.toSchema = toSchema;
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/schema/Schema.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/schema/Schema.java b/common/src/main/java/org/apache/sqoop/schema/Schema.java
index bbebab8..40c362c 100644
--- a/common/src/main/java/org/apache/sqoop/schema/Schema.java
+++ b/common/src/main/java/org/apache/sqoop/schema/Schema.java
@@ -77,7 +77,7 @@ public class Schema {
* same name will lead to an exception being thrown.
*
* @param column Column that should be added to the schema at the end.
- * @return
+ * @return a reference to this object
*/
public Schema addColumn(Column column) {
if(column.getName() == null) {
@@ -121,6 +121,15 @@ public class Schema {
return columns;
}
+ public boolean isEmpty() {
+ if (columns.size()==0) {
+ return true;
+ } else {
+ return false;
+ }
+
+ }
+
public String toString() {
return new StringBuilder("Schema{")
.append("name=").append(name).append("")
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/schema/SchemaError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/schema/SchemaError.java b/common/src/main/java/org/apache/sqoop/schema/SchemaError.java
index 7c8c61e..d430a64 100644
--- a/common/src/main/java/org/apache/sqoop/schema/SchemaError.java
+++ b/common/src/main/java/org/apache/sqoop/schema/SchemaError.java
@@ -30,6 +30,12 @@ public enum SchemaError implements ErrorCode {
SCHEMA_0002("Duplicate column name"),
+ SCHEMA_0003("Source and Target schemas don't match"),
+
+ SCHEMA_0004("Non-null target column has no matching source column"),
+
+ SCHEMA_0005("No matching method available for source and target schemas")
+
;
private final String message;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/schema/SchemaMatchOption.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/schema/SchemaMatchOption.java b/common/src/main/java/org/apache/sqoop/schema/SchemaMatchOption.java
new file mode 100644
index 0000000..e3ab026
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/schema/SchemaMatchOption.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.schema;
+
+/**
+ * The order of the matching options here indicates an order of preference
+ * if it is possible to use both NAME and LOCATION matching options, we will prefer NAME
+ *
+ * NAME - match columns in FROM and TO schemas by column name. Data from column "hello"
+ * will be written to a column named "hello". If TO schema doesn't have a column with
+ * identical name, the column will be skipped.
+ *
+ * LOCATION - match columns in FROM and TO schemas by the column location.
+ * Data from first column goes into first column in TO link.
+ * If FROM link has more columns than TO, the extra columns will be skipped.
+ *
+ * USER_DEFINED - not implemented yet.
+ */
+public enum SchemaMatchOption {
+ NAME,
+ LOCATION,
+ //TODO: SQOOP-1546 - SQOOP2: Allow users to define their own schema matching
+ USER_DEFINED
+ }
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/main/java/org/apache/sqoop/schema/type/Column.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/schema/type/Column.java b/common/src/main/java/org/apache/sqoop/schema/type/Column.java
index 30c26a3..97bd303 100644
--- a/common/src/main/java/org/apache/sqoop/schema/type/Column.java
+++ b/common/src/main/java/org/apache/sqoop/schema/type/Column.java
@@ -32,11 +32,15 @@ public abstract class Column {
*/
Boolean nullable;
+ /**
+ * By default columns are empty name and are nullable
+ */
public Column() {
+ this("", true);
}
public Column(String name) {
- setName(name);
+ this(name, true);
}
public Column(String name, Boolean nullable) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java b/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java
index ab5bbd4..b652b32 100644
--- a/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java
+++ b/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java
@@ -172,6 +172,6 @@ public class TestSchemaSerialization {
String transferredString = extractJson.toJSONString();
JSONObject restoreJson = (JSONObject) JSONValue.parse(transferredString);
- return SchemaSerialization.restoreSchemna(restoreJson);
+ return SchemaSerialization.restoreSchema(restoreJson);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
index 776359a..5f091de 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
@@ -81,7 +81,7 @@ public class TestExtractor extends TestCase {
Extractor extractor = new GenericJdbcExtractor();
DummyWriter writer = new DummyWriter();
- ExtractorContext extractorContext = new ExtractorContext(context, writer, null);
+ ExtractorContext extractorContext = new ExtractorContext(context, writer);
partition = new GenericJdbcPartition();
partition.setConditions("-50.0 <= DCOL AND DCOL < -16.6666666666666665");
@@ -115,7 +115,7 @@ public class TestExtractor extends TestCase {
Extractor extractor = new GenericJdbcExtractor();
DummyWriter writer = new DummyWriter();
- ExtractorContext extractorContext = new ExtractorContext(context, writer, null);
+ ExtractorContext extractorContext = new ExtractorContext(context, writer);
partition = new GenericJdbcPartition();
partition.setConditions("-50 <= ICOL AND ICOL < -16");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
index 70833a0..cd5350e 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
@@ -19,6 +19,7 @@
package org.apache.sqoop.connector.hdfs;
import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.common.VersionInfo;
import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
@@ -87,7 +88,9 @@ public class HdfsConnector extends SqoopConnector {
case TO:
return ToJobConfiguration.class;
default:
- return null;
+ throw new SqoopException(
+ HdfsConnectorError.GENERIC_HDFS_CONNECTOR_0006,
+ String.valueOf(jobType));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnectorError.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnectorError.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnectorError.java
index 8a095d2..71f0a03 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnectorError.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnectorError.java
@@ -31,7 +31,8 @@ public enum HdfsConnectorError implements ErrorCode{
/** The system was unable to instantiate the specified class. */
GENERIC_HDFS_CONNECTOR_0004("Unable to instantiate the specified class"),
/** Error occurs during loader run */
- GENERIC_HDFS_CONNECTOR_0005("Error occurs during loader run")
+ GENERIC_HDFS_CONNECTOR_0005("Error occurs during loader run"),
+ GENERIC_HDFS_CONNECTOR_0006("Unknown job type")
;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
index 923f904..c2dc1a5 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
@@ -39,6 +39,6 @@ public class HdfsInitializer extends Initializer {
@Override
public Schema getSchema(InitializerContext context, Object linkConf, Object jobConf) {
- return null;
+ return new Schema("HDFS file");
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
index 7942d59..c6d2f90 100644
--- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
@@ -129,8 +129,10 @@ public class TestExtractor extends TestHdfsBase {
public void writeRecord(Object obj) {
throw new AssertionError("Should not be writing object.");
}
- }, null);
+ });
+
LinkConfiguration connConf = new LinkConfiguration();
+
FromJobConfiguration jobConf = new FromJobConfiguration();
HdfsPartition partition = createPartition(FileUtils.listDir(inputDirectory));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
index df5cb9c..2a49221 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
@@ -22,7 +22,12 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.idf.matcher.AbstractMatcher;
+import org.apache.sqoop.connector.idf.matcher.LocationMatcher;
+import org.apache.sqoop.connector.idf.matcher.NameMatcher;
import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.SchemaError;
+import org.apache.sqoop.schema.SchemaMatchOption;
import org.apache.sqoop.schema.type.Column;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.FloatingPoint;
@@ -36,6 +41,7 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
import java.util.regex.Matcher;
public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
@@ -65,7 +71,8 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
private final List<Integer> stringFieldIndices = new ArrayList<Integer>();
private final List<Integer> byteFieldIndices = new ArrayList<Integer>();
- private Schema schema;
+ private Schema fromSchema;
+ private Schema toSchema;
/**
* {@inheritDoc}
@@ -87,19 +94,11 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
* {@inheritDoc}
*/
@Override
- public Schema getSchema() {
- return schema;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setSchema(Schema schema) {
+ public void setFromSchema(Schema schema) {
if(schema == null) {
return;
}
- this.schema = schema;
+ this.fromSchema = schema;
List<Column> columns = schema.getColumns();
int i = 0;
for(Column col : columns) {
@@ -113,6 +112,19 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
}
/**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setToSchema(Schema schema) {
+ if(schema == null) {
+ return;
+ }
+ this.toSchema = schema;
+ }
+
+
+
+ /**
* Custom CSV parser that honors quoting and escaped quotes.
* All other escaping is handled elsewhere.
*
@@ -168,6 +180,19 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
/**
* {@inheritDoc}
+ *
+ * The CSV data is ordered according to the fromSchema. We "translate" it to the TO schema.
+ * We currently have 3 methods of matching fields in one schema to another:
+ * - by location
+ * - by name
+ * - user-defined matching
+ *
+ * If one schema exists (either to or from) and the other is empty
+ * We'll match fields based on location.
+ * If both schemas exist, we'll match names of fields.
+ *
+ * In the future, we may want to let users choose the method
+ * Currently nothing is implemented for user-defined matching
*/
@Override
public Object[] getObjectData() {
@@ -176,51 +201,53 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
return null;
}
- if (schema == null) {
+ if (fromSchema == null || toSchema == null || (toSchema.isEmpty() && fromSchema.isEmpty())) {
throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0006);
}
- if (fields.length != schema.getColumns().size()) {
- throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0005,
- "The data " + getTextData() + " has the wrong number of fields.");
- }
+ AbstractMatcher matcher = getMatcher(fromSchema,toSchema);
+ String[] outFields = matcher.getMatchingData(fields, fromSchema, toSchema);
+ Object[] out = new Object[outFields.length];
+
+ int i = 0;
- Object[] out = new Object[fields.length];
- Column[] cols = schema.getColumns().toArray(new Column[fields.length]);
- for (int i = 0; i < fields.length; i++) {
- Type colType = cols[i].getType();
- //TODO: Replace with proper isNull method. Actually the entire content of the loop should be a parse method
- if (fields[i].equals("NULL") || fields[i].equals("null") || fields[i].equals("'null'") || fields[i].isEmpty()) {
+ // After getting back the data in order that matches the output schema
+ // We need to un-do the CSV escaping
+ for (Column col: matcher.getMatchingSchema(fromSchema,toSchema).getColumns()) {
+ Type colType = col.getType();
+ if (outFields[i] == null) {
out[i] = null;
continue;
}
if (colType == Type.TEXT) {
- out[i] = unescapeStrings(fields[i]);
+ out[i] = unescapeStrings(outFields[i]);
} else if (colType == Type.BINARY) {
- out[i] = unescapeByteArray(fields[i]);
+ out[i] = unescapeByteArray(outFields[i]);
} else if (colType == Type.FIXED_POINT) {
- Long byteSize = ((FixedPoint) cols[i]).getByteSize();
+ Long byteSize = ((FixedPoint) col).getByteSize();
if (byteSize != null && byteSize <= Integer.SIZE) {
- out[i] = Integer.valueOf(fields[i]);
+ out[i] = Integer.valueOf(outFields[i]);
} else {
- out[i] = Long.valueOf(fields[i]);
+ out[i] = Long.valueOf(outFields[i]);
}
} else if (colType == Type.FLOATING_POINT) {
- Long byteSize = ((FloatingPoint) cols[i]).getByteSize();
+ Long byteSize = ((FloatingPoint) col).getByteSize();
if (byteSize != null && byteSize <= Float.SIZE) {
- out[i] = Float.valueOf(fields[i]);
+ out[i] = Float.valueOf(outFields[i]);
} else {
- out[i] = Double.valueOf(fields[i]);
+ out[i] = Double.valueOf(outFields[i]);
}
} else if (colType == Type.DECIMAL) {
- out[i] = new BigDecimal(fields[i]);
+ out[i] = new BigDecimal(outFields[i]);
} else {
throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0004, "Column type from schema was not recognized for " + colType);
}
+ i++;
}
return out;
}
+
/**
* {@inheritDoc}
*/
@@ -353,4 +380,15 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
public String toString() {
return data;
}
+
+ private AbstractMatcher getMatcher(Schema fromSchema, Schema toSchema) {
+ if (toSchema.isEmpty() || fromSchema.isEmpty()) {
+ return new LocationMatcher();
+ } else {
+ return new NameMatcher();
+ }
+
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
index 74b9518..d98b779 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
@@ -81,7 +81,8 @@ public abstract class IntermediateDataFormat<T> {
/**
* Get one row of data as CSV.
*
- * @return - String representing the data in CSV
+ * @return - String representing the data in CSV, according to the "FROM" schema.
+ * No schema conversion is done on textData, to keep it as "high performance" option.
*/
public abstract String getTextData();
@@ -95,6 +96,7 @@ public abstract class IntermediateDataFormat<T> {
* Get one row of data as an Object array.
*
* @return - String representing the data as an Object array
+ * If FROM and TO schema exist, we will use SchemaMatcher to get the data according to "TO" schema
*/
public abstract Object[] getObjectData();
@@ -105,18 +107,18 @@ public abstract class IntermediateDataFormat<T> {
public abstract void setObjectData(Object[] data);
/**
- * Set the schema to be used.
+ * Set the schema for reading data.
*
- * @param schema - the schema to be used
+ * @param schema - the schema used for reading data
*/
- public abstract void setSchema(Schema schema);
+ public abstract void setFromSchema(Schema schema);
/**
- * Get the schema of the data.
+ * Set the schema for writing data.
*
- * @return - The schema of the data.
+ * @param schema - the schema used for writing data
*/
- public abstract Schema getSchema();
+ public abstract void setToSchema(Schema schema);
/**
* Serialize the fields of this object to <code>out</code>.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/AbstractMatcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/AbstractMatcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/AbstractMatcher.java
new file mode 100644
index 0000000..e6b2316
--- /dev/null
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/AbstractMatcher.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.idf.matcher;
+
+import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.type.Column;
+
+public abstract class AbstractMatcher {
+
+ //NOTE: This is currently tightly coupled to the CSV idf. We'll need refactoring after adding additional formats
+ //NOTE: There's is a very blatant special case of empty schemas that seem to apply only to HDFS.
+
+ /**
+ *
+ * @param fields
+ * @param fromSchema
+ * @param toSchema
+ * @return Return the data in "fields" converted from matching the fromSchema to matching the toSchema.
+ * Right not "converted" means re-ordering if needed and handling nulls.
+ */
+ abstract public String[] getMatchingData(String[] fields, Schema fromSchema, Schema toSchema);
+
+ /***
+ *
+ * @param fromSchema
+ * @param toSchema
+ * @return return a schema with which to read the output data
+ * This always returns the toSchema (since this is used when getting output data), unless its empty
+ */
+ public Schema getMatchingSchema(Schema fromSchema, Schema toSchema) {
+ if (toSchema.isEmpty()) {
+ return fromSchema;
+ } else {
+ return toSchema;
+ }
+
+ }
+
+ protected boolean isNull(String value) {
+ if (value.equals("NULL") || value.equals("null") || value.equals("'null'") || value.isEmpty()) {
+ return true;
+ }
+ return false;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/LocationMatcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/LocationMatcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/LocationMatcher.java
new file mode 100644
index 0000000..938a5df
--- /dev/null
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/LocationMatcher.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.idf.matcher;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.idf.IntermediateDataFormatError;
+import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.SchemaError;
+import org.apache.sqoop.schema.SchemaMatchOption;
+import org.apache.sqoop.schema.type.Column;
+import org.apache.sqoop.schema.type.FixedPoint;
+import org.apache.sqoop.schema.type.FloatingPoint;
+import org.apache.sqoop.schema.type.Type;
+
+import java.math.BigDecimal;
+import java.util.Iterator;
+
+
+/**
+ * Convert data according to FROM schema to data according to TO schema
+ * This is done based on column location
+ * So data in first column in FROM goes into first column in TO, etc
+ * If TO schema has more fields and they are "nullable", the value will be set to null
+ * If TO schema has extra non-null fields, we'll throw an exception
+ */
+public class LocationMatcher extends AbstractMatcher {
+
+ public static final Logger LOG = Logger.getLogger(LocationMatcher.class);
+ @Override
+ public String[] getMatchingData(String[] fields, Schema fromSchema, Schema toSchema) {
+
+ String[] out = new String[toSchema.getColumns().size()];
+
+ int i = 0;
+
+ if (toSchema.isEmpty()) {
+ // If there's no destination schema, no need to convert anything
+ // Just use the original data
+ return fields;
+ }
+
+ for (Column col: toSchema.getColumns())
+ {
+ if (i < fields.length) {
+ if (isNull(fields[i])) {
+ out[i] = null;
+ } else {
+ out[i] = fields[i];
+ }
+ }
+ // We ran out of fields before we ran out of schema
+ else {
+ if (!col.getNullable()) {
+ throw new SqoopException(SchemaError.SCHEMA_0004,"target column " + col + " didn't match with any source column and cannot be null");
+ } else {
+ LOG.warn("Column " + col + " has no matching source column. Will be ignored. ");
+ out[i] = null;
+ }
+ }
+ i++;
+ }
+ return out;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/NameMatcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/NameMatcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/NameMatcher.java
new file mode 100644
index 0000000..417c85b
--- /dev/null
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/NameMatcher.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.idf.matcher;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.SchemaError;
+import org.apache.sqoop.schema.type.Column;
+
+import java.util.HashMap;
+
+public class NameMatcher extends AbstractMatcher {
+ public static final Logger LOG = Logger.getLogger(NameMatcher.class);
+
+ @Override
+ public String[] getMatchingData(String[] fields, Schema fromSchema, Schema toSchema) {
+ String[] out = new String[toSchema.getColumns().size()];
+
+ HashMap<String,Column> colNames = new HashMap<String, Column>();
+
+ for (Column fromCol: fromSchema.getColumns()) {
+ colNames.put(fromCol.getName(), fromCol);
+ }
+
+ int toIndex = 0;
+
+ for (Column toCol: toSchema.getColumns()) {
+ Column fromCol = colNames.get(toCol.getName());
+
+ if (fromCol != null) {
+ int fromIndex = fromSchema.getColumns().indexOf(fromCol);
+ if (isNull(fields[fromIndex])) {
+ out[toIndex] = null;
+ } else {
+ out[toIndex] = fields[fromIndex];
+ }
+ } else {
+ //column exists in TO schema but not in FROM schema
+ if (toCol.getNullable() == false) {
+ throw new SqoopException(SchemaError.SCHEMA_0004,"target column " + toCol + " didn't match with any source column and cannot be null");
+ } else {
+ LOG.warn("Column " + toCol + " has no matching source column. Will be ignored. ");
+ out[toIndex] = null;
+ }
+ }
+
+ toIndex++;
+ }
+
+ return out;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
index 8c83a71..3954039 100644
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
@@ -20,6 +20,7 @@ package org.apache.sqoop.connector.idf;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.SchemaMatchOption;
import org.apache.sqoop.schema.type.Binary;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.Text;
@@ -39,6 +40,8 @@ public class TestCSVIntermediateDataFormat {
private IntermediateDataFormat<?> data;
+ private Schema emptySchema = new Schema("empty");
+
@Before
public void setUp() {
data = new CSVIntermediateDataFormat();
@@ -70,7 +73,7 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setSchema(schema);
+ data.setFromSchema(schema);
data.setTextData(null);
Object[] out = data.getObjectData();
@@ -87,7 +90,7 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setSchema(schema);
+ data.setFromSchema(schema);
data.setTextData("");
data.getObjectData();
@@ -106,7 +109,9 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setSchema(schema);
+
+ data.setFromSchema(schema);
+ data.setToSchema(emptySchema);
data.setTextData(testData);
Object[] out = data.getObjectData();
@@ -129,7 +134,7 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setSchema(schema);
+ data.setFromSchema(schema);
byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54};
Object[] in = new Object[6];
@@ -159,7 +164,8 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setSchema(schema);
+ data.setFromSchema(schema);
+ data.setToSchema(emptySchema);
Object[] in = new Object[6];
in[0] = new Long(10);
@@ -181,7 +187,9 @@ public class TestCSVIntermediateDataFormat {
public void testStringFullRangeOfCharacters() {
Schema schema = new Schema("test");
schema.addColumn(new Text("1"));
- data.setSchema(schema);
+
+ data.setFromSchema(schema);
+ data.setToSchema(emptySchema);
char[] allCharArr = new char[256];
for(int i = 0; i < allCharArr.length; ++i) {
@@ -204,7 +212,8 @@ public class TestCSVIntermediateDataFormat {
public void testByteArrayFullRangeOfCharacters() {
Schema schema = new Schema("test");
schema.addColumn(new Binary("1"));
- data.setSchema(schema);
+ data.setFromSchema(schema);
+ data.setToSchema(emptySchema);
byte[] allCharByteArr = new byte[256];
for(int i = 0; i < allCharByteArr.length; ++i) {
@@ -219,4 +228,132 @@ public class TestCSVIntermediateDataFormat {
data.setObjectData(in);
assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
}
+
+ /**
+ * Note that we don't have an EmptyTo matching test
+ * Because most tests above have empty "to" schema
+ */
+ @Test
+ public void testMatchingEmptyFrom() {
+
+ data.setFromSchema(emptySchema);
+
+ Schema toSchema = new Schema("To");
+ toSchema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"));
+ data.setToSchema(toSchema);
+
+ Object[] in = new Object[2];
+ in[0] = new Long(10);
+ in[1] = new Long(34);
+
+ Object[] out = new Object[2];
+ out[0] = new Long(10);
+ out[1] = new Long(34);
+
+ data.setObjectData(in);
+
+ assertTrue(Arrays.deepEquals(out, data.getObjectData()));
+ }
+
+ @Test(expected=SqoopException.class)
+ public void testMatchingTwoEmptySchema() {
+ data.setFromSchema(emptySchema);
+ data.setToSchema(emptySchema);
+
+ Object[] in = new Object[2];
+ in[0] = new Long(10);
+ in[1] = new Long(34);
+
+ data.setObjectData(in);
+
+ data.getObjectData();
+ }
+
+ @Test
+ public void testMatchingFewerFromColumns(){
+ Schema fromSchema = new Schema("From");
+ fromSchema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"));
+ data.setFromSchema(fromSchema);
+
+ Schema toSchema = new Schema("To");
+ toSchema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"))
+ .addColumn(new Text("3"));
+ data.setToSchema(toSchema);
+
+ Object[] in = new Object[2];
+ in[0] = new Long(10);
+ in[1] = new Long(34);
+
+ Object[] out = new Object[3];
+ out[0] = new Long(10);
+ out[1] = new Long(34);
+ out[2] = null;
+
+ data.setObjectData(in);
+
+ assertTrue(Arrays.deepEquals(out, data.getObjectData()));
+ }
+
+ @Test
+ public void testMatchingFewerToColumns(){
+ Schema fromSchema = new Schema("From");
+ fromSchema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"))
+ .addColumn(new FixedPoint("3"));
+ data.setFromSchema(fromSchema);
+
+ Schema toSchema = new Schema("To");
+ toSchema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"));
+ data.setToSchema(toSchema);
+
+ Object[] in = new Object[3];
+ in[0] = new Long(10);
+ in[1] = new Long(34);
+ in[2] = new Long(50);
+
+ Object[] out = new Object[2];
+ out[0] = new Long(10);
+ out[1] = new Long(34);
+
+
+ data.setObjectData(in);
+
+ assertTrue(Arrays.deepEquals(out, data.getObjectData()));
+ }
+
+
+ @Test
+ public void testWithSomeNonMatchingFields(){
+
+ Schema fromSchema = new Schema("From");
+ fromSchema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"))
+ .addColumn(new FixedPoint("3"));
+ data.setFromSchema(fromSchema);
+
+ Schema toSchema = new Schema("From");
+ toSchema.addColumn(new FixedPoint("2"))
+ .addColumn(new FixedPoint("3"))
+ .addColumn(new FixedPoint("4"));
+ data.setToSchema(toSchema);
+
+ Object[] in = new Object[3];
+ in[0] = new Long(10);
+ in[1] = new Long(34);
+ in[2] = new Long(50);
+
+ Object[] out = new Object[3];
+ out[0] = new Long(34);
+ out[1] = new Long(50);
+ out[2] = null;
+
+ data.setObjectData(in);
+
+ assertTrue(Arrays.deepEquals(out, data.getObjectData()));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/core/src/main/java/org/apache/sqoop/driver/JobManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobManager.java b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
index 277c6be..e91c436 100644
--- a/core/src/main/java/org/apache/sqoop/driver/JobManager.java
+++ b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
@@ -374,13 +374,10 @@ public class JobManager implements Reconfigurable {
Schema fromSchema = getSchemaFromConnector(jobRequest, Direction.FROM);
Schema toSchema = getSchemaFromConnector(jobRequest, Direction.TO);
- // TODO(Gwen): Need better logic here once the Schema refactor: SQOOP-1378
- if (fromSchema != null) {
- jobRequest.getSummary().setFromSchema(fromSchema);
- }
- else {
- jobRequest.getSummary().setFromSchema(toSchema);
- }
+
+ jobRequest.getSummary().setFromSchema(fromSchema);
+ jobRequest.getSummary().setToSchema(toSchema);
+
LOG.debug("Using entities: " + jobRequest.getFrom() + ", " + jobRequest.getTo());
return jobRequest;
}
@@ -458,7 +455,7 @@ public class JobManager implements Reconfigurable {
initializer.initialize(initializerContext, jobRequest.getConnectorLinkConfig(direction),
jobRequest.getConnectorJobConfig(direction));
- // TODO(Abe): Alter behavior of Schema here.
+
return initializer.getSchema(initializerContext,
jobRequest.getConnectorLinkConfig(direction),
jobRequest.getConnectorJobConfig(direction));
@@ -709,4 +706,4 @@ public class JobManager implements Reconfigurable {
LOG.info("Ending submission manager update thread");
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/docs/src/site/sphinx/Tools.rst
----------------------------------------------------------------------
diff --git a/docs/src/site/sphinx/Tools.rst b/docs/src/site/sphinx/Tools.rst
index 84cbd5f..6d36b27 100644
--- a/docs/src/site/sphinx/Tools.rst
+++ b/docs/src/site/sphinx/Tools.rst
@@ -23,8 +23,7 @@ Tools are server commands that administrators can execute on the Sqoop server ma
In order to perform the maintenance task each tool is suppose to do, they need to be executed in exactly the same environment as the main Sqoop server. The tool binary will take care of setting up the ``CLASSPATH`` and other environmental variables that might be required. However it's up to the administrator himself to run the tool under the same user as is used for the server. This is usually configured automatically for various Hadoop distributions (such as Apache Bigtop).
-.. note:: Running tools under a different user such as ``root`` might prevent Sqoop Server from running correctly.
-
+.. note:: Running tools while the Sqoop Server is also running is not recommended as it might lead to a data corruption and service disruption.
List of available tools:
* verify
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
index 2ed06a8..b533837 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/ConfigurationUtils.java
@@ -82,13 +82,13 @@ public final class ConfigurationUtils {
private static final Text JOB_CONFIG_FRAMEWORK_JOB_KEY = new Text(JOB_CONFIG_FRAMEWORK_JOB);
- private static final String SCHEMA_FROM_CONNECTOR = JobConstants.PREFIX_JOB_CONFIG + "schema.connector.from";
+ private static final String SCHEMA_FROM = JobConstants.PREFIX_JOB_CONFIG + "schema.connector.from";
- private static final Text SCHEMA_FROM_CONNECTOR_KEY = new Text(SCHEMA_FROM_CONNECTOR);
+ private static final Text SCHEMA_FROM_KEY = new Text(SCHEMA_FROM);
- private static final String SCHEMA_TO_CONNECTOR = JobConstants.PREFIX_JOB_CONFIG + "schema.connector.to";
+ private static final String SCHEMA_TO = JobConstants.PREFIX_JOB_CONFIG + "schema.connector.to";
- private static final Text SCHEMA_TO_CONNECTOR_KEY = new Text(SCHEMA_TO_CONNECTOR);
+ private static final Text SCHEMA_TO_KEY = new Text(SCHEMA_TO);
/**
@@ -163,6 +163,27 @@ public final class ConfigurationUtils {
}
/**
+ * Persist Connector generated schema.
+ *
+ * @param type Direction of schema we are persisting
+ * @param job MapReduce Job object
+ * @param schema Schema
+ */
+ public static void setConnectorSchema(Direction type, Job job, Schema schema) {
+ if(schema != null) {
+ String jsonSchema = SchemaSerialization.extractSchema(schema).toJSONString();
+ switch (type) {
+ case FROM:
+ job.getCredentials().addSecretKey(SCHEMA_FROM_KEY,jsonSchema.getBytes());
+ return;
+ case TO:
+ job.getCredentials().addSecretKey(SCHEMA_TO_KEY, jsonSchema.getBytes());
+ return;
+ }
+ }
+ }
+
+ /**
* Retrieve Connector configuration object for connection.
*
* @param configuration MapReduce configuration object
@@ -226,23 +247,7 @@ public final class ConfigurationUtils {
return loadConfiguration((JobConf) configuration, JOB_CONFIG_CLASS_FRAMEWORK_JOB, JOB_CONFIG_FRAMEWORK_JOB_KEY);
}
- /**
- * Persist From Connector generated schema.
- *
- * @param job MapReduce Job object
- * @param schema Schema
- */
- public static void setConnectorSchema(Direction type, Job job, Schema schema) {
- if(schema != null) {
- switch (type) {
- case FROM:
- job.getCredentials().addSecretKey(SCHEMA_FROM_CONNECTOR_KEY, SchemaSerialization.extractSchema(schema).toJSONString().getBytes());
- case TO:
- job.getCredentials().addSecretKey(SCHEMA_TO_CONNECTOR_KEY, SchemaSerialization.extractSchema(schema).toJSONString().getBytes());
- }
- }
- }
/**
* Retrieve Connector generated schema.
@@ -253,10 +258,10 @@ public final class ConfigurationUtils {
public static Schema getConnectorSchema(Direction type, Configuration configuration) {
switch (type) {
case FROM:
- return getSchemaFromBytes(((JobConf) configuration).getCredentials().getSecretKey(SCHEMA_FROM_CONNECTOR_KEY));
+ return getSchemaFromBytes(((JobConf) configuration).getCredentials().getSecretKey(SCHEMA_FROM_KEY));
case TO:
- return getSchemaFromBytes(((JobConf) configuration).getCredentials().getSecretKey(SCHEMA_TO_CONNECTOR_KEY));
+ return getSchemaFromBytes(((JobConf) configuration).getCredentials().getSecretKey(SCHEMA_TO_KEY));
}
return null;
@@ -274,7 +279,9 @@ public final class ConfigurationUtils {
if(bytes == null) {
return null;
}
- return SchemaSerialization.restoreSchemna((JSONObject) JSONValue.parse(new String(bytes)));
+
+ JSONObject jsonSchema = (JSONObject) JSONValue.parse(new String(bytes));
+ return SchemaSerialization.restoreSchema(jsonSchema);
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
index 6680f60..8c88d52 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
@@ -64,20 +64,18 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
String extractorName = conf.get(JobConstants.JOB_ETL_EXTRACTOR);
Extractor extractor = (Extractor) ClassUtils.instantiate(extractorName);
- // TODO(Abe/Gwen): Change to conditional choosing between Connector schemas.
- Schema schema = ConfigurationUtils.getConnectorSchema(Direction.FROM, conf);
- if (schema == null) {
- schema = ConfigurationUtils.getConnectorSchema(Direction.TO, conf);
- }
- if (schema == null) {
- LOG.info("setting an empty schema");
- }
+
+ Schema fromSchema = ConfigurationUtils.getConnectorSchema(Direction.FROM, conf);
+ Schema toSchema = ConfigurationUtils.getConnectorSchema(Direction.TO, conf);
String intermediateDataFormatName = conf.get(JobConstants.INTERMEDIATE_DATA_FORMAT);
dataFormat = (IntermediateDataFormat<String>) ClassUtils
.instantiate(intermediateDataFormatName);
- dataFormat.setSchema(schema);
+
+ dataFormat.setFromSchema(fromSchema);
+ dataFormat.setToSchema(toSchema);
+
dataOut = new SqoopWritable();
// Objects that should be passed to the Executor execution
@@ -86,7 +84,7 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
Object fromJob = ConfigurationUtils.getConnectorJobConfig(Direction.FROM, conf);
SqoopSplit split = context.getCurrentKey();
- ExtractorContext extractorContext = new ExtractorContext(subContext, new SqoopMapDataWriter(context), schema);
+ ExtractorContext extractorContext = new ExtractorContext(subContext, new SqoopMapDataWriter(context));
try {
LOG.info("Starting progress service");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
index eea0623..941b31d 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
@@ -73,12 +73,11 @@ public class SqoopOutputFormatLoadExecutor {
dataFormat = (IntermediateDataFormat<String>) ClassUtils.instantiate(context
.getConfiguration().get(JobConstants.INTERMEDIATE_DATA_FORMAT));
- Schema schema = ConfigurationUtils.getConnectorSchema(Direction.FROM, context.getConfiguration());
- if (schema==null) {
- schema = ConfigurationUtils.getConnectorSchema(Direction.TO, context.getConfiguration());
- }
+ Schema fromSchema = ConfigurationUtils.getConnectorSchema(Direction.FROM, context.getConfiguration());
+ dataFormat.setFromSchema(fromSchema);
- dataFormat.setSchema(schema);
+ Schema toSchema = ConfigurationUtils.getConnectorSchema(Direction.TO, context.getConfiguration());
+ dataFormat.setToSchema(toSchema);
}
public RecordWriter<SqoopWritable, NullWritable> getRecordWriter() {
@@ -231,10 +230,8 @@ public class SqoopOutputFormatLoadExecutor {
Schema schema = null;
if (!isTest) {
- // Propagate connector schema in every case for now
- // TODO: Change to coditional choosing between Connector schemas.
- // @TODO(Abe): Maybe use TO schema?
- schema = ConfigurationUtils.getConnectorSchema(Direction.FROM, conf);
+ // Using the TO schema since the IDF returns data in TO schema
+ schema = ConfigurationUtils.getConnectorSchema(Direction.TO, conf);
subContext = new PrefixContext(conf, JobConstants.PREFIX_CONNECTOR_TO_CONTEXT);
configConnection = ConfigurationUtils.getConnectorConnectionConfig(Direction.TO, conf);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
index 60acfb6..0e2a38d 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/SubmissionDisplayer.java
@@ -68,12 +68,12 @@ public final class SubmissionDisplayer {
}
}
- if(isVerbose() && submission.getFromSchema() != null) {
+ if(isVerbose() && submission.getFromSchema() != null && !submission.getFromSchema().isEmpty() ) {
print(resourceString(Constants.RES_FROM_SCHEMA)+": ");
println(submission.getFromSchema());
}
- if(isVerbose() && submission.getToSchema() != null) {
+ if(isVerbose() && submission.getToSchema() != null && !submission.getToSchema().isEmpty() ) {
print(resourceString(Constants.RES_TO_SCHEMA)+": ");
println(submission.getToSchema());
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/shell/src/main/resources/shell-resource.properties
----------------------------------------------------------------------
diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties
index b59bd81..c0f86f7 100644
--- a/shell/src/main/resources/shell-resource.properties
+++ b/shell/src/main/resources/shell-resource.properties
@@ -227,5 +227,6 @@ submission.progress_not_available = Progress is not available
submission.counters = Counters
submission.executed_success = Job executed successfully
submission.server_url = Server URL
-submission.from_schema = From schema
-submission.to_schema = To schema
\ No newline at end of file
+submission.from_schema = Source Connector schema
+submission.to_schema = Target Connector schema
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2c20d920/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
index fe92ac4..25255ae 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
@@ -206,11 +206,13 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getConnectorJobConfig(Direction.FROM));
ConfigurationUtils.setConnectorConnectionConfig(Direction.TO, job, request.getConnectorLinkConfig(Direction.TO));
ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getConnectorJobConfig(Direction.TO));
+
ConfigurationUtils.setFrameworkConnectionConfig(Direction.FROM, job, request.getFrameworkLinkConfig(Direction.FROM));
ConfigurationUtils.setFrameworkConnectionConfig(Direction.TO, job, request.getFrameworkLinkConfig(Direction.TO));
ConfigurationUtils.setFrameworkJobConfig(job, request.getFrameworkJobConfig());
- // @TODO(Abe): Persist TO schema.
+
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, request.getSummary().getFromSchema());
+ ConfigurationUtils.setConnectorSchema(Direction.TO, job, request.getSummary().getToSchema());
if(request.getJobName() != null) {
job.setJobName("Sqoop: " + request.getJobName());
@@ -413,4 +415,5 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
return "local".equals(globalConfiguration.get("mapreduce.jobtracker.address"))
|| "local".equals(globalConfiguration.get("mapred.job.tracker"));
}
-}
\ No newline at end of file
+
+}
[33/52] [abbrv] git commit: SQOOP-1561: Sqoop2: Date and DateTime
handling in CSV IDF
Posted by ab...@apache.org.
SQOOP-1561: Sqoop2: Date and DateTime handling in CSV IDF
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/1e9db011
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/1e9db011
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/1e9db011
Branch: refs/heads/SQOOP-1367
Commit: 1e9db0113fe6b83f843faabc6a58eb07b19b0fee
Parents: 8362c73
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Sat Oct 4 14:02:41 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 18:22:51 2014 -0700
----------------------------------------------------------------------
connector/connector-sdk/pom.xml | 4 +++
.../idf/CSVIntermediateDataFormat.java | 6 ++++
.../idf/TestCSVIntermediateDataFormat.java | 27 ++++++++++++++++
pom.xml | 33 ++++++++++++--------
4 files changed, 57 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e9db011/connector/connector-sdk/pom.xml
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/pom.xml b/connector/connector-sdk/pom.xml
index f54837d..6720027 100644
--- a/connector/connector-sdk/pom.xml
+++ b/connector/connector-sdk/pom.xml
@@ -34,6 +34,10 @@ limitations under the License.
<dependencies>
<dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e9db011/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
index 02d1a51..3c3543c 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
@@ -27,6 +27,8 @@ import org.apache.sqoop.schema.type.Column;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.FloatingPoint;
import org.apache.sqoop.schema.type.Type;
+import org.joda.time.LocalDate;
+import org.joda.time.LocalDateTime;
import java.io.DataInput;
import java.io.DataOutput;
@@ -215,7 +217,11 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
out[i] = new BigDecimal(fields[i]);
break;
case DATE:
+ out[i] = LocalDate.parse(fields[i]);
+ break;
case DATE_TIME:
+ out[i] = LocalDateTime.parse(fields[i]);
+ break;
case BIT:
out[i] = fields[i];
break;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e9db011/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
index f5fbab7..3159883 100644
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
@@ -28,6 +28,8 @@ import java.util.Arrays;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Binary;
+import org.apache.sqoop.schema.type.Date;
+import org.apache.sqoop.schema.type.DateTime;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.Text;
import org.junit.Before;
@@ -222,6 +224,31 @@ public class TestCSVIntermediateDataFormat {
assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
}
+ @Test
+ public void testDate() {
+ Schema schema = new Schema("test");
+ schema.addColumn(new Date("1"));
+ data.setSchema(schema);
+
+ data.setTextData("2014-10-01");
+ assertEquals("2014-10-01", data.getObjectData()[0].toString());
+ }
+
+ @Test
+ public void testDateTime() {
+ Schema schema = new Schema("test");
+ schema.addColumn(new DateTime("1"));
+ data.setSchema(schema);
+
+ for (String dateTime : new String[]{
+ "2014-10-01T12:00:00",
+ "2014-10-01T12:00:00.000"
+ }) {
+ data.setTextData(dateTime);
+ assertEquals("2014-10-01T12:00:00.000", data.getObjectData()[0].toString());
+ }
+ }
+
@Test(expected=SqoopException.class)
public void testEmptySchema() {
String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e9db011/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f2cd070..f25a29f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -14,7 +14,8 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
---><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
@@ -111,6 +112,7 @@ limitations under the License.
<jdbc.sqlserver.version>4.0</jdbc.sqlserver.version>
<jdbc.teradata.version>14.00.00.21</jdbc.teradata.version>
<jdbc.netezza.version>6.0</jdbc.netezza.version>
+ <joda.version>2.4</joda.version>
</properties>
<dependencies>
@@ -440,18 +442,23 @@ limitations under the License.
<artifactId>nzjdbc3</artifactId>
<version>${jdbc.netezza.version}</version>
</dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.tomcat</groupId>
- <artifactId>catalina</artifactId>
- <version>${tomcat.version}</version>
- <scope>provided</scope>
- </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>${joda.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>catalina</artifactId>
+ <version>${tomcat.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
[08/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java b/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
index 93b3643..34f02c9 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
@@ -21,18 +21,18 @@ package org.apache.sqoop.job.etl;
* This allows connector to extract data from a source system
* based on each partition.
*/
-public abstract class Extractor<ConnectionConfiguration, JobConfiguration, Partition> {
+public abstract class Extractor<LinkConfiguration, JobConfiguration, Partition> {
/**
- * Extract data from source and pass them into the framework.
+ * Extract data from source and pass them into the Sqoop.
*
* @param context Extractor context object
- * @param connectionConfiguration Connection configuration
+ * @param linkConfiguration link configuration
* @param jobConfiguration Job configuration
* @param partition Partition that this extract should work on
*/
public abstract void extract(ExtractorContext context,
- ConnectionConfiguration connectionConfiguration,
+ LinkConfiguration linkConfiguration,
JobConfiguration jobConfiguration,
Partition partition);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/job/etl/From.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/From.java b/spi/src/main/java/org/apache/sqoop/job/etl/From.java
index 80f4f29..3dd8fb9 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/From.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/From.java
@@ -23,7 +23,7 @@ package org.apache.sqoop.job.etl;
* Initializer
* -> Partitioner
* -> Extractor
- * -> (framework-defined steps)
+ * -> (Sqoop-defined steps)
* -> Destroyer
*/
public class From extends Transferable {
@@ -50,7 +50,7 @@ public class From extends Transferable {
@Override
public String toString() {
- return "Importer{" + super.toString() +
+ return "From{" + super.toString() +
", partitioner=" + partitioner.getName() +
", extractor=" + extractor.getName() +
'}';
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
index 88744ea..c9ae540 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
@@ -17,16 +17,16 @@
*/
package org.apache.sqoop.job.etl;
-import org.apache.sqoop.schema.Schema;
-
import java.util.LinkedList;
import java.util.List;
+import org.apache.sqoop.schema.Schema;
+
/**
* This allows connector to define initialization work for execution,
* for example, context configuration.
*/
-public abstract class Initializer<ConnectionConfiguration, JobConfiguration> {
+public abstract class Initializer<LinkConfiguration, JobConfiguration> {
/**
* Initialize new submission based on given configuration properties. Any
@@ -34,11 +34,11 @@ public abstract class Initializer<ConnectionConfiguration, JobConfiguration> {
* promoted to all other part of the workflow automatically.
*
* @param context Initializer context object
- * @param connectionConfiguration Connector's connection configuration object
+ * @param linkConfiguration Connector's link configuration object
* @param jobConfiguration Connector's job configuration object
*/
public abstract void initialize(InitializerContext context,
- ConnectionConfiguration connectionConfiguration,
+ LinkConfiguration linkConfiguration,
JobConfiguration jobConfiguration);
/**
@@ -49,13 +49,13 @@ public abstract class Initializer<ConnectionConfiguration, JobConfiguration> {
* @return
*/
public List<String> getJars(InitializerContext context,
- ConnectionConfiguration connectionConfiguration,
+ LinkConfiguration linkConfiguration,
JobConfiguration jobConfiguration) {
return new LinkedList<String>();
}
public abstract Schema getSchema(InitializerContext context,
- ConnectionConfiguration connectionConfiguration,
+ LinkConfiguration linkConfiguration,
JobConfiguration jobConfiguration);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java b/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
index 0b32314..434dcf6 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
@@ -20,18 +20,18 @@ package org.apache.sqoop.job.etl;
/**
* This allows connector to load data into a target system.
*/
-public abstract class Loader<ConnectionConfiguration, JobConfiguration> {
+public abstract class Loader<LinkConfiguration, JobConfiguration> {
/**
* Load data to target.
*
* @param context Loader context object
- * @param connectionConfiguration Connection configuration
+ * @param linkConfiguration link configuration
* @param jobConfiguration Job configuration
* @throws Exception
*/
public abstract void load(LoaderContext context,
- ConnectionConfiguration connectionConfiguration,
+ LinkConfiguration linkConfiguration,
JobConfiguration jobConfiguration) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java b/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
index f977233..8156762 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
@@ -23,7 +23,7 @@ import java.util.List;
* This allows connector to define how input data to be partitioned.
* The number of data partitions also determines the degree of parallelism.
*/
-public abstract class Partitioner<ConnectionConfiguration, JobConfiguration> {
+public abstract class Partitioner<LinkConfiguration, JobConfiguration> {
/**
* Partition input data into partitions.
@@ -31,12 +31,12 @@ public abstract class Partitioner<ConnectionConfiguration, JobConfiguration> {
* Each partition will be then processed in separate extractor.
*
* @param context Partitioner context object
- * @param connectionConfiguration Connection configuration
+ * @param linkConfiguration link configuration
* @param jobConfiguration Job configuration
* @return
*/
public abstract List<Partition> getPartitions(PartitionerContext context,
- ConnectionConfiguration connectionConfiguration,
+ LinkConfiguration linkConfiguration,
JobConfiguration jobConfiguration);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/job/etl/To.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/To.java b/spi/src/main/java/org/apache/sqoop/job/etl/To.java
index b8717ae..64ba225 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/To.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/To.java
@@ -21,7 +21,7 @@ package org.apache.sqoop.job.etl;
* This specifies classes that perform connector-defined steps
* within export execution:
* Initializer
- * -> (framework-defined steps)
+ * -> (Sqoop-defined steps)
* -> Loader
* -> Destroyer
*/
@@ -44,7 +44,7 @@ public class To extends Transferable {
@Override
public String toString() {
- return "Exporter{" + super.toString() +
+ return "To {" + super.toString() +
", loader=" + loader +
'}';
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/spi/src/main/java/org/apache/sqoop/validation/Validator.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/validation/Validator.java b/spi/src/main/java/org/apache/sqoop/validation/Validator.java
index f31adb5..894f412 100644
--- a/spi/src/main/java/org/apache/sqoop/validation/Validator.java
+++ b/spi/src/main/java/org/apache/sqoop/validation/Validator.java
@@ -19,20 +19,20 @@ package org.apache.sqoop.validation;
/**
- * Connection and job metadata validator.
+ * Link and Job config validator.
*
* This class should be extended by connector to provide configuration
- * validation for connection and job configuration objects.
+ * validation for link and job configuration objects.
*/
public class Validator {
/**
- * Validate connection configuration object.
+ * Validate link configuration object.
*
- * @param connectionConfiguration Connection object to be validated
+ * @param linkConfiguration Connection object to be validated
* @return Validation status
*/
- public Validation validateConnection(Object connectionConfiguration) {
+ public Validation validateLink(Object linkConfiguration) {
return new Validation(EmptyClass.class);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
index 93741e6..fe92ac4 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
@@ -17,6 +17,12 @@
*/
package org.apache.sqoop.submission.mapreduce;
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.Map;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobClient;
@@ -29,22 +35,16 @@ import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.MapContext;
import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.driver.SubmissionEngine;
import org.apache.sqoop.execution.mapreduce.MRJobRequest;
import org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine;
-import org.apache.sqoop.framework.JobRequest;
-import org.apache.sqoop.framework.SubmissionEngine;
+import org.apache.sqoop.driver.JobRequest;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.mr.ConfigurationUtils;
+import org.apache.sqoop.submission.SubmissionStatus;
import org.apache.sqoop.submission.counter.Counter;
import org.apache.sqoop.submission.counter.CounterGroup;
import org.apache.sqoop.submission.counter.Counters;
-import org.apache.sqoop.submission.SubmissionStatus;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.util.Map;
/**
@@ -156,10 +156,10 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
// Clone global configuration
Configuration configuration = new Configuration(globalConfiguration);
- // Serialize framework context into job configuration
- for(Map.Entry<String, String> entry: request.getFrameworkContext()) {
+ // Serialize driver context into job configuration
+ for(Map.Entry<String, String> entry: request.getDriverContext()) {
if (entry.getValue() == null) {
- LOG.warn("Ignoring null framework context value for key " + entry.getKey());
+ LOG.warn("Ignoring null driver context value for key " + entry.getKey());
continue;
}
configuration.set(entry.getKey(), entry.getValue());
@@ -202,13 +202,13 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
Job job = new Job(configuration);
// And finally put all configuration objects to credentials cache
- ConfigurationUtils.setConnectorConnectionConfig(Direction.FROM, job, request.getConnectorConnectionConfig(Direction.FROM));
+ ConfigurationUtils.setConnectorConnectionConfig(Direction.FROM, job, request.getConnectorLinkConfig(Direction.FROM));
ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getConnectorJobConfig(Direction.FROM));
- ConfigurationUtils.setConnectorConnectionConfig(Direction.TO, job, request.getConnectorConnectionConfig(Direction.TO));
+ ConfigurationUtils.setConnectorConnectionConfig(Direction.TO, job, request.getConnectorLinkConfig(Direction.TO));
ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getConnectorJobConfig(Direction.TO));
- ConfigurationUtils.setFrameworkConnectionConfig(Direction.FROM, job, request.getFrameworkConnectionConfig(Direction.FROM));
- ConfigurationUtils.setFrameworkConnectionConfig(Direction.TO, job, request.getFrameworkConnectionConfig(Direction.TO));
- ConfigurationUtils.setFrameworkJobConfig(job, request.getConfigFrameworkJob());
+ ConfigurationUtils.setFrameworkConnectionConfig(Direction.FROM, job, request.getFrameworkLinkConfig(Direction.FROM));
+ ConfigurationUtils.setFrameworkConnectionConfig(Direction.TO, job, request.getFrameworkLinkConfig(Direction.TO));
+ ConfigurationUtils.setFrameworkJobConfig(job, request.getFrameworkJobConfig());
// @TODO(Abe): Persist TO schema.
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, request.getSummary().getFromSchema());
@@ -413,4 +413,4 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
return "local".equals(globalConfiguration.get("mapreduce.jobtracker.address"))
|| "local".equals(globalConfiguration.get("mapred.job.tracker"));
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java b/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java
index e0cc7c9..3ebdd5e 100644
--- a/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java
+++ b/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java
@@ -43,7 +43,7 @@ abstract public class DatabaseProvider {
/**
* Internal connection to the database.
*/
- private Connection connection;
+ private Connection databaseConnection;
/**
* JDBC Url to the remote database system.
@@ -140,9 +140,9 @@ abstract public class DatabaseProvider {
*/
public void stop() {
// Close connection to the database server
- if(connection != null) {
+ if(databaseConnection != null) {
try {
- connection.close();
+ databaseConnection.close();
} catch (SQLException e) {
LOG.info("Ignored exception on closing connection", e);
}
@@ -155,7 +155,7 @@ abstract public class DatabaseProvider {
* @return
*/
public Connection getConnection() {
- return connection;
+ return databaseConnection;
}
/**
@@ -164,7 +164,7 @@ abstract public class DatabaseProvider {
* @param connection New connection object
*/
protected void setConnection(Connection connection) {
- this.connection = connection;
+ databaseConnection = connection;
}
/**
@@ -179,7 +179,7 @@ abstract public class DatabaseProvider {
Statement stmt = null;
try {
- stmt = connection.createStatement();
+ stmt = databaseConnection.createStatement();
stmt.executeUpdate(query);
} catch (SQLException e) {
LOG.error("Error in executing query", e);
@@ -208,7 +208,7 @@ abstract public class DatabaseProvider {
Statement stmt = null;
try {
- stmt = connection.createStatement();
+ stmt = databaseConnection.createStatement();
return stmt.executeQuery(query);
} catch (SQLException e) {
LOG.error("Error in executing query", e);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java b/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java
index bc430f1..330863a 100644
--- a/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java
+++ b/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java
@@ -32,7 +32,7 @@ public class DatabaseProviderFactory {
return new DerbyProvider();
}
- Class klass = Class.forName(className);
+ Class<?> klass = Class.forName(className);
return (DatabaseProvider)klass.newInstance();
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
index 8a699f5..6074d36 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
@@ -22,8 +22,8 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.log4j.Logger;
import org.apache.sqoop.client.SubmissionCallback;
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MFormList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MPersistableEntity;
@@ -121,39 +121,38 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
}
/**
- * Fill connection form based on currently active provider.
+ * Fill link form based on currently active provider.
*
- * @param connection MConnection object to fill
+ * @param link MLink object to fill
*/
- protected void fillRdbmsConnectionForm(MConnection connection) {
- MFormList forms = connection.getConnectorPart();
- forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver());
- forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl());
- forms.getStringInput("connection.username").setValue(provider.getConnectionUsername());
- forms.getStringInput("connection.password").setValue(provider.getConnectionPassword());
+ protected void fillRdbmsLinkForm(MLink link) {
+ MFormList forms = link.getConnectorPart();
+ forms.getStringInput("link.jdbcDriver").setValue(provider.getJdbcDriver());
+ forms.getStringInput("link.connectionString").setValue(provider.getConnectionUrl());
+ forms.getStringInput("link.username").setValue(provider.getConnectionUsername());
+ forms.getStringInput("link.password").setValue(provider.getConnectionPassword());
}
/**
- * Fill output form with specific storage and output type. Mapreduce output directory
- * will be set to default test value.
+ * Fill TO form with specific storage and output type.
*
- * @param job MJOb object to fill
+ * @param job MJob object to fill
* @param output Output type that should be set
*/
- protected void fillOutputForm(MJob job, OutputFormat output) {
- MFormList forms = job.getConnectorPart(Direction.TO);
- forms.getEnumInput("output.outputFormat").setValue(output);
- forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory());
+ protected void fillToJobForm(MJob job, ToFormat output) {
+ MFormList toForms = job.getConnectorPart(Direction.TO);
+ toForms.getEnumInput("toJobConfig.outputFormat").setValue(output);
+ toForms.getStringInput("toJobConfig.outputDirectory").setValue(getMapreduceDirectory());
}
/**
- * Fill input form. Mapreduce input directory will be set to default test value.
+ * Fill FROM form
*
- * @param job MJOb object to fill
+ * @param job MJob object to fill
*/
- protected void fillInputForm(MJob job) {
+ protected void fillFromJobForm(MJob job) {
MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("input.inputDirectory").setValue(getMapreduceDirectory());
+ forms.getStringInput("fromJobConfig.inputDirectory").setValue(getMapreduceDirectory());
}
/**
@@ -204,15 +203,15 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
}
/**
- * Create connection.
+ * Create link.
*
* With asserts to make sure that it was created correctly.
*
- * @param connection
+ * @param link
*/
- protected void createConnection(MConnection connection) {
- assertEquals(Status.FINE, getClient().createConnection(connection));
- assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId());
+ protected void saveLink(MLink link) {
+ assertEquals(Status.FINE, getClient().saveLink(link));
+ assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, link.getPersistenceId());
}
/**
@@ -222,8 +221,8 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
*
* @param job
*/
- protected void createJob(MJob job) {
- assertEquals(Status.FINE, getClient().createJob(job));
+ protected void saveJob(MJob job) {
+ assertEquals(Status.FINE, getClient().saveJob(job));
assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
}
@@ -233,7 +232,7 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
* @param jid Job id
* @throws Exception
*/
- protected void runJob(long jid) throws Exception {
+ protected void executeJob(long jid) throws Exception {
getClient().startSubmission(jid, DEFAULT_SUBMISSION_CALLBACKS, 100);
}
@@ -243,7 +242,7 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
* @param job Job object
* @throws Exception
*/
- protected void runJob(MJob job) throws Exception {
- runJob(job.getPersistenceId());
+ protected void executeJob(MJob job) throws Exception {
+ executeJob(job.getPersistenceId());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
index 7e2558f..63736ab 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
@@ -156,7 +156,7 @@ abstract public class TomcatTestCase {
}
/**
- * Assert that mapreduce has generated following lines.
+ * Assert that execution has generated following lines.
*
* As the lines can be spread between multiple files the ordering do not make
* a difference.
@@ -164,27 +164,29 @@ abstract public class TomcatTestCase {
* @param lines
* @throws IOException
*/
- protected void assertMapreduceOutput(String... lines) throws IOException {
+ protected void assertTo(String... lines) throws IOException {
+ // TODO(VB): fix this to be not directly dependent on mapreduce
HdfsAsserts.assertMapreduceOutput(hdfsClient, getMapreduceDirectory(), lines);
}
/**
- * Verify number of output mapreduce files.
+ * Verify number of TO files.
*
* @param expectedFiles Expected number of files
*/
- protected void assertMapreduceOutputFiles(int expectedFiles) throws IOException {
+ protected void assertToFiles(int expectedFiles) throws IOException {
+ // TODO(VB): fix this to be not directly dependent on mapreduce
HdfsAsserts.assertMapreduceOutputFiles(hdfsClient, getMapreduceDirectory(), expectedFiles);
}
/**
- * Create mapreduce input file with specified content.
+ * Create FROM file with specified content.
*
* @param filename Input file name
* @param lines Individual lines that should be written into the file
* @throws IOException
*/
- protected void createInputMapreduceFile(String filename, String...lines) throws IOException {
+ protected void createFromFile(String filename, String...lines) throws IOException {
HdfsUtils.createFile(hdfsClient, HdfsUtils.joinPathFragments(getMapreduceDirectory(), filename), lines);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
index c01aa80..5e1abc1 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
@@ -17,10 +17,9 @@
*/
package org.apache.sqoop.integration.connector.jdbc.generic;
-import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MFormList;
import org.apache.sqoop.model.MJob;
import org.junit.Test;
@@ -32,38 +31,35 @@ import static org.junit.Assert.assertEquals;
*/
public class FromHDFSToRDBMSTest extends ConnectorTestCase {
- private static final Logger LOG = Logger.getLogger(FromHDFSToRDBMSTest.class);
-
@Test
public void testBasic() throws Exception {
createTableCities();
- createInputMapreduceFile("input-0001",
+ createFromFile("input-0001",
"1,'USA','San Francisco'",
"2,'USA','Sunnyvale'",
"3,'Czech Republic','Brno'",
"4,'USA','Palo Alto'"
);
- // RDBMS connection
- MConnection rdbmsConnection = getClient().newConnection("generic-jdbc-connector");
- fillRdbmsConnectionForm(rdbmsConnection);
- createConnection(rdbmsConnection);
+ // RDBMS link
+ MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
+ fillRdbmsLinkForm(rdbmsLink);
+ saveLink(rdbmsLink);
- // HDFS connection
- MConnection hdfsConnection = getClient().newConnection("hdfs-connector");
- createConnection(hdfsConnection);
+ // HDFS link
+ MLink hdfsLink = getClient().createLink("hdfs-connector");
+ saveLink(hdfsLink);
// Job creation
- MJob job = getClient().newJob(hdfsConnection.getPersistenceId(), rdbmsConnection.getPersistenceId());
+ MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId());
// Connector values
- MFormList fromForms = job.getConnectorPart(Direction.FROM);
MFormList toForms = job.getConnectorPart(Direction.TO);
- toForms.getStringInput("toTable.tableName").setValue(provider.escapeTableName(getTableName()));
- fillInputForm(job);
- createJob(job);
+ toForms.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ fillFromJobForm(job);
+ saveJob(job);
- runJob(job);
+ executeJob(job);
assertEquals(4L, rowCount());
assertRowInCities(1, "USA", "San Francisco");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
index f976f29..2dc0613 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
@@ -17,10 +17,9 @@
*/
package org.apache.sqoop.integration.connector.jdbc.generic;
-import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MFormList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
@@ -34,35 +33,33 @@ import static org.junit.Assert.assertTrue;
*/
public class FromRDBMSToHDFSTest extends ConnectorTestCase {
- private static final Logger LOG = Logger.getLogger(FromRDBMSToHDFSTest.class);
-
@Test
public void testBasic() throws Exception {
createAndLoadTableCities();
- // RDBMS connection
- MConnection rdbmsConnection = getClient().newConnection("generic-jdbc-connector");
- fillRdbmsConnectionForm(rdbmsConnection);
- createConnection(rdbmsConnection);
+ // RDBMS link
+ MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector");
+ fillRdbmsLinkForm(rdbmsConnection);
+ saveLink(rdbmsConnection);
- // HDFS connection
- MConnection hdfsConnection = getClient().newConnection("hdfs-connector");
- createConnection(hdfsConnection);
+ // HDFS link
+ MLink hdfsConnection = getClient().createLink("hdfs-connector");
+ saveLink(hdfsConnection);
// Job creation
- MJob job = getClient().newJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());
+ MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());
// Connector values
MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromTable.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromTable.partitionColumn").setValue(provider.escapeColumnName("id"));
- fillOutputForm(job, OutputFormat.TEXT_FILE);
- createJob(job);
+ forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
+ fillToJobForm(job, ToFormat.TEXT_FILE);
+ saveJob(job);
- runJob(job);
+ executeJob(job);
// Assert correct output
- assertMapreduceOutput(
+ assertTo(
"1,'USA','San Francisco'",
"2,'USA','Sunnyvale'",
"3,'Czech Republic','Brno'",
@@ -77,25 +74,25 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
public void testColumns() throws Exception {
createAndLoadTableCities();
- // RDBMS connection
- MConnection rdbmsConnection = getClient().newConnection("generic-jdbc-connector");
- fillRdbmsConnectionForm(rdbmsConnection);
- createConnection(rdbmsConnection);
+ // RDBMS link
+ MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
+ fillRdbmsLinkForm(rdbmsLink);
+ saveLink(rdbmsLink);
- // HDFS connection
- MConnection hdfsConnection = getClient().newConnection("hdfs-connector");
- createConnection(hdfsConnection);
+ // HDFS link
+ MLink hdfsLink = getClient().createLink("hdfs-connector");
+ saveLink(hdfsLink);
// Job creation
- MJob job = getClient().newJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());
+ MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
// Connector values
MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromTable.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromTable.partitionColumn").setValue(provider.escapeColumnName("id"));
- forms.getStringInput("fromTable.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country"));
- fillOutputForm(job, OutputFormat.TEXT_FILE);
- createJob(job);
+ forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
+ forms.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country"));
+ fillToJobForm(job, ToFormat.TEXT_FILE);
+ saveJob(job);
MSubmission submission = getClient().startSubmission(job.getPersistenceId());
assertTrue(submission.getStatus().isRunning());
@@ -108,7 +105,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
} while(submission.getStatus().isRunning());
// Assert correct output
- assertMapreduceOutput(
+ assertTo(
"1,'USA'",
"2,'USA'",
"3,'Czech Republic'",
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
index fac7e8b..729f95e 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
@@ -17,10 +17,9 @@
*/
package org.apache.sqoop.integration.connector.jdbc.generic;
-import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MFormList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
@@ -35,8 +34,6 @@ import org.junit.Test;
@RunWith(Parameterized.class)
public class PartitionerTest extends ConnectorTestCase {
- private static final Logger LOG = Logger.getLogger(PartitionerTest.class);
-
/**
* Columns that we will use as partition column with maximal number of
* partitions that can be created for such column.
@@ -75,32 +72,32 @@ public class PartitionerTest extends ConnectorTestCase {
public void testSplitter() throws Exception {
createAndLoadTableUbuntuReleases();
- // RDBMS connection
- MConnection rdbmsConnection = getClient().newConnection("generic-jdbc-connector");
- fillRdbmsConnectionForm(rdbmsConnection);
- createConnection(rdbmsConnection);
+ // RDBMS link
+ MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
+ fillRdbmsLinkForm(rdbmsLink);
+ saveLink(rdbmsLink);
- // HDFS connection
- MConnection hdfsConnection = getClient().newConnection("hdfs-connector");
- createConnection(hdfsConnection);
+ // HDFS link
+ MLink hdfsLink = getClient().createLink("hdfs-connector");
+ saveLink(hdfsLink);
// Job creation
- MJob job = getClient().newJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());
+ MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
// Connector values
MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromTable.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromTable.partitionColumn").setValue(provider.escapeColumnName(partitionColumn));
- fillOutputForm(job, OutputFormat.TEXT_FILE);
+ forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn));
+ fillToJobForm(job, ToFormat.TEXT_FILE);
forms = job.getFrameworkPart();
forms.getIntegerInput("throttling.extractors").setValue(extractors);
- createJob(job);
+ saveJob(job);
- runJob(job);
+ executeJob(job);
// Assert correct output
- assertMapreduceOutputFiles((extractors > maxOutputFiles) ? maxOutputFiles : extractors);
- assertMapreduceOutput(
+ assertToFiles((extractors > maxOutputFiles) ? maxOutputFiles : extractors);
+ assertTo(
"1,'Warty Warthog',4.10,2004-10-20,false",
"2,'Hoary Hedgehog',5.04,2005-04-08,false",
"3,'Breezy Badger',5.10,2005-10-13,false",
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
index 1af0cdc..562a6a6 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.integration.connector.jdbc.generic;
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MFormList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.test.data.Cities;
@@ -36,7 +36,7 @@ public class TableStagedRDBMSTest extends ConnectorTestCase {
public void testStagedTransfer() throws Exception {
final String stageTableName = "STAGE_" + getTableName();
createTableCities();
- createInputMapreduceFile("input-0001",
+ createFromFile("input-0001",
"1,'USA','San Francisco'",
"2,'USA','Sunnyvale'",
"3,'Czech Republic','Brno'",
@@ -44,29 +44,29 @@ public class TableStagedRDBMSTest extends ConnectorTestCase {
);
new Cities(provider, stageTableName).createTables();
- // RDBMS connection
- MConnection rdbmsConnection = getClient().newConnection("generic-jdbc-connector");
- fillRdbmsConnectionForm(rdbmsConnection);
- createConnection(rdbmsConnection);
+ // RDBMS link
+ MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
+ fillRdbmsLinkForm(rdbmsLink);
+ saveLink(rdbmsLink);
- // HDFS connection
- MConnection hdfsConnection = getClient().newConnection("hdfs-connector");
- createConnection(hdfsConnection);
+ // HDFS link
+ MLink hdfsLink = getClient().createLink("hdfs-connector");
+ saveLink(hdfsLink);
// Job creation
- MJob job = getClient().newJob(hdfsConnection.getPersistenceId(),
- rdbmsConnection.getPersistenceId());
+ MJob job = getClient().createJob(hdfsLink.getPersistenceId(),
+ rdbmsLink.getPersistenceId());
// Connector values
MFormList forms = job.getConnectorPart(Direction.TO);
- forms.getStringInput("toTable.tableName").setValue(
+ forms.getStringInput("toJobConfig.tableName").setValue(
provider.escapeTableName(getTableName()));
- forms.getStringInput("toTable.stageTableName").setValue(
+ forms.getStringInput("toJobConfig.stageTableName").setValue(
provider.escapeTableName(stageTableName));
- fillInputForm(job);
- createJob(job);
+ fillFromJobForm(job);
+ saveJob(job);
- runJob(job);
+ executeJob(job);
assertEquals(0L, provider.rowCount(stageTableName));
assertEquals(4L, rowCount());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
index 3c01cb0..507ac53 100644
--- a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
@@ -20,9 +20,9 @@ package org.apache.sqoop.integration.server;
import org.apache.sqoop.client.ClientError;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
-import org.apache.sqoop.framework.FrameworkError;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
+import org.apache.sqoop.driver.DriverError;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MFormList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
@@ -39,7 +39,7 @@ import static org.junit.Assert.fail;
/**
* Ensure that server will reject starting job when either job itself
- * or corresponding connection is disabled.
+ * or corresponding link is disabled.
*/
@RunWith(Parameterized.class)
public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase {
@@ -53,11 +53,11 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase {
});
}
- private boolean enabledConnection;
+ private boolean enabledLink;
private boolean enabledJob;
- public SubmissionWithDisabledModelObjectsTest(boolean enabledConnection, boolean enabledJob) {
- this.enabledConnection = enabledConnection;
+ public SubmissionWithDisabledModelObjectsTest(boolean enabledLink, boolean enabledJob) {
+ this.enabledLink = enabledLink;
this.enabledJob = enabledJob;
}
@@ -65,33 +65,33 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase {
public void testWithDisabledObjects() throws Exception {
createAndLoadTableCities();
- // RDBMS connection
- MConnection rdbmsConnection = getClient().newConnection("generic-jdbc-connector");
- fillRdbmsConnectionForm(rdbmsConnection);
- createConnection(rdbmsConnection);
+ // RDBMS link
+ MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
+ fillRdbmsLinkForm(rdbmsLink);
+ saveLink(rdbmsLink);
- // HDFS connection
- MConnection hdfsConnection = getClient().newConnection("hdfs-connector");
- createConnection(hdfsConnection);
+ // HDFS link
+ MLink hdfsLink = getClient().createLink("hdfs-connector");
+ saveLink(hdfsLink);
// Job creation
- MJob job = getClient().newJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());
+ MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
// Connector values
MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromTable.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromTable.partitionColumn").setValue(provider.escapeColumnName("id"));
+ forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
// Framework values
- fillOutputForm(job, OutputFormat.TEXT_FILE);
- createJob(job);
+ fillToJobForm(job, ToFormat.TEXT_FILE);
+ saveJob(job);
// Disable model entities as per parametrized run
- getClient().enableConnection(rdbmsConnection.getPersistenceId(), enabledConnection);
+ getClient().enableLink(rdbmsLink.getPersistenceId(), enabledLink);
getClient().enableJob(job.getPersistenceId(), enabledJob);
- // Try to run the job and verify that the it was not executed
+ // Try to execute the job and verify that the it was not executed
try {
- runJob(job);
+ executeJob(job);
fail("Expected exception as the model classes are disabled.");
} catch(SqoopException ex) {
// Top level exception should be CLIENT_0001
@@ -104,9 +104,9 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase {
assertNotNull(cause);
if(!enabledJob) {
- assertTrue(cause.getMessage().startsWith(FrameworkError.FRAMEWORK_0009.toString()));
- } else if(!enabledConnection) {
- assertTrue(cause.getMessage().startsWith(FrameworkError.FRAMEWORK_0010.toString()));
+ assertTrue(cause.getMessage().startsWith(DriverError.DRIVER_0009.toString()));
+ } else if(!enabledLink) {
+ assertTrue(cause.getMessage().startsWith(DriverError.DRIVER_0010.toString()));
} else {
fail("Unexpected expception retrieved from server " + cause);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
index cea24b9..beed82e 100644
--- a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.integration.server;
-import org.apache.sqoop.client.request.VersionRequest;
+import org.apache.sqoop.client.request.VersionResourceRequest;
import org.apache.sqoop.common.VersionInfo;
import org.apache.sqoop.test.testcases.TomcatTestCase;
import org.apache.sqoop.json.VersionBean;
import org.junit.Test;
-import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertEquals;
/**
* Basic test to check that server is working and returning correct version info.
@@ -32,8 +32,8 @@ public class VersionTest extends TomcatTestCase {
@Test
public void testVersion() {
- VersionRequest versionRequest = new VersionRequest();
- VersionBean versionBean = versionRequest.doGet(getServerUrl());
+ VersionResourceRequest versionRequest = new VersionResourceRequest();
+ VersionBean versionBean = versionRequest.read(getServerUrl());
assertEquals(versionBean.getVersion(), VersionInfo.getVersion());
assertEquals(versionBean.getDate(), VersionInfo.getDate());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/tools/src/main/java/org/apache/sqoop/tools/tool/UpgradeTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/UpgradeTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/UpgradeTool.java
index b8a15cb..f117411 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/UpgradeTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/UpgradeTool.java
@@ -18,16 +18,15 @@
package org.apache.sqoop.tools.tool;
import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.framework.FrameworkManager;
+import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.tools.ConfiguredTool;
-
import org.apache.log4j.Logger;
/**
* Upgrade all versionable components inside Sqoop2. This includes any
- * structural changes inside repository and the Connector and Framework
- * metadata. This tool is idempotent.
+ * structural changes inside repository and the Connector and Driver entity data
+ * This tool is idempotent.
*/
public class UpgradeTool extends ConfiguredTool {
@@ -39,24 +38,24 @@ public class UpgradeTool extends ConfiguredTool {
LOG.info("Initializing the RepositoryManager with immutable option turned off.");
RepositoryManager.getInstance().initialize(false);
- LOG.info("Initializing the FrameworkManager with upgrade option turned on.");
- FrameworkManager.getInstance().initialize(true);
+ LOG.info("Initializing the Driver with upgrade option turned on.");
+ Driver.getInstance().initialize(true);
- LOG.info("Initializing the FrameworkManager with upgrade option turned on.");
+ LOG.info("Initializing the Connection Manager with upgrade option turned on.");
ConnectorManager.getInstance().initialize(true);
LOG.info("Upgrade completed successfully.");
LOG.info("Tearing all managers down.");
ConnectorManager.getInstance().destroy();
- FrameworkManager.getInstance().destroy();
+ Driver.getInstance().destroy();
RepositoryManager.getInstance().destroy();
return true;
} catch (Exception ex) {
- LOG.error("Can't finish upgrading all components:", ex);
+ LOG.error("Can't finish upgrading RepositoryManager, Driver and ConnectionManager:", ex);
System.out.println("Upgrade has failed, please check Server logs for further details.");
return false;
}
}
-}
+}
\ No newline at end of file
[11/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index 68aea9c..5dd7970 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -30,8 +30,6 @@ import java.sql.Timestamp;
import java.sql.Types;
import java.util.*;
-import javax.sql.DataSource;
-
import org.apache.log4j.Logger;
import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.common.Direction;
@@ -40,7 +38,7 @@ import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.ConnectorHandler;
import org.apache.sqoop.connector.ConnectorManagerUtils;
import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MEnumInput;
import org.apache.sqoop.model.MIntegerInput;
@@ -49,7 +47,7 @@ import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MFormType;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MInputType;
import org.apache.sqoop.model.MMapInput;
@@ -57,7 +55,6 @@ import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.repository.JdbcRepositoryContext;
import org.apache.sqoop.repository.JdbcRepositoryHandler;
-import org.apache.sqoop.repository.JdbcRepositoryTransactionFactory;
import org.apache.sqoop.submission.SubmissionStatus;
import org.apache.sqoop.submission.counter.Counter;
import org.apache.sqoop.submission.counter.CounterGroup;
@@ -78,15 +75,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
/**
* Unique name of HDFS Connector.
- * HDFS Connector was originally part of the Sqoop framework, but now is its
+ * HDFS Connector was originally part of the Sqoop driver, but now is its
* own connector. This constant is used to pre-register the HDFS Connector
* so that jobs that are being upgraded can reference the HDFS Connector.
*/
private static final String CONNECTOR_HDFS = "hdfs-connector";
private JdbcRepositoryContext repoContext;
- private DataSource dataSource;
- private JdbcRepositoryTransactionFactory txFactory;
/**
* {@inheritDoc}
@@ -105,10 +100,10 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* Helper method to insert the forms from the into the
* repository. The job and connector forms within <code>mc</code> will get
* updated with the id of the forms when this function returns.
- * @param mf The MFramework instance to use to upgrade.
- * @param conn JDBC connection to use for updating the forms
+ * @param mDriverConfig The MFramework instance to use to upgrade.
+ * @param conn JDBC link to use for updating the forms
*/
- private void insertFormsForFramework(MFramework mf, Connection conn) {
+ private void insertFormsForFramework(MDriverConfig mDriverConfig, Connection conn) {
PreparedStatement baseFormStmt = null;
PreparedStatement baseInputStmt = null;
try{
@@ -119,15 +114,15 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
Statement.RETURN_GENERATED_KEYS);
// Register connector forms
- registerForms(null, null, mf.getConnectionForms().getForms(),
+ registerForms(null, null, mDriverConfig.getConnectionForms().getForms(),
MFormType.CONNECTION.name(), baseFormStmt, baseInputStmt);
// Register job forms
- registerForms(null, null, mf.getJobForms().getForms(),
+ registerForms(null, null, mDriverConfig.getJobForms().getForms(),
MFormType.JOB.name(), baseFormStmt, baseInputStmt);
} catch (SQLException ex) {
- throw new SqoopException(DerbyRepoError.DERBYREPO_0014, mf.toString(), ex);
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0014, mDriverConfig.toString(), ex);
} finally {
closeStatements(baseFormStmt, baseInputStmt);
}
@@ -138,7 +133,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* repository. The job and connector forms within <code>mc</code> will get
* updated with the id of the forms when this function returns.
* @param mc The connector to use for updating forms
- * @param conn JDBC connection to use for updating the forms
+ * @param conn JDBC link to use for updating the forms
*/
private void insertFormsForConnector (MConnector mc, Connection conn) {
long connectorId = mc.getPersistenceId();
@@ -205,8 +200,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
@Override
public synchronized void initialize(JdbcRepositoryContext ctx) {
repoContext = ctx;
- dataSource = repoContext.getDataSource();
- txFactory = repoContext.getTransactionFactory();
+ repoContext.getDataSource();
LOG.info("DerbyRepositoryHandler initialized.");
}
@@ -313,24 +307,24 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
/**
- * Detect version of the framework
+ * Detect version of the driver
*
- * @param conn Connection to metadata repository
- * @return Version of the MFramework
+ * @param conn Connection to the repository
+ * @return Version of the Driver
*/
- private String detectFrameworkVersion (Connection conn) {
+ private String detectDriverVersion (Connection conn) {
ResultSet rs = null;
PreparedStatement stmt = null;
try {
stmt = conn.prepareStatement(DerbySchemaQuery.STMT_SELECT_SYSTEM);
- stmt.setString(1, DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION);
+ stmt.setString(1, DerbyRepoConstants.SYSKEY_DRIVER_VERSION);
rs = stmt.executeQuery();
if(!rs.next()) {
return null;
}
return rs.getString(1);
} catch (SQLException e) {
- LOG.info("Can't fetch framework version.", e);
+ LOG.info("Can't fetch driver version.", e);
return null;
} finally {
closeResultSets(rs);
@@ -339,23 +333,22 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
/**
- * Create or update framework version
- * @param conn Connection to the metadata repository
- * @param mFramework
+ * Create or update driver version
+ * @param conn Connection to the the repository
+ * @param mDriverConfig
*/
- private void createOrUpdateFrameworkVersion(Connection conn,
- MFramework mFramework) {
+ private void createOrUpdateDriverVersion(Connection conn, MDriverConfig mDriverConfig) {
ResultSet rs = null;
PreparedStatement stmt = null;
try {
stmt = conn.prepareStatement(STMT_DELETE_SYSTEM);
- stmt.setString(1, DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION);
+ stmt.setString(1, DerbyRepoConstants.SYSKEY_DRIVER_VERSION);
stmt.executeUpdate();
closeStatements(stmt);
stmt = conn.prepareStatement(STMT_INSERT_SYSTEM);
- stmt.setString(1, DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION);
- stmt.setString(2, mFramework.getVersion());
+ stmt.setString(1, DerbyRepoConstants.SYSKEY_DRIVER_VERSION);
+ stmt.setString(2, mDriverConfig.getVersion());
stmt.executeUpdate();
} catch (SQLException e) {
logException(e);
@@ -446,7 +439,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
/**
* Upgrade job data from IMPORT/EXPORT to FROM/TO.
* Since the framework is no longer responsible for HDFS,
- * the HDFS connector/connection must be added.
+ * the HDFS connector/link must be added.
* Also, the framework forms are moved around such that
* they belong to the added HDFS connector. Any extra forms
* are removed.
@@ -471,11 +464,11 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* its dependencies point to the IMPORT throttling form.
* Then make sure the throttling form does not have a direction.
* Framework forms should not have a direction.
- * 9. Create an HDFS connection to reference and update
- * jobs to reference that connection. IMPORT jobs
+ * 9. Create an HDFS link to reference and update
+ * jobs to reference that link. IMPORT jobs
* should have TO HDFS connector, EXPORT jobs should have
* FROM HDFS connector.
- * 10. Update 'table' form names to 'fromTable' and 'toTable'.
+ * 10. Update 'table' form names to 'fromJobConfig' and 'toTable'.
* Also update the relevant inputs as well.
* @param conn
*/
@@ -510,7 +503,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_FRAMEWORK_INDEX, conn,
new Long(0), "throttling");
- MConnection hdfsConnection = createHdfsConnection(conn);
+ MLink hdfsConnection = createHdfsConnection(conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION_COPY_SQB_FROM_CONNECTION, conn,
"EXPORT");
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_FROM_CONNECTION, conn,
@@ -519,13 +512,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
new Long(hdfsConnection.getPersistenceId()), "IMPORT");
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_NAME, conn,
- "fromTable", "table", Direction.FROM.toString());
+ "fromJobConfig", "table", Direction.FROM.toString());
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES, conn,
- Direction.FROM.toString().toLowerCase(), "fromTable", Direction.FROM.toString());
+ Direction.FROM.toString().toLowerCase(), "fromJobConfig", Direction.FROM.toString());
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_NAME, conn,
- "toTable", "table", Direction.TO.toString());
+ "toJobConfig", "table", Direction.TO.toString());
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES, conn,
- Direction.TO.toString().toLowerCase(), "toTable", Direction.TO.toString());
+ Direction.TO.toString().toLowerCase(), "toJobConfig", Direction.TO.toString());
if (LOG.isTraceEnabled()) {
LOG.trace("Updated existing data for generic connectors.");
@@ -583,27 +576,27 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
/**
- * Create an HDFS connection.
- * Intended to be used when moving HDFS connector out of framework
+ * Create an HDFS link.
+ * Intended to be used when moving HDFS connector out of driverConfig
* to its own connector.
*
* NOTE: Upgrade path only!
*/
- private MConnection createHdfsConnection(Connection conn) {
+ private MLink createHdfsConnection(Connection conn) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Creating HDFS connection.");
+ LOG.trace("Creating HDFS link.");
}
MConnector hdfsConnector = this.findConnector(CONNECTOR_HDFS, conn);
- MFramework framework = findFramework(conn);
- MConnection hdfsConnection = new MConnection(
+ MDriverConfig driverConfig = findDriverConfig(conn);
+ MLink hdfsConnection = new MLink(
hdfsConnector.getPersistenceId(),
hdfsConnector.getConnectionForms(),
- framework.getConnectionForms());
- this.createConnection(hdfsConnection, conn);
+ driverConfig.getConnectionForms());
+ this.createLink(hdfsConnection, conn);
if (LOG.isTraceEnabled()) {
- LOG.trace("Created HDFS connection.");
+ LOG.trace("Created HDFS link.");
}
return hdfsConnection;
@@ -682,8 +675,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public void registerFramework(MFramework mf, Connection conn) {
- if (mf.hasPersistenceId()) {
+ public void registerDriverConfig(MDriverConfig mDriverConfig, Connection conn) {
+ if (mDriverConfig.hasPersistenceId()) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0011,
"Framework metadata");
}
@@ -697,32 +690,32 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
Statement.RETURN_GENERATED_KEYS);
// Register connector forms
- registerForms(null, null, mf.getConnectionForms().getForms(),
+ registerForms(null, null, mDriverConfig.getConnectionForms().getForms(),
MFormType.CONNECTION.name(), baseFormStmt, baseInputStmt);
// Register all jobs
- registerForms(null, null, mf.getJobForms().getForms(),
+ registerForms(null, null, mDriverConfig.getJobForms().getForms(),
MFormType.JOB.name(), baseFormStmt, baseInputStmt);
// We're using hardcoded value for framework metadata as they are
// represented as NULL in the database.
- mf.setPersistenceId(1);
+ mDriverConfig.setPersistenceId(1);
} catch (SQLException ex) {
- logException(ex, mf);
+ logException(ex, mDriverConfig);
throw new SqoopException(DerbyRepoError.DERBYREPO_0014, ex);
} finally {
closeStatements(baseFormStmt, baseInputStmt);
}
- createOrUpdateFrameworkVersion(conn, mf);
+ createOrUpdateDriverVersion(conn, mDriverConfig);
}
/**
* {@inheritDoc}
*/
@Override
- public MFramework findFramework(Connection conn) {
- LOG.debug("Looking up framework metadata");
- MFramework mf = null;
+ public MDriverConfig findDriverConfig(Connection conn) {
+ LOG.debug("Looking up driver config");
+ MDriverConfig mDriverConfig = null;
PreparedStatement formFetchStmt = null;
PreparedStatement inputFetchStmt = null;
try {
@@ -739,16 +732,16 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
return null;
}
- mf = new MFramework(new MConnectionForms(connectionForms),
- new MJobForms(jobForms), detectFrameworkVersion(conn));
+ mDriverConfig = new MDriverConfig(new MConnectionForms(connectionForms),
+ new MJobForms(jobForms), detectDriverVersion(conn));
- // We're using hardcoded value for framework metadata as they are
+ // We're using hardcoded value for driver config as they are
// represented as NULL in the database.
- mf.setPersistenceId(1);
+ mDriverConfig.setPersistenceId(1);
} catch (SQLException ex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0004,
- "Framework metadata", ex);
+ "Driver config", ex);
} finally {
if (formFetchStmt != null) {
try {
@@ -766,10 +759,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
}
- LOG.debug("Looking up framework metadta found: " + mf);
-
- // Returned loaded framework metadata
- return mf;
+ LOG.debug("Looking up driver config found:" + mDriverConfig);
+ return mDriverConfig;
}
/**
@@ -784,19 +775,19 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public void createConnection(MConnection connection, Connection conn) {
+ public void createLink(MLink link, Connection conn) {
PreparedStatement stmt = null;
int result;
try {
stmt = conn.prepareStatement(STMT_INSERT_CONNECTION,
Statement.RETURN_GENERATED_KEYS);
- stmt.setString(1, connection.getName());
- stmt.setLong(2, connection.getConnectorId());
- stmt.setBoolean(3, connection.getEnabled());
- stmt.setString(4, connection.getCreationUser());
- stmt.setTimestamp(5, new Timestamp(connection.getCreationDate().getTime()));
- stmt.setString(6, connection.getLastUpdateUser());
- stmt.setTimestamp(7, new Timestamp(connection.getLastUpdateDate().getTime()));
+ stmt.setString(1, link.getName());
+ stmt.setLong(2, link.getConnectorId());
+ stmt.setBoolean(3, link.getEnabled());
+ stmt.setString(4, link.getCreationUser());
+ stmt.setTimestamp(5, new Timestamp(link.getCreationDate().getTime()));
+ stmt.setString(6, link.getLastUpdateUser());
+ stmt.setTimestamp(7, new Timestamp(link.getLastUpdateDate().getTime()));
result = stmt.executeUpdate();
if (result != 1) {
@@ -814,17 +805,17 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
createInputValues(STMT_INSERT_CONNECTION_INPUT,
connectionId,
- connection.getConnectorPart().getForms(),
+ link.getConnectorPart().getForms(),
conn);
createInputValues(STMT_INSERT_CONNECTION_INPUT,
connectionId,
- connection.getFrameworkPart().getForms(),
+ link.getFrameworkPart().getForms(),
conn);
- connection.setPersistenceId(connectionId);
+ link.setPersistenceId(connectionId);
} catch (SQLException ex) {
- logException(ex, connection);
+ logException(ex, link);
throw new SqoopException(DerbyRepoError.DERBYREPO_0019, ex);
} finally {
closeStatements(stmt);
@@ -835,36 +826,36 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public void updateConnection(MConnection connection, Connection conn) {
+ public void updateLink(MLink link, Connection conn) {
PreparedStatement deleteStmt = null;
PreparedStatement updateStmt = null;
try {
// Firstly remove old values
deleteStmt = conn.prepareStatement(STMT_DELETE_CONNECTION_INPUT);
- deleteStmt.setLong(1, connection.getPersistenceId());
+ deleteStmt.setLong(1, link.getPersistenceId());
deleteStmt.executeUpdate();
// Update CONNECTION table
updateStmt = conn.prepareStatement(STMT_UPDATE_CONNECTION);
- updateStmt.setString(1, connection.getName());
- updateStmt.setString(2, connection.getLastUpdateUser());
+ updateStmt.setString(1, link.getName());
+ updateStmt.setString(2, link.getLastUpdateUser());
updateStmt.setTimestamp(3, new Timestamp(new Date().getTime()));
- updateStmt.setLong(4, connection.getPersistenceId());
+ updateStmt.setLong(4, link.getPersistenceId());
updateStmt.executeUpdate();
// And reinsert new values
createInputValues(STMT_INSERT_CONNECTION_INPUT,
- connection.getPersistenceId(),
- connection.getConnectorPart().getForms(),
+ link.getPersistenceId(),
+ link.getConnectorPart().getForms(),
conn);
createInputValues(STMT_INSERT_CONNECTION_INPUT,
- connection.getPersistenceId(),
- connection.getFrameworkPart().getForms(),
+ link.getPersistenceId(),
+ link.getFrameworkPart().getForms(),
conn);
} catch (SQLException ex) {
- logException(ex, connection);
+ logException(ex, link);
throw new SqoopException(DerbyRepoError.DERBYREPO_0021, ex);
} finally {
closeStatements(deleteStmt, updateStmt);
@@ -875,7 +866,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public boolean existsConnection(long id, Connection conn) {
+ public boolean existsLink(long id, Connection conn) {
PreparedStatement stmt = null;
ResultSet rs = null;
try {
@@ -897,7 +888,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
@Override
- public boolean inUseConnection(long connectionId, Connection conn) {
+ public boolean inUseLink(long connectionId, Connection conn) {
PreparedStatement stmt = null;
ResultSet rs = null;
@@ -921,7 +912,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
@Override
- public void enableConnection(long connectionId, boolean enabled, Connection conn) {
+ public void enableLink(long connectionId, boolean enabled, Connection conn) {
PreparedStatement enableConn = null;
try {
@@ -941,11 +932,11 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public void deleteConnection(long id, Connection conn) {
+ public void deleteLink(long id, Connection conn) {
PreparedStatement dltConn = null;
try {
- deleteConnectionInputs(id, conn);
+ deleteLinkInputs(id, conn);
dltConn = conn.prepareStatement(STMT_DELETE_CONNECTION);
dltConn.setLong(1, id);
dltConn.executeUpdate();
@@ -961,7 +952,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public void deleteConnectionInputs(long id, Connection conn) {
+ public void deleteLinkInputs(long id, Connection conn) {
PreparedStatement dltConnInput = null;
try {
dltConnInput = conn.prepareStatement(STMT_DELETE_CONNECTION_INPUT);
@@ -979,20 +970,20 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public MConnection findConnection(long id, Connection conn) {
+ public MLink findLink(long id, Connection conn) {
PreparedStatement stmt = null;
try {
stmt = conn.prepareStatement(STMT_SELECT_CONNECTION_SINGLE);
stmt.setLong(1, id);
- List<MConnection> connections = loadConnections(stmt, conn);
+ List<MLink> connections = loadLinks(stmt, conn);
if(connections.size() != 1) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0024, "Couldn't find"
- + " connection with id " + id);
+ + " link with id " + id);
}
- // Return the first and only one connection object
+ // Return the first and only one link object
return connections.get(0);
} catch (SQLException ex) {
@@ -1007,12 +998,12 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public List<MConnection> findConnections(Connection conn) {
+ public List<MLink> findLinks(Connection conn) {
PreparedStatement stmt = null;
try {
stmt = conn.prepareStatement(STMT_SELECT_CONNECTION_ALL);
- return loadConnections(stmt, conn);
+ return loadLinks(stmt, conn);
} catch (SQLException ex) {
logException(ex);
@@ -1029,13 +1020,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
*
*/
@Override
- public List<MConnection> findConnectionsForConnector(long connectorID, Connection conn) {
+ public List<MLink> findLinksForConnector(long connectorID, Connection conn) {
PreparedStatement stmt = null;
try {
stmt = conn.prepareStatement(STMT_SELECT_CONNECTION_FOR_CONNECTOR);
stmt.setLong(1, connectorID);
- return loadConnections(stmt, conn);
+ return loadLinks(stmt, conn);
} catch (SQLException ex) {
logException(ex, connectorID);
@@ -1084,7 +1075,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public void updateFramework(MFramework mFramework, Connection conn) {
+ public void updateDriverConfig(MDriverConfig mDriverConfig, Connection conn) {
PreparedStatement deleteForm = null;
PreparedStatement deleteInput = null;
try {
@@ -1095,13 +1086,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
deleteForm.executeUpdate();
} catch (SQLException e) {
- logException(e, mFramework);
+ logException(e, mDriverConfig);
throw new SqoopException(DerbyRepoError.DERBYREPO_0044, e);
} finally {
closeStatements(deleteForm, deleteInput);
}
- createOrUpdateFrameworkVersion(conn, mFramework);
- insertFormsForFramework(mFramework, conn);
+ createOrUpdateDriverVersion(conn, mDriverConfig);
+ insertFormsForFramework(mDriverConfig, conn);
}
@@ -1116,8 +1107,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
stmt = conn.prepareStatement(STMT_INSERT_JOB,
Statement.RETURN_GENERATED_KEYS);
stmt.setString(1, job.getName());
- stmt.setLong(2, job.getConnectionId(Direction.FROM));
- stmt.setLong(3, job.getConnectionId(Direction.TO));
+ stmt.setLong(2, job.getLinkId(Direction.FROM));
+ stmt.setLong(3, job.getLinkId(Direction.TO));
stmt.setBoolean(4, job.getEnabled());
stmt.setString(5, job.getCreationUser());
stmt.setTimestamp(6, new Timestamp(job.getCreationDate().getTime()));
@@ -1318,7 +1309,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
+ " job with id " + id);
}
- // Return the first and only one connection object
+ // Return the first and only one link object
return jobs.get(0);
} catch (SQLException ex) {
@@ -1652,7 +1643,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* Resolves counter group database id.
*
* @param group Given group
- * @param conn Connection to metastore
+ * @param conn Connection to database
* @return Id
* @throws SQLException
*/
@@ -1693,7 +1684,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* Resolves counter id.
*
* @param counter Given counter
- * @param conn connection to metastore
+ * @param conn Connection to database
* @return Id
* @throws SQLException
*/
@@ -1734,7 +1725,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* Create MSubmission structure from result set.
*
* @param rs Result set, only active row will be fetched
- * @param conn Connection to metastore
+ * @param conn Connection to database
* @return Created MSubmission structure
* @throws SQLException
*/
@@ -1836,10 +1827,10 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
return connectors;
}
- private List<MConnection> loadConnections(PreparedStatement stmt,
+ private List<MLink> loadLinks(PreparedStatement stmt,
Connection conn)
throws SQLException {
- List<MConnection> connections = new ArrayList<MConnection>();
+ List<MLink> links = new ArrayList<MLink>();
ResultSet rsConnection = null;
PreparedStatement formConnectorFetchStmt = null;
PreparedStatement formFrameworkFetchStmt = null;
@@ -1879,19 +1870,19 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
loadFrameworkForms(frameworkConnForms, frameworkJobForms,
formFrameworkFetchStmt, inputFetchStmt, 2);
- MConnection connection = new MConnection(connectorId,
+ MLink link = new MLink(connectorId,
new MConnectionForms(connectorConnForms),
new MConnectionForms(frameworkConnForms));
- connection.setPersistenceId(id);
- connection.setName(name);
- connection.setCreationUser(creationUser);
- connection.setCreationDate(creationDate);
- connection.setLastUpdateUser(updateUser);
- connection.setLastUpdateDate(lastUpdateDate);
- connection.setEnabled(enabled);
+ link.setPersistenceId(id);
+ link.setName(name);
+ link.setCreationUser(creationUser);
+ link.setCreationDate(creationDate);
+ link.setLastUpdateUser(updateUser);
+ link.setLastUpdateDate(lastUpdateDate);
+ link.setEnabled(enabled);
- connections.add(connection);
+ links.add(link);
}
} finally {
closeResultSets(rsConnection);
@@ -1899,7 +1890,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
formFrameworkFetchStmt, inputFetchStmt);
}
- return connections;
+ return links;
}
private List<MJob> loadJobs(PreparedStatement stmt,
@@ -1925,8 +1916,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
long toConnectorId = rsJob.getLong(2);
long id = rsJob.getLong(3);
String name = rsJob.getString(4);
- long fromConnectionId = rsJob.getLong(5);
- long toConnectionId = rsJob.getLong(6);
+ long fromLinkId = rsJob.getLong(5);
+ long toLinkId = rsJob.getLong(6);
boolean enabled = rsJob.getBoolean(7);
String createBy = rsJob.getString(8);
Date creationDate = rsJob.getTimestamp(9);
@@ -1971,7 +1962,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
MJob job = new MJob(
fromConnectorId, toConnectorId,
- fromConnectionId, toConnectionId,
+ fromLinkId, toLinkId,
new MJobForms(fromConnectorFromJobForms),
new MJobForms(toConnectorToJobForms),
new MJobForms(frameworkJobForms));
@@ -2142,7 +2133,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* Use given prepared statements to load all forms and corresponding inputs
* from Derby.
*
- * @param connectionForms List of connection forms that will be filled up
+ * @param connectionForms List of link forms that will be filled up
* @param jobForms Map with job forms that will be filled up
* @param formFetchStmt Prepared statement for fetching forms
* @param inputFetchStmt Prepare statement for fetching inputs
@@ -2159,14 +2150,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
while (rsetForm.next()) {
long formId = rsetForm.getLong(1);
Long formConnectorId = rsetForm.getLong(2);
- String direction = rsetForm.getString(3);
String formName = rsetForm.getString(4);
String formType = rsetForm.getString(5);
int formIndex = rsetForm.getInt(6);
List<MInput<?>> formInputs = new ArrayList<MInput<?>>();
- MForm mf = new MForm(formName, formInputs);
- mf.setPersistenceId(formId);
+ MForm mDriverConfig = new MForm(formName, formInputs);
+ mDriverConfig.setPersistenceId(formId);
inputFetchStmt.setLong(formPosition, formId);
@@ -2217,52 +2207,52 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
input.restoreFromUrlSafeValueString(value);
}
- if (mf.getInputs().size() != inputIndex) {
+ if (mDriverConfig.getInputs().size() != inputIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0009,
- "form: " + mf
+ "form: " + mDriverConfig
+ "; input: " + input
+ "; index: " + inputIndex
- + "; expected: " + mf.getInputs().size()
+ + "; expected: " + mDriverConfig.getInputs().size()
);
}
- mf.getInputs().add(input);
+ mDriverConfig.getInputs().add(input);
}
- if (mf.getInputs().size() == 0) {
+ if (mDriverConfig.getInputs().size() == 0) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0008,
"connector-" + formConnectorId
- + "; form: " + mf
+ + "; form: " + mDriverConfig
);
}
- MFormType mft = MFormType.valueOf(formType);
- switch (mft) {
+ MFormType mDriverConfigt = MFormType.valueOf(formType);
+ switch (mDriverConfigt) {
case CONNECTION:
if (connectionForms.size() != formIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0010,
"connector-" + formConnectorId
- + "; form: " + mf
+ + "; form: " + mDriverConfig
+ "; index: " + formIndex
+ "; expected: " + connectionForms.size()
);
}
- connectionForms.add(mf);
+ connectionForms.add(mDriverConfig);
break;
case JOB:
if (jobForms.size() != formIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0010,
"connector-" + formConnectorId
- + "; form: " + mf
+ + "; form: " + mDriverConfig
+ "; index: " + formIndex
+ "; expected: " + jobForms.size()
);
}
- jobForms.add(mf);
+ jobForms.add(mDriverConfig);
break;
default:
throw new SqoopException(DerbyRepoError.DERBYREPO_0007,
- "connector-" + formConnectorId + ":" + mf);
+ "connector-" + formConnectorId + ":" + mDriverConfig);
}
}
}
@@ -2273,7 +2263,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* Use given prepared statements to load all forms and corresponding inputs
* from Derby.
*
- * @param connectionForms List of connection forms that will be filled up
+ * @param connectionForms List of link forms that will be filled up
* @param fromJobForms FROM job forms that will be filled up
* @param toJobForms TO job forms that will be filled up
* @param formFetchStmt Prepared statement for fetching forms
@@ -2298,8 +2288,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
int formIndex = rsetForm.getInt(6);
List<MInput<?>> formInputs = new ArrayList<MInput<?>>();
- MForm mf = new MForm(formName, formInputs);
- mf.setPersistenceId(formId);
+ MForm mDriverConfig = new MForm(formName, formInputs);
+ mDriverConfig.setPersistenceId(formId);
inputFetchStmt.setLong(formPosition, formId);
@@ -2317,7 +2307,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
MInputType mit = MInputType.valueOf(inputType);
- MInput input = null;
+ MInput<?> input = null;
switch (mit) {
case STRING:
input = new MStringInput(inputName, inputSensitivity, inputStrLength);
@@ -2350,37 +2340,37 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
input.restoreFromUrlSafeValueString(value);
}
- if (mf.getInputs().size() != inputIndex) {
+ if (mDriverConfig.getInputs().size() != inputIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0009,
- "form: " + mf
+ "form: " + mDriverConfig
+ "; input: " + input
+ "; index: " + inputIndex
- + "; expected: " + mf.getInputs().size()
+ + "; expected: " + mDriverConfig.getInputs().size()
);
}
- mf.getInputs().add(input);
+ mDriverConfig.getInputs().add(input);
}
- if (mf.getInputs().size() == 0) {
+ if (mDriverConfig.getInputs().size() == 0) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0008,
"connector-" + formConnectorId
- + "; form: " + mf
+ + "; form: " + mDriverConfig
);
}
- MFormType mft = MFormType.valueOf(formType);
- switch (mft) {
+ MFormType mDriverConfigt = MFormType.valueOf(formType);
+ switch (mDriverConfigt) {
case CONNECTION:
if (connectionForms.size() != formIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0010,
"connector-" + formConnectorId
- + "; form: " + mf
+ + "; form: " + mDriverConfig
+ "; index: " + formIndex
+ "; expected: " + connectionForms.size()
);
}
- connectionForms.add(mf);
+ connectionForms.add(mDriverConfig);
break;
case JOB:
Direction type = Direction.valueOf(operation);
@@ -2401,17 +2391,17 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
if (jobForms.size() != formIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0010,
"connector-" + formConnectorId
- + "; form: " + mf
+ + "; form: " + mDriverConfig
+ "; index: " + formIndex
+ "; expected: " + jobForms.size()
);
}
- jobForms.add(mf);
+ jobForms.add(mDriverConfig);
break;
default:
throw new SqoopException(DerbyRepoError.DERBYREPO_0007,
- "connector-" + formConnectorId + ":" + mf);
+ "connector-" + formConnectorId + ":" + mDriverConfig);
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
index 061551e..ad42901 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
@@ -1012,7 +1012,7 @@ public final class DerbySchemaQuery {
/**
* Intended to rename forms based on direction.
* e.g. If SQ_FORM.SQF_NAME = 'table' and parameter 1 = 'from'
- * then SQ_FORM.SQF_NAME = 'fromTable'.
+ * then SQ_FORM.SQF_NAME = 'fromJobConfig'.
*/
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES =
"UPDATE " + TABLE_SQ_INPUT + " SET "
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index 29da340..998f5b7 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -18,13 +18,14 @@
package org.apache.sqoop.repository.derby;
import junit.framework.TestCase;
+
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.framework.FrameworkManager;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.driver.Driver;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MJobForms;
@@ -48,7 +49,7 @@ import static org.apache.sqoop.repository.derby.DerbySchemaQuery.*;
*/
abstract public class DerbyTestCase extends TestCase {
- private static int SYSTEM_VERSION = 4;
+ private static int LATEST_SYSTEM_VERSION = 4;
public static final String DERBY_DRIVER =
"org.apache.derby.jdbc.EmbeddedDriver";
@@ -62,14 +63,14 @@ abstract public class DerbyTestCase extends TestCase {
public void setUp() throws Exception {
super.setUp();
- // Create connection to the database
+ // Create link to the database
Class.forName(DERBY_DRIVER).newInstance();
connection = DriverManager.getConnection(getStartJdbcUrl());
}
@Override
public void tearDown() throws Exception {
- // Close active connection
+ // Close active link
if(connection != null) {
connection.close();
}
@@ -130,11 +131,11 @@ abstract public class DerbyTestCase extends TestCase {
runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) VALUES('version', '" + version + "')");
runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) " +
- "VALUES('framework.version', '1')");
+ "VALUES('" + DerbyRepoConstants.SYSKEY_DRIVER_VERSION + "', '1')");
}
protected void createSchema() throws Exception {
- createSchema(SYSTEM_VERSION);
+ createSchema(LATEST_SYSTEM_VERSION);
}
/**
@@ -146,7 +147,7 @@ abstract public class DerbyTestCase extends TestCase {
protected void runQuery(String query, String... args) throws Exception {
PreparedStatement stmt = null;
try {
- stmt = getDerbyConnection().prepareStatement(query);
+ stmt = getDerbyDatabaseConnection().prepareStatement(query);
for (int i = 0; i < args.length; ++i) {
stmt.setString(i + 1, args[i]);
@@ -160,7 +161,7 @@ abstract public class DerbyTestCase extends TestCase {
}
}
- protected Connection getDerbyConnection() {
+ protected Connection getDerbyDatabaseConnection() {
return connection;
}
@@ -176,7 +177,7 @@ abstract public class DerbyTestCase extends TestCase {
return JDBC_URL + ";drop=true";
}
- protected void loadConnectorAndFrameworkVersion2() throws Exception {
+ protected void loadConnectorAndDriverConfigVersion2() throws Exception {
// Connector entry
runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ "VALUES('A', 'org.apache.sqoop.test.A', '1.0-test')");
@@ -255,7 +256,7 @@ abstract public class DerbyTestCase extends TestCase {
+ "('throttling.loaders',10,1,'INTEGER','false',NULL,NULL)");
}
- protected void loadConnectorAndFrameworkVersion4() throws Exception {
+ protected void loadConnectorAndDriverConfigVersion4() throws Exception {
// Connector entry
runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ "VALUES('A', 'org.apache.sqoop.test.A', '1.0-test')");
@@ -308,10 +309,10 @@ abstract public class DerbyTestCase extends TestCase {
}
// Input entries
- // Connector connection parts: 0-3
+ // Connector link parts: 0-3
// Connector job (FROM) parts: 4-7
// Connector job (TO) parts: 8-11
- // Framework connection parts: 12-15
+ // Framework link parts: 12-15
// Framework job parts: 16-19
for (int i = 0; i < 5; i++) {
// First form
@@ -333,19 +334,19 @@ abstract public class DerbyTestCase extends TestCase {
}
/**
- * Load testing connector and framework metadata into repository.
+ * Load testing connector and driver config into repository.
*
* @param version system version (2 or 4)
* @throws Exception
*/
- protected void loadConnectorAndFramework(int version) throws Exception {
+ protected void loadConnectorAndDriverConfig(int version) throws Exception {
switch(version) {
case 2:
- loadConnectorAndFrameworkVersion2();
+ loadConnectorAndDriverConfigVersion2();
break;
case 4:
- loadConnectorAndFrameworkVersion4();
+ loadConnectorAndDriverConfigVersion4();
break;
default:
@@ -353,20 +354,20 @@ abstract public class DerbyTestCase extends TestCase {
}
}
- protected void loadConnectorAndFramework() throws Exception {
- loadConnectorAndFramework(SYSTEM_VERSION);
+ protected void loadConnectorAndDriverConfig() throws Exception {
+ loadConnectorAndDriverConfig(LATEST_SYSTEM_VERSION);
}
/**
- * Load testing connection objects into metadata repository.
+ * Load testing link objects into repository.
*
* @param version system version (2 or 4)
* @throws Exception
*/
- public void loadConnections(int version) throws Exception {
+ public void loadLinks(int version) throws Exception {
switch (version) {
case 2:
- // Insert two connections - CA and CB
+ // Insert two links - CA and CB
runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ "VALUES('CA', 1)");
runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
@@ -382,7 +383,7 @@ abstract public class DerbyTestCase extends TestCase {
break;
case 4:
- // Insert two connections - CA and CB
+ // Insert two links - CA and CB
runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ "VALUES('CA', 1)");
runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
@@ -402,12 +403,12 @@ abstract public class DerbyTestCase extends TestCase {
}
}
- public void loadConnections() throws Exception {
- loadConnections(SYSTEM_VERSION);
+ public void loadLinks() throws Exception {
+ loadLinks(LATEST_SYSTEM_VERSION);
}
/**
- * Load testing job objects into metadata repository.
+ * Load testing job objects into repository.
*
* @param version system version (2 or 4)
* @throws Exception
@@ -470,7 +471,7 @@ abstract public class DerbyTestCase extends TestCase {
}
public void loadJobs() throws Exception {
- loadJobs(SYSTEM_VERSION);
+ loadJobs(LATEST_SYSTEM_VERSION);
}
/**
@@ -483,7 +484,7 @@ abstract public class DerbyTestCase extends TestCase {
}
/**
- * Load testing submissions into the metadata repository.
+ * Load testing submissions into the repository.
*
* @throws Exception
*/
@@ -527,22 +528,22 @@ abstract public class DerbyTestCase extends TestCase {
protected MConnector getConnector() {
return new MConnector("A", "org.apache.sqoop.test.A", "1.0-test",
- getConnectionForms(), new MJobForms(getForms()), new MJobForms(getForms()));
+ getConnectionForms(), getJobForms(), getJobForms());
}
- protected MFramework getFramework() {
- return new MFramework(getConnectionForms(), new MJobForms(getForms()),
- FrameworkManager.CURRENT_FRAMEWORK_VERSION);
+ protected MDriverConfig getDriverConfig() {
+ return new MDriverConfig(getConnectionForms(), getJobForms(),
+ Driver.CURRENT_DRIVER_VERSION);
}
- protected void fillConnection(MConnection connection) {
+ protected void fillLink(MLink link) {
List<MForm> forms;
- forms = connection.getConnectorPart().getForms();
+ forms = link.getConnectorPart().getForms();
((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
- forms = connection.getFrameworkPart().getForms();
+ forms = link.getFrameworkPart().getForms();
((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value13");
((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value15");
}
@@ -566,6 +567,10 @@ abstract public class DerbyTestCase extends TestCase {
protected MConnectionForms getConnectionForms() {
return new MConnectionForms(getForms());
}
+
+ protected MJobForms getJobForms() {
+ return new MJobForms(getForms());
+ }
protected List<MForm> getForms() {
List<MForm> forms = new LinkedList<MForm>();
@@ -602,7 +607,7 @@ abstract public class DerbyTestCase extends TestCase {
ResultSet rs = null;
try {
- stmt = getDerbyConnection().createStatement();
+ stmt = getDerbyDatabaseConnection().createStatement();
rs = stmt.executeQuery("SELECT COUNT(*) FROM "+ table);
rs.next();
@@ -658,7 +663,7 @@ abstract public class DerbyTestCase extends TestCase {
ResultSetMetaData rsmt = null;
try {
- ps = getDerbyConnection().prepareStatement("SELECT * FROM " + table);
+ ps = getDerbyDatabaseConnection().prepareStatement("SELECT * FROM " + table);
rs = ps.executeQuery();
rsmt = rs.getMetaData();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java
deleted file mode 100644
index f9e9217..0000000
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.repository.derby;
-
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MMapInput;
-import org.apache.sqoop.model.MStringInput;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Test connection methods on Derby repository.
- */
-public class TestConnectionHandling extends DerbyTestCase {
-
- DerbyRepositoryHandler handler;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
-
- handler = new DerbyRepositoryHandler();
-
- // We always needs schema for this test case
- createSchema();
-
- // We always needs connector and framework structures in place
- loadConnectorAndFramework();
- }
-
- public void testFindConnection() throws Exception {
- // Let's try to find non existing connection
- try {
- handler.findConnection(1, getDerbyConnection());
- fail();
- } catch(SqoopException ex) {
- assertEquals(DerbyRepoError.DERBYREPO_0024, ex.getErrorCode());
- }
-
- // Load prepared connections into database
- loadConnections();
-
- MConnection connA = handler.findConnection(1, getDerbyConnection());
- assertNotNull(connA);
- assertEquals(1, connA.getPersistenceId());
- assertEquals("CA", connA.getName());
-
- List<MForm> forms;
-
- // Check connector part
- forms = connA.getConnectorPart().getForms();
- assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value3", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- // Check framework part
- forms = connA.getFrameworkPart().getForms();
- assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
- }
-
- public void testFindConnections() throws Exception {
- List<MConnection> list;
-
- // Load empty list on empty repository
- list = handler.findConnections(getDerbyConnection());
- assertEquals(0, list.size());
-
- loadConnections();
-
- // Load all two connections on loaded repository
- list = handler.findConnections(getDerbyConnection());
- assertEquals(2, list.size());
-
- assertEquals("CA", list.get(0).getName());
- assertEquals("CB", list.get(1).getName());
- }
-
- public void testExistsConnection() throws Exception {
- // There shouldn't be anything on empty repository
- assertFalse(handler.existsConnection(1, getDerbyConnection()));
- assertFalse(handler.existsConnection(2, getDerbyConnection()));
- assertFalse(handler.existsConnection(3, getDerbyConnection()));
-
- loadConnections();
-
- assertTrue(handler.existsConnection(1, getDerbyConnection()));
- assertTrue(handler.existsConnection(2, getDerbyConnection()));
- assertFalse(handler.existsConnection(3, getDerbyConnection()));
- }
-
- public void testCreateConnection() throws Exception {
- MConnection connection = getConnection();
-
- // Load some data
- fillConnection(connection);
-
- handler.createConnection(connection, getDerbyConnection());
-
- assertEquals(1, connection.getPersistenceId());
- assertCountForTable("SQOOP.SQ_CONNECTION", 1);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
-
- MConnection retrieved = handler.findConnection(1, getDerbyConnection());
- assertEquals(1, retrieved.getPersistenceId());
-
- List<MForm> forms;
- forms = connection.getConnectorPart().getForms();
- assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value2", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- forms = connection.getFrameworkPart().getForms();
- assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- // Let's create second connection
- connection = getConnection();
- fillConnection(connection);
-
- handler.createConnection(connection, getDerbyConnection());
-
- assertEquals(2, connection.getPersistenceId());
- assertCountForTable("SQOOP.SQ_CONNECTION", 2);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 8);
- }
-
- public void testInUseConnection() throws Exception {
- loadConnections();
-
- assertFalse(handler.inUseConnection(1, getDerbyConnection()));
-
- loadJobs();
-
- assertTrue(handler.inUseConnection(1, getDerbyConnection()));
- }
-
- public void testUpdateConnection() throws Exception {
- loadConnections();
-
- MConnection connection = handler.findConnection(1, getDerbyConnection());
-
- List<MForm> forms;
-
- forms = connection.getConnectorPart().getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(1).getInputs().get(1)).setValue(null);
-
- forms = connection.getFrameworkPart().getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
-
- connection.setName("name");
-
- handler.updateConnection(connection, getDerbyConnection());
-
- assertEquals(1, connection.getPersistenceId());
- assertCountForTable("SQOOP.SQ_CONNECTION", 2);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 10);
-
- MConnection retrieved = handler.findConnection(1, getDerbyConnection());
- assertEquals("name", connection.getName());
-
- forms = retrieved.getConnectorPart().getForms();
- assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- forms = retrieved.getFrameworkPart().getForms();
- assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
- assertNotNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
- assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
- assertNotNull(forms.get(1).getInputs().get(1).getValue());
- assertEquals(((Map)forms.get(1).getInputs().get(1).getValue()).size(), 0);
- }
-
- public void testEnableAndDisableConnection() throws Exception {
- loadConnections();
-
- // disable connection 1
- handler.enableConnection(1, false, getDerbyConnection());
-
- MConnection retrieved = handler.findConnection(1, getDerbyConnection());
- assertNotNull(retrieved);
- assertEquals(false, retrieved.getEnabled());
-
- // enable connection 1
- handler.enableConnection(1, true, getDerbyConnection());
-
- retrieved = handler.findConnection(1, getDerbyConnection());
- assertNotNull(retrieved);
- assertEquals(true, retrieved.getEnabled());
- }
-
- public void testDeleteConnection() throws Exception {
- loadConnections();
-
- handler.deleteConnection(1, getDerbyConnection());
- assertCountForTable("SQOOP.SQ_CONNECTION", 1);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
-
- handler.deleteConnection(2, getDerbyConnection());
- assertCountForTable("SQOOP.SQ_CONNECTION", 0);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 0);
- }
-
- public MConnection getConnection() {
- return new MConnection(1,
- handler.findConnector("A", getDerbyConnection()).getConnectionForms(),
- handler.findFramework(getDerbyConnection()).getConnectionForms()
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
index 745e128..8a5823d 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
@@ -40,14 +40,14 @@ public class TestConnectorHandling extends DerbyTestCase {
public void testFindConnector() throws Exception {
// On empty repository, no connectors should be there
- assertNull(handler.findConnector("A", getDerbyConnection()));
- assertNull(handler.findConnector("B", getDerbyConnection()));
+ assertNull(handler.findConnector("A", getDerbyDatabaseConnection()));
+ assertNull(handler.findConnector("B", getDerbyDatabaseConnection()));
// Load connector into repository
- loadConnectorAndFramework();
+ loadConnectorAndDriverConfig();
// Retrieve it
- MConnector connector = handler.findConnector("A", getDerbyConnection());
+ MConnector connector = handler.findConnector("A", getDerbyDatabaseConnection());
assertNotNull(connector);
// Get original structure
@@ -59,13 +59,13 @@ public class TestConnectorHandling extends DerbyTestCase {
public void testFindAllConnectors() throws Exception {
// No connectors in an empty repository, we expect an empty list
- assertEquals(handler.findConnectors(getDerbyConnection()).size(),0);
+ assertEquals(handler.findConnectors(getDerbyDatabaseConnection()).size(),0);
- loadConnectorAndFramework();
+ loadConnectorAndDriverConfig();
addConnector();
// Retrieve connectors
- List<MConnector> connectors = handler.findConnectors(getDerbyConnection());
+ List<MConnector> connectors = handler.findConnectors(getDerbyDatabaseConnection());
assertNotNull(connectors);
assertEquals(connectors.size(),2);
assertEquals(connectors.get(0).getUniqueName(),"A");
@@ -77,7 +77,7 @@ public class TestConnectorHandling extends DerbyTestCase {
public void testRegisterConnector() throws Exception {
MConnector connector = getConnector();
- handler.registerConnector(connector, getDerbyConnection());
+ handler.registerConnector(connector, getDerbyDatabaseConnection());
// Connector should get persistence ID
assertEquals(1, connector.getPersistenceId());
@@ -88,7 +88,7 @@ public class TestConnectorHandling extends DerbyTestCase {
assertCountForTable("SQOOP.SQ_INPUT", 12);
// Registered connector should be easily recovered back
- MConnector retrieved = handler.findConnector("A", getDerbyConnection());
+ MConnector retrieved = handler.findConnector("A", getDerbyDatabaseConnection());
assertNotNull(retrieved);
assertEquals(connector, retrieved);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
new file mode 100644
index 0000000..567dda6
--- /dev/null
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestDriverConfigHandling.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.repository.derby;
+
+import org.apache.sqoop.driver.Driver;
+import org.apache.sqoop.model.MDriverConfig;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Test driver config methods on Derby repository.
+ */
+public class TestDriverConfigHandling extends DerbyTestCase {
+
+ DerbyRepositoryHandler handler;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+ }
+
+ public void testFindDriverConfig() throws Exception {
+ // On empty repository, no driverConfig should be there
+ assertNull(handler.findDriverConfig(getDerbyDatabaseConnection()));
+ // Load Connector and DriverConfig into repository
+ loadConnectorAndDriverConfig();
+ // Retrieve it
+ MDriverConfig driverConfig = handler.findDriverConfig(getDerbyDatabaseConnection());
+ assertNotNull(driverConfig);
+
+ // Get original structure
+ MDriverConfig originalDriverConfig = getDriverConfig();
+
+ // And compare them
+ assertEquals(originalDriverConfig, driverConfig);
+ }
+
+ public void testRegisterConnector() throws Exception {
+ MDriverConfig driverConfig = getDriverConfig();
+ handler.registerDriverConfig(driverConfig, getDerbyDatabaseConnection());
+
+ // Connector should get persistence ID
+ assertEquals(1, driverConfig.getPersistenceId());
+
+ // Now check content in corresponding tables
+ assertCountForTable("SQOOP.SQ_CONNECTOR", 0);
+ assertCountForTable("SQOOP.SQ_FORM", 4);
+ assertCountForTable("SQOOP.SQ_INPUT", 8);
+
+ // Registered framework should be easily recovered back
+ MDriverConfig retrieved = handler.findDriverConfig(getDerbyDatabaseConnection());
+ assertNotNull(retrieved);
+ assertEquals(driverConfig, retrieved);
+ assertEquals(driverConfig.getVersion(), retrieved.getVersion());
+ }
+
+ private String getDriverVersion() throws Exception {
+ final String frameworkVersionQuery =
+ "SELECT SQM_VALUE FROM SQOOP.SQ_SYSTEM WHERE SQM_KEY=?";
+ String retVal = null;
+ PreparedStatement preparedStmt = null;
+ ResultSet resultSet = null;
+ try {
+ preparedStmt =
+ getDerbyDatabaseConnection().prepareStatement(frameworkVersionQuery);
+ preparedStmt.setString(1, DerbyRepoConstants.SYSKEY_DRIVER_VERSION);
+ resultSet = preparedStmt.executeQuery();
+ if(resultSet.next())
+ retVal = resultSet.getString(1);
+ return retVal;
+ } finally {
+ if(preparedStmt !=null) {
+ try {
+ preparedStmt.close();
+ } catch(SQLException e) {
+ }
+ }
+ if(resultSet != null) {
+ try {
+ resultSet.close();
+ } catch(SQLException e) {
+ }
+ }
+ }
+ }
+
+ public void testDriverVersion() throws Exception {
+ handler.registerDriverConfig(getDriverConfig(), getDerbyDatabaseConnection());
+
+ final String lowerVersion = Integer.toString(
+ Integer.parseInt(Driver.CURRENT_DRIVER_VERSION) - 1);
+ assertEquals(Driver.CURRENT_DRIVER_VERSION, getDriverVersion());
+ runQuery("UPDATE SQOOP.SQ_SYSTEM SET SQM_VALUE='" + lowerVersion +
+ "' WHERE SQM_KEY = '" + DerbyRepoConstants.SYSKEY_DRIVER_VERSION + "'");
+ assertEquals(lowerVersion, getDriverVersion());
+
+ MDriverConfig framework = getDriverConfig();
+ handler.updateDriverConfig(framework, getDerbyDatabaseConnection());
+
+ assertEquals(Driver.CURRENT_DRIVER_VERSION, framework.getVersion());
+
+ assertEquals(Driver.CURRENT_DRIVER_VERSION, getDriverVersion());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java
deleted file mode 100644
index 006ec9c..0000000
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.repository.derby;
-
-import org.apache.sqoop.framework.FrameworkManager;
-import org.apache.sqoop.model.MFramework;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-/**
- * Test framework methods on Derby repository.
- */
-public class TestFrameworkHandling extends DerbyTestCase {
-
- DerbyRepositoryHandler handler;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
-
- handler = new DerbyRepositoryHandler();
-
- // We always needs schema for this test case
- createSchema();
- }
-
- public void testFindFramework() throws Exception {
- // On empty repository, no framework should be there
- assertNull(handler.findFramework(getDerbyConnection()));
-
- // Load framework into repository
- loadConnectorAndFramework();
-
- // Retrieve it
- MFramework framework = handler.findFramework(getDerbyConnection());
- assertNotNull(framework);
-
- // Get original structure
- MFramework original = getFramework();
-
- // And compare them
- assertEquals(original, framework);
- }
-
- public void testRegisterConnector() throws Exception {
- MFramework framework = getFramework();
-
- handler.registerFramework(framework, getDerbyConnection());
-
- // Connector should get persistence ID
- assertEquals(1, framework.getPersistenceId());
-
- // Now check content in corresponding tables
- assertCountForTable("SQOOP.SQ_CONNECTOR", 0);
- assertCountForTable("SQOOP.SQ_FORM", 4);
- assertCountForTable("SQOOP.SQ_INPUT", 8);
-
- // Registered framework should be easily recovered back
- MFramework retrieved = handler.findFramework(getDerbyConnection());
- assertNotNull(retrieved);
- assertEquals(framework, retrieved);
- assertEquals(framework.getVersion(), retrieved.getVersion());
- }
-
- private String getFrameworkVersion() throws Exception {
- final String frameworkVersionQuery =
- "SELECT SQM_VALUE FROM SQOOP.SQ_SYSTEM WHERE SQM_KEY=?";
- String retVal = null;
- PreparedStatement preparedStmt = null;
- ResultSet resultSet = null;
- try {
- preparedStmt =
- getDerbyConnection().prepareStatement(frameworkVersionQuery);
- preparedStmt.setString(1, DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION);
- resultSet = preparedStmt.executeQuery();
- if(resultSet.next())
- retVal = resultSet.getString(1);
- return retVal;
- } finally {
- if(preparedStmt !=null) {
- try {
- preparedStmt.close();
- } catch(SQLException e) {
- }
- }
- if(resultSet != null) {
- try {
- resultSet.close();
- } catch(SQLException e) {
- }
- }
- }
- }
-
- public void testFrameworkVersion() throws Exception {
- handler.registerFramework(getFramework(), getDerbyConnection());
-
- final String lowerVersion = Integer.toString(
- Integer.parseInt(FrameworkManager.CURRENT_FRAMEWORK_VERSION) - 1);
- assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, getFrameworkVersion());
- runQuery("UPDATE SQOOP.SQ_SYSTEM SET SQM_VALUE='" + lowerVersion +
- "' WHERE SQM_KEY = '" + DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION + "'");
- assertEquals(lowerVersion, getFrameworkVersion());
-
- MFramework framework = getFramework();
- handler.updateFramework(framework, getDerbyConnection());
-
- assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, framework.getVersion());
-
- assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, getFrameworkVersion());
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
index 15f9539..c4a5829 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
@@ -18,12 +18,12 @@
package org.apache.sqoop.repository.derby;
import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MEnumInput;
import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MMapInput;
@@ -56,17 +56,17 @@ public class TestInputTypes extends DerbyTestCase {
* Ensure that metadata with all various data types can be successfully
* serialized into repository and retrieved back.
*/
- public void testMetadataSerialization() throws Exception {
+ public void testEntitySerialization() throws Exception {
MConnector connector = getConnector();
// Serialize the connector with all data types into repository
- handler.registerConnector(connector, getDerbyConnection());
+ handler.registerConnector(connector, getDerbyDatabaseConnection());
// Successful serialization should update the ID
assertNotSame(connector.getPersistenceId(), MPersistableEntity.PERSISTANCE_ID_DEFAULT);
// Retrieve registered connector
- MConnector retrieved = handler.findConnector(connector.getUniqueName(), getDerbyConnection());
+ MConnector retrieved = handler.findConnector(connector.getUniqueName(), getDerbyDatabaseConnection());
assertNotNull(retrieved);
// Original and retrieved connectors should be the same
@@ -76,33 +76,33 @@ public class TestInputTypes extends DerbyTestCase {
/**
* Test that serializing actual data is not an issue.
*/
- public void testDataSerialization() throws Exception {
+ public void testEntityDataSerialization() throws Exception {
MConnector connector = getConnector();
- MFramework framework = getFramework();
+ MDriverConfig driverConfig = getDriverConfig();
- // Register metadata for everything and our new connector
- handler.registerConnector(connector, getDerbyConnection());
- handler.registerFramework(framework, getDerbyConnection());
+ // Register objects for everything and our new connector
+ handler.registerConnector(connector, getDerbyDatabaseConnection());
+ handler.registerDriverConfig(driverConfig, getDerbyDatabaseConnection());
// Inserted values
Map<String, String> map = new HashMap<String, String>();
map.put("A", "B");
// Connection object with all various values
- MConnection connection = new MConnection(connector.getPersistenceId(), connector.getConnectionForms(), framework.getConnectionForms());
- MConnectionForms forms = connection.getConnectorPart();
+ MLink link = new MLink(connector.getPersistenceId(), connector.getConnectionForms(), driverConfig.getConnectionForms());
+ MConnectionForms forms = link.getConnectorPart();
forms.getStringInput("f.String").setValue("A");
forms.getMapInput("f.Map").setValue(map);
forms.getIntegerInput("f.Integer").setValue(1);
forms.getBooleanInput("f.Boolean").setValue(true);
forms.getEnumInput("f.Enum").setValue("YES");
- // Create the connection in repository
- handler.createConnection(connection, getDerbyConnection());
- assertNotSame(connection.getPersistenceId(), MPersistableEntity.PERSISTANCE_ID_DEFAULT);
+ // Create the link in repository
+ handler.createLink(link, getDerbyDatabaseConnection());
+ assertNotSame(link.getPersistenceId(), MPersistableEntity.PERSISTANCE_ID_DEFAULT);
- // Retrieve created connection
- MConnection retrieved = handler.findConnection(connection.getPersistenceId(), getDerbyConnection());
+ // Retrieve created link
+ MLink retrieved = handler.findLink(link.getPersistenceId(), getDerbyDatabaseConnection());
forms = retrieved.getConnectorPart();
assertEquals("A", forms.getStringInput("f.String").getValue());
assertEquals(map, forms.getMapInput("f.Map").getValue());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
index 913439b..37aa8d4 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
@@ -34,25 +34,25 @@ public class TestInternals extends DerbyTestCase {
}
public void testSuitableInternals() throws Exception {
- assertFalse(handler.haveSuitableInternals(getDerbyConnection()));
+ assertFalse(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
createSchema(); // Test code is building the structures
- assertTrue(handler.haveSuitableInternals(getDerbyConnection()));
+ assertTrue(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
}
public void testCreateorUpdateInternals() throws Exception {
- assertFalse(handler.haveSuitableInternals(getDerbyConnection()));
- handler.createOrUpdateInternals(getDerbyConnection());
- assertTrue(handler.haveSuitableInternals(getDerbyConnection()));
+ assertFalse(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
+ handler.createOrUpdateInternals(getDerbyDatabaseConnection());
+ assertTrue(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
}
public void testUpgradeVersion2ToVersion4() throws Exception {
createSchema(2);
- assertFalse(handler.haveSuitableInternals(getDerbyConnection()));
- loadConnectorAndFramework(2);
- loadConnections(2);
+ assertFalse(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
+ loadConnectorAndDriverConfig(2);
+ loadLinks(2);
loadJobs(2);
- handler.createOrUpdateInternals(getDerbyConnection());
- assertTrue(handler.haveSuitableInternals(getDerbyConnection()));
+ handler.createOrUpdateInternals(getDerbyDatabaseConnection());
+ assertTrue(handler.haveSuitableInternals(getDerbyDatabaseConnection()));
}
private class TestDerbyRepositoryHandler extends DerbyRepositoryHandler {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
index e658c11..f22c351 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
@@ -45,16 +45,16 @@ public class TestJobHandling extends DerbyTestCase {
createSchema();
// We always needs connector and framework structures in place
- loadConnectorAndFramework();
+ loadConnectorAndDriverConfig();
// We always needs connection metadata in place
- loadConnections();
+ loadLinks();
}
public void testFindJob() throws Exception {
// Let's try to find non existing job
try {
- handler.findJob(1, getDerbyConnection());
+ handler.findJob(1, getDerbyDatabaseConnection());
fail();
} catch(SqoopException ex) {
assertEquals(DerbyRepoError.DERBYREPO_0030, ex.getErrorCode());
@@ -63,7 +63,7 @@ public class TestJobHandling extends DerbyTestCase {
// Load prepared connections into database
loadJobs();
- MJob jobImport = handler.findJob(1, getDerbyConnection());
+ MJob jobImport = handler.findJob(1, getDerbyDatabaseConnection());
assertNotNull(jobImport);
assertEquals(1, jobImport.getPersistenceId());
assertEquals("JA", jobImport.getName());
@@ -98,13 +98,13 @@ public class TestJobHandling extends DerbyTestCase {
List<MJob> list;
// Load empty list on empty repository
- list = handler.findJobs(getDerbyConnection());
+ list = handler.findJobs(getDerbyDatabaseConnection());
assertEquals(0, list.size());
loadJobs();
// Load all two connections on loaded repository
- list = handler.findJobs(getDerbyConnection());
+ list = handler.findJobs(getDerbyDatabaseConnection());
assertEquals(4, list.size());
assertEquals("JA", list.get(0).getName());
@@ -118,29 +118,29 @@ public class TestJobHandling extends DerbyTestCase {
public void testExistsJob() throws Exception {
// There shouldn't be anything on empty repository
- assertFalse(handler.existsJob(1, getDerbyConnection()));
- assertFalse(handler.existsJob(2, getDerbyConnection()));
- assertFalse(handler.existsJob(3, getDerbyConnection()));
- assertFalse(handler.existsJob(4, getDerbyConnection()));
- assertFalse(handler.existsJob(5, getDerbyConnection()));
+ assertFalse(handler.existsJob(1, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsJob(2, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsJob(3, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsJob(4, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsJob(5, getDerbyDatabaseConnection()));
loadJobs();
- assertTrue(handler.existsJob(1, getDerbyConnection()));
- assertTrue(handler.existsJob(2, getDerbyConnection()));
- assertTrue(handler.existsJob(3, getDerbyConnection()));
- assertTrue(handler.existsJob(4, getDerbyConnection()));
- assertFalse(handler.existsJob(5, getDerbyConnection()));
+ assertTrue(handler.existsJob(1, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsJob(2, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsJob(3, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsJob(4, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsJob(5, getDerbyDatabaseConnection()));
}
public void testInUseJob() throws Exception {
loadJobs();
loadSubmissions();
- assertTrue(handler.inUseJob(1, getDerbyConnection()));
- assertFalse(handler.inUseJob(2, getDerbyConnection()));
- assertFalse(handler.inUseJob(3, getDerbyConnection()));
- assertFalse(handler.inUseJob(4, getDerbyConnection()));
+ assertTrue(handler.inUseJob(1, getDerbyDatabaseConnection()));
+ assertFalse(handler.inUseJob(2, getDerbyDatabaseConnection()));
+ assertFalse(handler.inUseJob(3, getDerbyDatabaseConnection()));
+ assertFalse(handler.inUseJob(4, getDerbyDatabaseConnection()));
}
public void testCreateJob() throws Exception {
@@ -149,13 +149,13 @@ public class TestJobHandling extends DerbyTestCase {
// Load some data
fillJob(job);
- handler.createJob(job, getDerbyConnection());
+ handler.createJob(job, getDerbyDatabaseConnection());
assertEquals(1, job.getPersistenceId());
assertCountForTable("SQOOP.SQ_JOB", 1);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 6);
- MJob retrieved = handler.findJob(1, getDerbyConnection());
+ MJob retrieved = handler.findJob(1, getDerbyDatabaseConnection());
assertEquals(1, retrieved.getPersistenceId());
List<MForm> forms;
@@ -176,7 +176,7 @@ public class TestJobHandling extends DerbyTestCase {
job = getJob();
fillJob(job);
- handler.createJob(job, getDerbyConnection());
+ handler.createJob(job, getDerbyDatabaseConnection());
assertEquals(2, job.getPersistenceId());
assertCountForTable("SQOOP.SQ_JOB", 2);
@@ -189,7 +189,7 @@ public class TestJobHandling extends DerbyTestCase {
assertCountForTable("SQOOP.SQ_JOB", 4);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 24);
- MJob job = handler.findJob(1, getDerbyConnection());
+ MJob job = handler.findJob(1, getDerbyDatabaseConnection());
List<MForm> forms;
@@ -208,13 +208,13 @@ public class TestJobHandling extends DerbyTestCase {
job.setName("name");
- handler.updateJob(job, getDerbyConnection());
+ handler.updateJob(job, getDerbyDatabaseConnection());
assertEquals(1, job.getPersistenceId());
assertCountForTable("SQOOP.SQ_JOB", 4);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 26);
- MJob retrieved = handler.findJob(1, getDerbyConnection());
+ MJob retrieved = handler.findJob(1, getDerbyDatabaseConnection());
assertEquals("name", retrieved.getName());
forms = job.getConnectorPart(Direction.FROM).getForms();
@@ -237,16 +237,16 @@ public class TestJobHandling extends DerbyTestCase {
loadJobs();
// disable job 1
- handler.enableJob(1, false, getDerbyConnection());
+ handler.enableJob(1, false, getDerbyDatabaseConnection());
- MJob retrieved = handler.findJob(1, getDerbyConnection());
+ MJob retrieved = handler.findJob(1, getDerbyDatabaseConnection());
assertNotNull(retrieved);
assertEquals(false, retrieved.getEnabled());
// enable job 1
- handler.enableJob(1, true, getDerbyConnection());
+ handler.enableJob(1, true, getDerbyDatabaseConnection());
- retrieved = handler.findJob(1, getDerbyConnection());
+ retrieved = handler.findJob(1, getDerbyDatabaseConnection());
assertNotNull(retrieved);
assertEquals(true, retrieved.getEnabled());
}
@@ -254,28 +254,28 @@ public class TestJobHandling extends DerbyTestCase {
public void testDeleteJob() throws Exception {
loadJobs();
- handler.deleteJob(1, getDerbyConnection());
+ handler.deleteJob(1, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_JOB", 3);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 18);
- handler.deleteJob(2, getDerbyConnection());
+ handler.deleteJob(2, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_JOB", 2);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 12);
- handler.deleteJob(3, getDerbyConnection());
+ handler.deleteJob(3, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_JOB", 1);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 6);
- handler.deleteJob(4, getDerbyConnection());
+ handler.deleteJob(4, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_JOB", 0);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 0);
}
public MJob getJob() {
return new MJob(1, 1, 1, 1,
- handler.findConnector("A", getDerbyConnection()).getJobForms(Direction.FROM),
- handler.findConnector("A", getDerbyConnection()).getJobForms(Direction.TO),
- handler.findFramework(getDerbyConnection()).getJobForms()
+ handler.findConnector("A", getDerbyDatabaseConnection()).getJobForms(Direction.FROM),
+ handler.findConnector("A", getDerbyDatabaseConnection()).getJobForms(Direction.TO),
+ handler.findDriverConfig(getDerbyDatabaseConnection()).getJobForms()
);
}
}
[29/52] [abbrv] git commit: SQOOP-1560: Sqoop2: Move matcher out of
Schema
Posted by ab...@apache.org.
SQOOP-1560: Sqoop2: Move matcher out of Schema
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c1e53e5d
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c1e53e5d
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c1e53e5d
Branch: refs/heads/SQOOP-1367
Commit: c1e53e5d54e3e9453aefe74b2dd171ff9b25e5a3
Parents: 97da12a
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Oct 2 16:55:29 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:59:24 2014 -0700
----------------------------------------------------------------------
.../idf/CSVIntermediateDataFormat.java | 134 ++++-----
.../connector/idf/IntermediateDataFormat.java | 9 +-
.../connector/idf/matcher/AbstractMatcher.java | 62 -----
.../connector/idf/matcher/LocationMatcher.java | 82 ------
.../connector/idf/matcher/NameMatcher.java | 69 -----
.../connector/matcher/LocationMatcher.java | 78 ++++++
.../apache/sqoop/connector/matcher/Matcher.java | 69 +++++
.../sqoop/connector/matcher/MatcherError.java | 41 +++
.../sqoop/connector/matcher/MatcherFactory.java | 30 ++
.../sqoop/connector/matcher/NameMatcher.java | 74 +++++
.../idf/TestCSVIntermediateDataFormat.java | 160 ++---------
.../org/apache/sqoop/job/mr/SqoopMapper.java | 43 +--
.../job/mr/SqoopOutputFormatLoadExecutor.java | 16 +-
.../java/org/apache/sqoop/job/JobUtils.java | 12 +-
.../org/apache/sqoop/job/TestMapReduce.java | 12 +-
.../java/org/apache/sqoop/job/TestMatching.java | 275 +++++++++++++++++++
16 files changed, 681 insertions(+), 485 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
index 2a49221..02d1a51 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
@@ -22,12 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.idf.matcher.AbstractMatcher;
-import org.apache.sqoop.connector.idf.matcher.LocationMatcher;
-import org.apache.sqoop.connector.idf.matcher.NameMatcher;
import org.apache.sqoop.schema.Schema;
-import org.apache.sqoop.schema.SchemaError;
-import org.apache.sqoop.schema.SchemaMatchOption;
import org.apache.sqoop.schema.type.Column;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.FloatingPoint;
@@ -41,7 +36,6 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
import java.util.regex.Matcher;
public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
@@ -71,8 +65,7 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
private final List<Integer> stringFieldIndices = new ArrayList<Integer>();
private final List<Integer> byteFieldIndices = new ArrayList<Integer>();
- private Schema fromSchema;
- private Schema toSchema;
+ private Schema schema;
/**
* {@inheritDoc}
@@ -94,11 +87,11 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
* {@inheritDoc}
*/
@Override
- public void setFromSchema(Schema schema) {
+ public void setSchema(Schema schema) {
if(schema == null) {
return;
}
- this.fromSchema = schema;
+ this.schema = schema;
List<Column> columns = schema.getColumns();
int i = 0;
for(Column col : columns) {
@@ -112,19 +105,6 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
}
/**
- * {@inheritDoc}
- */
- @Override
- public void setToSchema(Schema schema) {
- if(schema == null) {
- return;
- }
- this.toSchema = schema;
- }
-
-
-
- /**
* Custom CSV parser that honors quoting and escaped quotes.
* All other escaping is handled elsewhere.
*
@@ -180,69 +160,68 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
/**
* {@inheritDoc}
- *
- * The CSV data is ordered according to the fromSchema. We "translate" it to the TO schema.
- * We currently have 3 methods of matching fields in one schema to another:
- * - by location
- * - by name
- * - user-defined matching
- *
- * If one schema exists (either to or from) and the other is empty
- * We'll match fields based on location.
- * If both schemas exist, we'll match names of fields.
- *
- * In the future, we may want to let users choose the method
- * Currently nothing is implemented for user-defined matching
*/
@Override
public Object[] getObjectData() {
+ if (schema.isEmpty()) {
+ throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0006);
+ }
+
String[] fields = getFields();
+
if (fields == null) {
return null;
}
- if (fromSchema == null || toSchema == null || (toSchema.isEmpty() && fromSchema.isEmpty())) {
- throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0006);
+ if (fields.length != schema.getColumns().size()) {
+ throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0005,
+ "The data " + getTextData() + " has the wrong number of fields.");
}
- AbstractMatcher matcher = getMatcher(fromSchema,toSchema);
- String[] outFields = matcher.getMatchingData(fields, fromSchema, toSchema);
- Object[] out = new Object[outFields.length];
-
- int i = 0;
-
- // After getting back the data in order that matches the output schema
- // We need to un-do the CSV escaping
- for (Column col: matcher.getMatchingSchema(fromSchema,toSchema).getColumns()) {
- Type colType = col.getType();
- if (outFields[i] == null) {
+ Object[] out = new Object[fields.length];
+ Column[] cols = schema.getColumns().toArray(new Column[fields.length]);
+ for (int i = 0; i < fields.length; i++) {
+ Type colType = cols[i].getType();
+ if (fields[i].equals("NULL")) {
out[i] = null;
continue;
}
- if (colType == Type.TEXT) {
- out[i] = unescapeStrings(outFields[i]);
- } else if (colType == Type.BINARY) {
- out[i] = unescapeByteArray(outFields[i]);
- } else if (colType == Type.FIXED_POINT) {
- Long byteSize = ((FixedPoint) col).getByteSize();
- if (byteSize != null && byteSize <= Integer.SIZE) {
- out[i] = Integer.valueOf(outFields[i]);
- } else {
- out[i] = Long.valueOf(outFields[i]);
- }
- } else if (colType == Type.FLOATING_POINT) {
- Long byteSize = ((FloatingPoint) col).getByteSize();
- if (byteSize != null && byteSize <= Float.SIZE) {
- out[i] = Float.valueOf(outFields[i]);
- } else {
- out[i] = Double.valueOf(outFields[i]);
- }
- } else if (colType == Type.DECIMAL) {
- out[i] = new BigDecimal(outFields[i]);
- } else {
- throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0004, "Column type from schema was not recognized for " + colType);
+
+ Long byteSize;
+ switch(colType) {
+ case TEXT:
+ out[i] = unescapeStrings(fields[i]);
+ break;
+ case BINARY:
+ out[i] = unescapeByteArray(fields[i]);
+ break;
+ case FIXED_POINT:
+ byteSize = ((FixedPoint) cols[i]).getByteSize();
+ if (byteSize != null && byteSize <= Integer.SIZE) {
+ out[i] = Integer.valueOf(fields[i]);
+ } else {
+ out[i] = Long.valueOf(fields[i]);
+ }
+ break;
+ case FLOATING_POINT:
+ byteSize = ((FloatingPoint) cols[i]).getByteSize();
+ if (byteSize != null && byteSize <= Float.SIZE) {
+ out[i] = Float.valueOf(fields[i]);
+ } else {
+ out[i] = Double.valueOf(fields[i]);
+ }
+ break;
+ case DECIMAL:
+ out[i] = new BigDecimal(fields[i]);
+ break;
+ case DATE:
+ case DATE_TIME:
+ case BIT:
+ out[i] = fields[i];
+ break;
+ default:
+ throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0004, "Column type from schema was not recognized for " + colType);
}
- i++;
}
return out;
}
@@ -380,15 +359,4 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
public String toString() {
return data;
}
-
- private AbstractMatcher getMatcher(Schema fromSchema, Schema toSchema) {
- if (toSchema.isEmpty() || fromSchema.isEmpty()) {
- return new LocationMatcher();
- } else {
- return new NameMatcher();
- }
-
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
index d98b779..5ef6fc6 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
@@ -111,14 +111,7 @@ public abstract class IntermediateDataFormat<T> {
*
* @param schema - the schema used for reading data
*/
- public abstract void setFromSchema(Schema schema);
-
- /**
- * Set the schema for writing data.
- *
- * @param schema - the schema used for writing data
- */
- public abstract void setToSchema(Schema schema);
+ public abstract void setSchema(Schema schema);
/**
* Serialize the fields of this object to <code>out</code>.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/AbstractMatcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/AbstractMatcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/AbstractMatcher.java
deleted file mode 100644
index e6b2316..0000000
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/AbstractMatcher.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.idf.matcher;
-
-import org.apache.sqoop.schema.Schema;
-import org.apache.sqoop.schema.type.Column;
-
-public abstract class AbstractMatcher {
-
- //NOTE: This is currently tightly coupled to the CSV idf. We'll need refactoring after adding additional formats
- //NOTE: There's is a very blatant special case of empty schemas that seem to apply only to HDFS.
-
- /**
- *
- * @param fields
- * @param fromSchema
- * @param toSchema
- * @return Return the data in "fields" converted from matching the fromSchema to matching the toSchema.
- * Right not "converted" means re-ordering if needed and handling nulls.
- */
- abstract public String[] getMatchingData(String[] fields, Schema fromSchema, Schema toSchema);
-
- /***
- *
- * @param fromSchema
- * @param toSchema
- * @return return a schema with which to read the output data
- * This always returns the toSchema (since this is used when getting output data), unless its empty
- */
- public Schema getMatchingSchema(Schema fromSchema, Schema toSchema) {
- if (toSchema.isEmpty()) {
- return fromSchema;
- } else {
- return toSchema;
- }
-
- }
-
- protected boolean isNull(String value) {
- if (value.equals("NULL") || value.equals("null") || value.equals("'null'") || value.isEmpty()) {
- return true;
- }
- return false;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/LocationMatcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/LocationMatcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/LocationMatcher.java
deleted file mode 100644
index 938a5df..0000000
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/LocationMatcher.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.idf.matcher;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.idf.IntermediateDataFormatError;
-import org.apache.sqoop.schema.Schema;
-import org.apache.sqoop.schema.SchemaError;
-import org.apache.sqoop.schema.SchemaMatchOption;
-import org.apache.sqoop.schema.type.Column;
-import org.apache.sqoop.schema.type.FixedPoint;
-import org.apache.sqoop.schema.type.FloatingPoint;
-import org.apache.sqoop.schema.type.Type;
-
-import java.math.BigDecimal;
-import java.util.Iterator;
-
-
-/**
- * Convert data according to FROM schema to data according to TO schema
- * This is done based on column location
- * So data in first column in FROM goes into first column in TO, etc
- * If TO schema has more fields and they are "nullable", the value will be set to null
- * If TO schema has extra non-null fields, we'll throw an exception
- */
-public class LocationMatcher extends AbstractMatcher {
-
- public static final Logger LOG = Logger.getLogger(LocationMatcher.class);
- @Override
- public String[] getMatchingData(String[] fields, Schema fromSchema, Schema toSchema) {
-
- String[] out = new String[toSchema.getColumns().size()];
-
- int i = 0;
-
- if (toSchema.isEmpty()) {
- // If there's no destination schema, no need to convert anything
- // Just use the original data
- return fields;
- }
-
- for (Column col: toSchema.getColumns())
- {
- if (i < fields.length) {
- if (isNull(fields[i])) {
- out[i] = null;
- } else {
- out[i] = fields[i];
- }
- }
- // We ran out of fields before we ran out of schema
- else {
- if (!col.getNullable()) {
- throw new SqoopException(SchemaError.SCHEMA_0004,"target column " + col + " didn't match with any source column and cannot be null");
- } else {
- LOG.warn("Column " + col + " has no matching source column. Will be ignored. ");
- out[i] = null;
- }
- }
- i++;
- }
- return out;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/NameMatcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/NameMatcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/NameMatcher.java
deleted file mode 100644
index 417c85b..0000000
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/matcher/NameMatcher.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.connector.idf.matcher;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.schema.Schema;
-import org.apache.sqoop.schema.SchemaError;
-import org.apache.sqoop.schema.type.Column;
-
-import java.util.HashMap;
-
-public class NameMatcher extends AbstractMatcher {
- public static final Logger LOG = Logger.getLogger(NameMatcher.class);
-
- @Override
- public String[] getMatchingData(String[] fields, Schema fromSchema, Schema toSchema) {
- String[] out = new String[toSchema.getColumns().size()];
-
- HashMap<String,Column> colNames = new HashMap<String, Column>();
-
- for (Column fromCol: fromSchema.getColumns()) {
- colNames.put(fromCol.getName(), fromCol);
- }
-
- int toIndex = 0;
-
- for (Column toCol: toSchema.getColumns()) {
- Column fromCol = colNames.get(toCol.getName());
-
- if (fromCol != null) {
- int fromIndex = fromSchema.getColumns().indexOf(fromCol);
- if (isNull(fields[fromIndex])) {
- out[toIndex] = null;
- } else {
- out[toIndex] = fields[fromIndex];
- }
- } else {
- //column exists in TO schema but not in FROM schema
- if (toCol.getNullable() == false) {
- throw new SqoopException(SchemaError.SCHEMA_0004,"target column " + toCol + " didn't match with any source column and cannot be null");
- } else {
- LOG.warn("Column " + toCol + " has no matching source column. Will be ignored. ");
- out[toIndex] = null;
- }
- }
-
- toIndex++;
- }
-
- return out;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/LocationMatcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/LocationMatcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/LocationMatcher.java
new file mode 100644
index 0000000..58b709e
--- /dev/null
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/LocationMatcher.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.matcher;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.SchemaError;
+import org.apache.sqoop.schema.type.Column;
+
+
+/**
+ * Convert data according to FROM schema to data according to TO schema
+ * This is done based on column location
+ * So data in first column in FROM goes into first column in TO, etc
+ * If TO schema has more fields and they are "nullable", the value will be set to null
+ * If TO schema has extra non-null fields, we'll throw an exception
+ */
+public class LocationMatcher extends Matcher {
+
+ public static final Logger LOG = Logger.getLogger(LocationMatcher.class);
+
+ public LocationMatcher(Schema from, Schema to) {
+ super(from, to);
+ }
+
+ @Override
+ public Object[] getMatchingData(Object[] fields) {
+
+ Object[] out = new Object[getToSchema().getColumns().size()];
+
+ int i = 0;
+
+ if (getToSchema().isEmpty()) {
+ // If there's no destination schema, no need to convert anything
+ // Just use the original data
+ return fields;
+ }
+
+ for (Column col: getToSchema().getColumns()) {
+ if (i < fields.length) {
+ if (isNull(fields[i])) {
+ out[i] = null;
+ } else {
+ out[i] = fields[i];
+ }
+ }
+ // We ran out of fields before we ran out of schema
+ else {
+ if (!col.getNullable()) {
+ throw new SqoopException(SchemaError.SCHEMA_0004,"target column " + col + " didn't match with any source column and cannot be null");
+ } else {
+ LOG.warn("Column " + col + " has no matching source column. Will be ignored. ");
+ out[i] = null;
+ }
+ }
+ i++;
+ }
+ return out;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/Matcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/Matcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/Matcher.java
new file mode 100644
index 0000000..8ab1318
--- /dev/null
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/Matcher.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.matcher;
+
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.schema.Schema;
+
+public abstract class Matcher {
+
+ private final Schema fromSchema;
+ private final Schema toSchema;
+
+ public Matcher(Schema fromSchema, Schema toSchema) {
+ if (fromSchema.isEmpty() && toSchema.isEmpty()) {
+ throw new SqoopException(MatcherError.MATCHER_0000, "Neither a FROM or TO schemas been provided.");
+ } else if (toSchema.isEmpty()) {
+ this.fromSchema = fromSchema;
+ this.toSchema = fromSchema;
+ } else if (fromSchema.isEmpty()) {
+ this.fromSchema = toSchema;
+ this.toSchema = toSchema;
+ } else {
+ this.fromSchema = fromSchema;
+ this.toSchema = toSchema;
+ }
+ }
+
+ /**
+ *
+ * @param fields
+ * @return Return the data in "fields" converted from matching the fromSchema to matching the toSchema.
+ * Right not "converted" means re-ordering if needed and handling nulls.
+ */
+ abstract public Object[] getMatchingData(Object[] fields);
+
+ public Schema getFromSchema() {
+ return fromSchema;
+ }
+
+ public Schema getToSchema() {
+ return toSchema;
+ }
+
+ protected boolean isNull(Object value) {
+ if (value == null || value.equals("NULL")
+ || value.equals("null") || value.equals("'null'")
+ || value.equals("")) {
+ return true;
+ }
+ return false;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/MatcherError.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/MatcherError.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/MatcherError.java
new file mode 100644
index 0000000..577b091
--- /dev/null
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/MatcherError.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.connector.matcher;
+
+import org.apache.sqoop.common.ErrorCode;
+
+public enum MatcherError implements ErrorCode {
+ MATCHER_0000("To few Schemas provided."),
+
+ ;
+
+ private final String message;
+
+ private MatcherError(String message) {
+ this.message = message;
+ }
+
+ public String getCode() {
+ return name();
+ }
+
+ public String getMessage() {
+ return message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/MatcherFactory.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/MatcherFactory.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/MatcherFactory.java
new file mode 100644
index 0000000..ae89e6c
--- /dev/null
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/MatcherFactory.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.matcher;
+
+import org.apache.sqoop.schema.Schema;
+
+public class MatcherFactory {
+ public static Matcher getMatcher(Schema fromSchema, Schema toSchema) {
+ if (toSchema.isEmpty() || fromSchema.isEmpty()) {
+ return new LocationMatcher(fromSchema, toSchema);
+ } else {
+ return new NameMatcher(fromSchema, toSchema);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/NameMatcher.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/NameMatcher.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/NameMatcher.java
new file mode 100644
index 0000000..69d5ebd
--- /dev/null
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/matcher/NameMatcher.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.matcher;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.SchemaError;
+import org.apache.sqoop.schema.type.Column;
+
+import java.util.HashMap;
+
+public class NameMatcher extends Matcher {
+
+ public static final Logger LOG = Logger.getLogger(NameMatcher.class);
+
+ public NameMatcher(Schema from, Schema to) {
+ super(from, to);
+ }
+
+ @Override
+ public Object[] getMatchingData(Object[] fields) {
+ Object[] out = new Object[getToSchema().getColumns().size()];
+
+ HashMap<String,Column> colNames = new HashMap<String, Column>();
+
+ for (Column fromCol: getFromSchema().getColumns()) {
+ colNames.put(fromCol.getName(), fromCol);
+ }
+
+ int toIndex = 0;
+
+ for (Column toCol: getToSchema().getColumns()) {
+ Column fromCol = colNames.get(toCol.getName());
+
+ if (fromCol != null) {
+ int fromIndex = getFromSchema().getColumns().indexOf(fromCol);
+ if (isNull(fields[fromIndex])) {
+ out[toIndex] = null;
+ } else {
+ out[toIndex] = fields[fromIndex];
+ }
+ } else {
+ //column exists in TO schema but not in FROM schema
+ if (toCol.getNullable() == false) {
+ throw new SqoopException(SchemaError.SCHEMA_0004,"target column " + toCol + " didn't match with any source column and cannot be null");
+ } else {
+ LOG.warn("Column " + toCol + " has no matching source column. Will be ignored. ");
+ out[toIndex] = null;
+ }
+ }
+
+ toIndex++;
+ }
+
+ return out;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
index 3954039..765bedd 100644
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
@@ -20,7 +20,6 @@ package org.apache.sqoop.connector.idf;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.schema.Schema;
-import org.apache.sqoop.schema.SchemaMatchOption;
import org.apache.sqoop.schema.type.Binary;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.Text;
@@ -40,8 +39,6 @@ public class TestCSVIntermediateDataFormat {
private IntermediateDataFormat<?> data;
- private Schema emptySchema = new Schema("empty");
-
@Before
public void setUp() {
data = new CSVIntermediateDataFormat();
@@ -73,7 +70,7 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setFromSchema(schema);
+ data.setSchema(schema);
data.setTextData(null);
Object[] out = data.getObjectData();
@@ -90,7 +87,7 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setFromSchema(schema);
+ data.setSchema(schema);
data.setTextData("");
data.getObjectData();
@@ -110,8 +107,7 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setFromSchema(schema);
- data.setToSchema(emptySchema);
+ data.setSchema(schema);
data.setTextData(testData);
Object[] out = data.getObjectData();
@@ -120,7 +116,7 @@ public class TestCSVIntermediateDataFormat {
assertEquals(new Long(34),out[1]);
assertEquals("54",out[2]);
assertEquals("random data",out[3]);
- assertEquals(-112, ((byte[])out[4])[0]);
+ assertEquals(-112, ((byte[]) out[4])[0]);
assertEquals(54, ((byte[])out[4])[1]);
assertEquals("\n", out[5].toString());
}
@@ -134,7 +130,7 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setFromSchema(schema);
+ data.setSchema(schema);
byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54};
Object[] in = new Object[6];
@@ -164,8 +160,7 @@ public class TestCSVIntermediateDataFormat {
.addColumn(new Text("4"))
.addColumn(new Binary("5"))
.addColumn(new Text("6"));
- data.setFromSchema(schema);
- data.setToSchema(emptySchema);
+ data.setSchema(schema);
Object[] in = new Object[6];
in[0] = new Long(10);
@@ -188,8 +183,7 @@ public class TestCSVIntermediateDataFormat {
Schema schema = new Schema("test");
schema.addColumn(new Text("1"));
- data.setFromSchema(schema);
- data.setToSchema(emptySchema);
+ data.setSchema(schema);
char[] allCharArr = new char[256];
for(int i = 0; i < allCharArr.length; ++i) {
@@ -212,148 +206,30 @@ public class TestCSVIntermediateDataFormat {
public void testByteArrayFullRangeOfCharacters() {
Schema schema = new Schema("test");
schema.addColumn(new Binary("1"));
- data.setFromSchema(schema);
- data.setToSchema(emptySchema);
+ data.setSchema(schema);
byte[] allCharByteArr = new byte[256];
- for(int i = 0; i < allCharByteArr.length; ++i) {
- allCharByteArr[i] = (byte)i;
+ for (int i = 0; i < allCharByteArr.length; ++i) {
+ allCharByteArr[i] = (byte) i;
}
Object[] in = {allCharByteArr};
Object[] inCopy = new Object[1];
- System.arraycopy(in,0,inCopy,0,in.length);
+ System.arraycopy(in, 0, inCopy, 0, in.length);
// Modifies the input array, so we use the copy to confirm
data.setObjectData(in);
assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
}
- /**
- * Note that we don't have an EmptyTo matching test
- * Because most tests above have empty "to" schema
- */
- @Test
- public void testMatchingEmptyFrom() {
-
- data.setFromSchema(emptySchema);
-
- Schema toSchema = new Schema("To");
- toSchema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"));
- data.setToSchema(toSchema);
-
- Object[] in = new Object[2];
- in[0] = new Long(10);
- in[1] = new Long(34);
-
- Object[] out = new Object[2];
- out[0] = new Long(10);
- out[1] = new Long(34);
-
- data.setObjectData(in);
-
- assertTrue(Arrays.deepEquals(out, data.getObjectData()));
- }
-
@Test(expected=SqoopException.class)
- public void testMatchingTwoEmptySchema() {
- data.setFromSchema(emptySchema);
- data.setToSchema(emptySchema);
-
- Object[] in = new Object[2];
- in[0] = new Long(10);
- in[1] = new Long(34);
-
- data.setObjectData(in);
-
- data.getObjectData();
- }
-
- @Test
- public void testMatchingFewerFromColumns(){
- Schema fromSchema = new Schema("From");
- fromSchema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"));
- data.setFromSchema(fromSchema);
-
- Schema toSchema = new Schema("To");
- toSchema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"))
- .addColumn(new Text("3"));
- data.setToSchema(toSchema);
-
- Object[] in = new Object[2];
- in[0] = new Long(10);
- in[1] = new Long(34);
-
- Object[] out = new Object[3];
- out[0] = new Long(10);
- out[1] = new Long(34);
- out[2] = null;
-
- data.setObjectData(in);
-
- assertTrue(Arrays.deepEquals(out, data.getObjectData()));
- }
-
- @Test
- public void testMatchingFewerToColumns(){
- Schema fromSchema = new Schema("From");
- fromSchema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"))
- .addColumn(new FixedPoint("3"));
- data.setFromSchema(fromSchema);
-
- Schema toSchema = new Schema("To");
- toSchema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"));
- data.setToSchema(toSchema);
-
- Object[] in = new Object[3];
- in[0] = new Long(10);
- in[1] = new Long(34);
- in[2] = new Long(50);
-
- Object[] out = new Object[2];
- out[0] = new Long(10);
- out[1] = new Long(34);
-
-
- data.setObjectData(in);
-
- assertTrue(Arrays.deepEquals(out, data.getObjectData()));
- }
-
-
- @Test
- public void testWithSomeNonMatchingFields(){
-
- Schema fromSchema = new Schema("From");
- fromSchema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"))
- .addColumn(new FixedPoint("3"));
- data.setFromSchema(fromSchema);
-
- Schema toSchema = new Schema("From");
- toSchema.addColumn(new FixedPoint("2"))
- .addColumn(new FixedPoint("3"))
- .addColumn(new FixedPoint("4"));
- data.setToSchema(toSchema);
-
- Object[] in = new Object[3];
- in[0] = new Long(10);
- in[1] = new Long(34);
- in[2] = new Long(50);
-
- Object[] out = new Object[3];
- out[0] = new Long(34);
- out[1] = new Long(50);
- out[2] = null;
-
- data.setObjectData(in);
+ public void testEmptySchema() {
+ String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
+ + ",'\\n'";
+ Schema schema = new Schema("Test");
+ data.setSchema(schema);
+ data.setTextData(testData);
- assertTrue(Arrays.deepEquals(out, data.getObjectData()));
+ Object[] out = data.getObjectData();
}
-
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
index 8c88d52..03d84d4 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
@@ -29,13 +29,14 @@ import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.idf.IntermediateDataFormat;
+import org.apache.sqoop.connector.matcher.Matcher;
+import org.apache.sqoop.connector.matcher.MatcherFactory;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.MapreduceExecutionError;
import org.apache.sqoop.common.PrefixContext;
import org.apache.sqoop.job.etl.Extractor;
import org.apache.sqoop.job.etl.ExtractorContext;
import org.apache.sqoop.etl.io.DataWriter;
-import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.job.io.SqoopWritable;
import org.apache.sqoop.submission.counter.SqoopCounters;
import org.apache.sqoop.utils.ClassUtils;
@@ -54,8 +55,9 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
* Service for reporting progress to mapreduce.
*/
private final ScheduledExecutorService progressService = Executors.newSingleThreadScheduledExecutor();
- private IntermediateDataFormat<String> dataFormat = null;
- private SqoopWritable dataOut = null;
+ private IntermediateDataFormat<String> fromDataFormat = null;
+ private IntermediateDataFormat<String> toDataFormat = null;
+ private Matcher matcher;
@Override
public void run(Context context) throws IOException, InterruptedException {
@@ -64,19 +66,17 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
String extractorName = conf.get(JobConstants.JOB_ETL_EXTRACTOR);
Extractor extractor = (Extractor) ClassUtils.instantiate(extractorName);
-
-
- Schema fromSchema = ConfigurationUtils.getConnectorSchema(Direction.FROM, conf);
- Schema toSchema = ConfigurationUtils.getConnectorSchema(Direction.TO, conf);
+ matcher = MatcherFactory.getMatcher(
+ ConfigurationUtils.getConnectorSchema(Direction.FROM, conf),
+ ConfigurationUtils.getConnectorSchema(Direction.TO, conf));
String intermediateDataFormatName = conf.get(JobConstants.INTERMEDIATE_DATA_FORMAT);
- dataFormat = (IntermediateDataFormat<String>) ClassUtils
+ fromDataFormat = (IntermediateDataFormat<String>) ClassUtils
.instantiate(intermediateDataFormatName);
-
- dataFormat.setFromSchema(fromSchema);
- dataFormat.setToSchema(toSchema);
-
- dataOut = new SqoopWritable();
+ fromDataFormat.setSchema(matcher.getFromSchema());
+ toDataFormat = (IntermediateDataFormat<String>) ClassUtils
+ .instantiate(intermediateDataFormatName);
+ toDataFormat.setSchema(matcher.getToSchema());
// Objects that should be passed to the Executor execution
PrefixContext subContext = new PrefixContext(conf, JobConstants.PREFIX_CONNECTOR_FROM_CONTEXT);
@@ -109,36 +109,41 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
private class SqoopMapDataWriter extends DataWriter {
private Context context;
+ private SqoopWritable writable;
public SqoopMapDataWriter(Context context) {
this.context = context;
+ this.writable = new SqoopWritable();
}
@Override
public void writeArrayRecord(Object[] array) {
- dataFormat.setObjectData(array);
+ fromDataFormat.setObjectData(array);
writeContent();
}
@Override
public void writeStringRecord(String text) {
- dataFormat.setTextData(text);
+ fromDataFormat.setTextData(text);
writeContent();
}
@Override
public void writeRecord(Object obj) {
- dataFormat.setData(obj.toString());
+ fromDataFormat.setData(obj.toString());
writeContent();
}
private void writeContent() {
try {
if (LOG.isDebugEnabled()) {
- LOG.debug("Extracted data: " + dataFormat.getTextData());
+ LOG.debug("Extracted data: " + fromDataFormat.getTextData());
}
- dataOut.setString(dataFormat.getTextData());
- context.write(dataOut, NullWritable.get());
+
+ toDataFormat.setObjectData( matcher.getMatchingData( fromDataFormat.getObjectData() ) );
+
+ writable.setString(toDataFormat.getTextData());
+ context.write(writable, NullWritable.get());
} catch (Exception e) {
throw new SqoopException(MapreduceExecutionError.MAPRED_EXEC_0013, e);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
index 941b31d..1ebd3e4 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java
@@ -34,6 +34,8 @@ import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.idf.CSVIntermediateDataFormat;
import org.apache.sqoop.connector.idf.IntermediateDataFormat;
+import org.apache.sqoop.connector.matcher.Matcher;
+import org.apache.sqoop.connector.matcher.MatcherFactory;
import org.apache.sqoop.job.JobConstants;
import org.apache.sqoop.job.MapreduceExecutionError;
import org.apache.sqoop.common.PrefixContext;
@@ -52,6 +54,7 @@ public class SqoopOutputFormatLoadExecutor {
private volatile boolean readerFinished = false;
private volatile boolean writerFinished = false;
private volatile IntermediateDataFormat<String> dataFormat;
+ private Matcher matcher;
private JobContext context;
private SqoopRecordWriter writer;
private Future<?> consumerFuture;
@@ -65,19 +68,18 @@ public class SqoopOutputFormatLoadExecutor {
this.loaderName = loaderName;
dataFormat = new CSVIntermediateDataFormat();
writer = new SqoopRecordWriter();
+ matcher = null;
}
public SqoopOutputFormatLoadExecutor(JobContext jobctx) {
context = jobctx;
writer = new SqoopRecordWriter();
+ matcher = MatcherFactory.getMatcher(
+ ConfigurationUtils.getConnectorSchema(Direction.FROM, context.getConfiguration()),
+ ConfigurationUtils.getConnectorSchema(Direction.TO, context.getConfiguration()));
dataFormat = (IntermediateDataFormat<String>) ClassUtils.instantiate(context
- .getConfiguration().get(JobConstants.INTERMEDIATE_DATA_FORMAT));
-
- Schema fromSchema = ConfigurationUtils.getConnectorSchema(Direction.FROM, context.getConfiguration());
- dataFormat.setFromSchema(fromSchema);
-
- Schema toSchema = ConfigurationUtils.getConnectorSchema(Direction.TO, context.getConfiguration());
- dataFormat.setToSchema(toSchema);
+ .getConfiguration().get(JobConstants.INTERMEDIATE_DATA_FORMAT));
+ dataFormat.setSchema(matcher.getToSchema());
}
public RecordWriter<SqoopWritable, NullWritable> getRecordWriter() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
index b5435ff..1952cbb 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/JobUtils.java
@@ -36,14 +36,7 @@ import org.apache.sqoop.job.mr.SqoopSplit;
public class JobUtils {
- public static void runJob(Configuration conf)
- throws IOException, InterruptedException, ClassNotFoundException {
- runJob(conf, SqoopInputFormat.class, SqoopMapper.class,
- (conf.get(JobConstants.HADOOP_OUTDIR) != null) ?
- SqoopFileOutputFormat.class : SqoopNullOutputFormat.class);
- }
-
- public static void runJob(Configuration conf,
+ public static boolean runJob(Configuration conf,
Class<? extends InputFormat<SqoopSplit, NullWritable>> input,
Class<? extends Mapper<SqoopSplit, NullWritable, SqoopWritable, NullWritable>> mapper,
Class<? extends OutputFormat<SqoopWritable, NullWritable>> output)
@@ -57,8 +50,7 @@ public class JobUtils {
job.setOutputKeyClass(SqoopWritable.class);
job.setOutputValueClass(NullWritable.class);
- boolean success = job.waitForCompletion(true);
- Assert.assertEquals("Job failed!", true, success);
+ return job.waitForCompletion(true);
}
private JobUtils() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
index 5662120..032cc11 100644
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMapReduce.java
@@ -54,6 +54,7 @@ import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.FixedPoint;
import org.apache.sqoop.schema.type.FloatingPoint;
import org.apache.sqoop.schema.type.Text;
+import org.junit.Assert;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -96,8 +97,10 @@ public class TestMapReduce {
Job job = new Job(conf);
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, schema);
- JobUtils.runJob(job.getConfiguration(), SqoopInputFormat.class, SqoopMapper.class,
- DummyOutputFormat.class);
+ ConfigurationUtils.setConnectorSchema(Direction.TO, job, schema);
+ boolean success = JobUtils.runJob(job.getConfiguration(),
+ SqoopInputFormat.class, SqoopMapper.class, DummyOutputFormat.class);
+ Assert.assertEquals("Job failed!", true, success);
}
@Test
@@ -116,8 +119,11 @@ public class TestMapReduce {
Job job = new Job(conf);
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, schema);
- JobUtils.runJob(job.getConfiguration(), SqoopInputFormat.class, SqoopMapper.class,
+ ConfigurationUtils.setConnectorSchema(Direction.TO, job, schema);
+ boolean success = JobUtils.runJob(job.getConfiguration(),
+ SqoopInputFormat.class, SqoopMapper.class,
SqoopNullOutputFormat.class);
+ Assert.assertEquals("Job failed!", true, success);
// Make sure both destroyers get called.
assertEquals(1, DummyFromDestroyer.count);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c1e53e5d/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java
new file mode 100644
index 0000000..7f9a147
--- /dev/null
+++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java
@@ -0,0 +1,275 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.job;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.connector.idf.CSVIntermediateDataFormat;
+import org.apache.sqoop.job.etl.Extractor;
+import org.apache.sqoop.job.etl.ExtractorContext;
+import org.apache.sqoop.job.etl.Partition;
+import org.apache.sqoop.job.etl.Partitioner;
+import org.apache.sqoop.job.etl.PartitionerContext;
+import org.apache.sqoop.job.io.Data;
+import org.apache.sqoop.job.io.SqoopWritable;
+import org.apache.sqoop.job.mr.ConfigurationUtils;
+import org.apache.sqoop.job.mr.SqoopInputFormat;
+import org.apache.sqoop.job.mr.SqoopMapper;
+import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.type.FixedPoint;
+import org.apache.sqoop.schema.type.FloatingPoint;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.junit.Assert.assertEquals;
+
+
+@RunWith(Parameterized.class)
+public class TestMatching {
+ private static final int START_PARTITION = 1;
+ private static final int NUMBER_OF_PARTITIONS = 1;
+ private static final int NUMBER_OF_ROWS_PER_PARTITION = 1;
+
+ private Schema from;
+ private Schema to;
+
+ public TestMatching(Schema from,
+ Schema to)
+ throws Exception {
+ this.from = from;
+ this.to = to;
+
+ System.out.println("Testing with Schemas\n\tFROM: " + this.from + "\n\tTO: " + this.to);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ List<Object[]> parameters = new ArrayList<Object[]>();
+
+ Schema emptyFrom = new Schema("FROM-EMPTY");
+ Schema emptyTo = new Schema("TO-EMPTY");
+ Schema from1 = new Schema("FROM-1");
+ Schema to1 = new Schema("TO-1");
+ Schema from2 = new Schema("FROM-2");
+ Schema to2 = new Schema("TO-2");
+
+ from1.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"))
+ .addColumn(new org.apache.sqoop.schema.type.Text("3"));
+ to1.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"))
+ .addColumn(new org.apache.sqoop.schema.type.Text("3"));
+ from2.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"));
+ to2.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"));
+
+ parameters.add(new Object[]{
+ emptyFrom,
+ emptyTo
+ });
+ parameters.add(new Object[]{
+ from1,
+ emptyTo
+ });
+ parameters.add(new Object[]{
+ emptyTo,
+ to1
+ });
+ parameters.add(new Object[]{
+ from1,
+ to1
+ });
+ parameters.add(new Object[]{
+ from2,
+ to1
+ });
+ parameters.add(new Object[]{
+ from1,
+ to2
+ });
+
+ return parameters;
+ }
+
+ @Test
+ public void testSchemaMatching() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
+ conf.set(JobConstants.JOB_ETL_EXTRACTOR, DummyExtractor.class.getName());
+ conf.set(JobConstants.INTERMEDIATE_DATA_FORMAT,
+ CSVIntermediateDataFormat.class.getName());
+
+ Job job = new Job(conf);
+ ConfigurationUtils.setConnectorSchema(Direction.FROM, job, from);
+ ConfigurationUtils.setConnectorSchema(Direction.TO, job, to);
+ JobUtils.runJob(job.getConfiguration(), SqoopInputFormat.class, SqoopMapper.class,
+ DummyOutputFormat.class);
+ boolean success = JobUtils.runJob(job.getConfiguration(),
+ SqoopInputFormat.class, SqoopMapper.class,
+ DummyOutputFormat.class);
+ if (from.getName().split("-")[1].equals("EMPTY")) {
+ if (to.getName().split("-")[1].equals("EMPTY")) {
+ Assert.assertEquals("Job succeeded!", false, success);
+ } else {
+ Assert.assertEquals("Job failed!", true, success);
+ }
+ } else {
+ if (to.getName().split("-")[1].equals("EMPTY")) {
+ Assert.assertEquals("Job failed!", true, success);
+ } else if (from.getName().split("-")[1].equals(to.getName().split("-")[1])) {
+ Assert.assertEquals("Job failed!", true, success);
+ } else {
+ Assert.assertEquals("Job succeeded!", false, success);
+ }
+ }
+ }
+
+ public static class DummyPartition extends Partition {
+ private int id;
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ id = in.readInt();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(id);
+ }
+
+ @Override
+ public String toString() {
+ return Integer.toString(id);
+ }
+ }
+
+ public static class DummyPartitioner extends Partitioner {
+ @Override
+ public List<Partition> getPartitions(PartitionerContext context, Object oc, Object oj) {
+ List<Partition> partitions = new LinkedList<Partition>();
+ for (int id = START_PARTITION; id <= NUMBER_OF_PARTITIONS; id++) {
+ DummyPartition partition = new DummyPartition();
+ partition.setId(id);
+ partitions.add(partition);
+ }
+ return partitions;
+ }
+ }
+
+ public static class DummyExtractor extends Extractor {
+ @Override
+ public void extract(ExtractorContext context, Object oc, Object oj, Object partition) {
+ int id = ((DummyPartition)partition).getId();
+ for (int row = 0; row < NUMBER_OF_ROWS_PER_PARTITION; row++) {
+ context.getDataWriter().writeArrayRecord(new Object[] {
+ id * NUMBER_OF_ROWS_PER_PARTITION + row,
+ (double) (id * NUMBER_OF_ROWS_PER_PARTITION + row),
+ String.valueOf(id*NUMBER_OF_ROWS_PER_PARTITION+row)});
+ }
+ }
+
+ @Override
+ public long getRowsRead() {
+ return NUMBER_OF_ROWS_PER_PARTITION;
+ }
+ }
+
+ public static class DummyOutputFormat
+ extends OutputFormat<SqoopWritable, NullWritable> {
+ @Override
+ public void checkOutputSpecs(JobContext context) {
+ // do nothing
+ }
+
+ @Override
+ public RecordWriter<SqoopWritable, NullWritable> getRecordWriter(
+ TaskAttemptContext context) {
+ return new DummyRecordWriter();
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context) {
+ return new DummyOutputCommitter();
+ }
+
+ public static class DummyRecordWriter
+ extends RecordWriter<SqoopWritable, NullWritable> {
+ private int index = START_PARTITION*NUMBER_OF_ROWS_PER_PARTITION;
+ private Data data = new Data();
+
+ @Override
+ public void write(SqoopWritable key, NullWritable value) {
+
+ data.setContent(new Object[] {
+ index,
+ (double) index,
+ String.valueOf(index)},
+ Data.ARRAY_RECORD);
+ index++;
+
+ assertEquals(data.toString(), key.toString());
+ }
+
+ @Override
+ public void close(TaskAttemptContext context) {
+ // do nothing
+ }
+ }
+
+ public static class DummyOutputCommitter extends OutputCommitter {
+ @Override
+ public void setupJob(JobContext jobContext) { }
+
+ @Override
+ public void setupTask(TaskAttemptContext taskContext) { }
+
+ @Override
+ public void commitTask(TaskAttemptContext taskContext) { }
+
+ @Override
+ public void abortTask(TaskAttemptContext taskContext) { }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext taskContext) {
+ return false;
+ }
+ }
+ }
+}
[04/52] [abbrv] git commit: SQOOP-1487: Sqoop2: From/To:
Refactor/Create HDFS connector test cases
Posted by ab...@apache.org.
SQOOP-1487: Sqoop2: From/To: Refactor/Create HDFS connector test cases
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c0b22b1d
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c0b22b1d
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c0b22b1d
Branch: refs/heads/SQOOP-1367
Commit: c0b22b1d62221f2a24294520b8ff3851c06d06c3
Parents: 8fee134
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Sep 3 09:40:11 2014 +0200
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:17 2014 -0700
----------------------------------------------------------------------
connector/connector-hdfs/pom.xml | 5 +
.../sqoop/connector/hdfs/HdfsExtractor.java | 2 -
.../apache/sqoop/connector/hdfs/FileUtils.java | 82 ++++++
.../sqoop/connector/hdfs/TestExtractor.java | 125 ++++++++++
.../sqoop/connector/hdfs/TestHdfsBase.java | 139 +++++++++++
.../apache/sqoop/connector/hdfs/TestLoader.java | 213 ++++++++++++++++
.../sqoop/connector/hdfs/TestPartitioner.java | 113 +++++++++
.../src/test/resources/log4j.properties | 24 ++
.../java/org/apache/sqoop/job/FileUtils.java | 69 -----
.../org/apache/sqoop/job/TestHdfsExtract.java | 241 ------------------
.../java/org/apache/sqoop/job/TestHdfsLoad.java | 250 -------------------
11 files changed, 701 insertions(+), 562 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/connector/connector-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/pom.xml b/connector/connector-hdfs/pom.xml
index 8df9f11..fa4330a 100644
--- a/connector/connector-hdfs/pom.xml
+++ b/connector/connector-hdfs/pom.xml
@@ -35,6 +35,11 @@ limitations under the License.
<dependencies>
<dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.sqoop</groupId>
<artifactId>sqoop-spi</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
index fc12381..7447071 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
@@ -42,8 +42,6 @@ import java.io.IOException;
* Extract from HDFS.
* Default field delimiter of a record is comma.
*/
-
-
public class HdfsExtractor extends Extractor<ConnectionConfiguration, FromJobConfiguration, HdfsPartition> {
public static final Logger LOG = Logger.getLogger(HdfsExtractor.class);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/FileUtils.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/FileUtils.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/FileUtils.java
new file mode 100644
index 0000000..8c19d01
--- /dev/null
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/FileUtils.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.LinkedList;
+import java.util.List;
+
+public class FileUtils {
+
+ public static boolean exists(String file) throws IOException {
+ Path path = new Path(file);
+ FileSystem fs = path.getFileSystem(new Configuration());
+ return fs.exists(path);
+ }
+
+ public static void delete(String file) throws IOException {
+ Path path = new Path(file);
+ FileSystem fs = path.getFileSystem(new Configuration());
+ if (fs.exists(path)) {
+ fs.delete(path, true);
+ }
+ }
+
+ public static void mkdirs(String directory) throws IOException {
+ Path path = new Path(directory);
+ FileSystem fs = path.getFileSystem(new Configuration());
+ if (!fs.exists(path)) {
+ fs.mkdirs(path);
+ }
+ }
+
+ public static InputStream open(String fileName)
+ throws IOException, ClassNotFoundException {
+ Path filepath = new Path(fileName);
+ FileSystem fs = filepath.getFileSystem(new Configuration());
+ return fs.open(filepath);
+ }
+
+ public static OutputStream create(String fileName) throws IOException {
+ Path filepath = new Path(fileName);
+ FileSystem fs = filepath.getFileSystem(new Configuration());
+ return fs.create(filepath, false);
+ }
+
+ public static Path[] listDir(String directory) throws IOException {
+ Path dirpath = new Path(directory);
+ FileSystem fs = dirpath.getFileSystem(new Configuration());
+ List<Path> paths = new LinkedList<Path>();
+ for (FileStatus fileStatus : fs.listStatus(dirpath)) {
+ paths.add(fileStatus.getPath());
+ }
+ return paths.toArray(new Path[paths.size()]);
+ }
+
+ private FileUtils() {
+ // Disable explicit object creation
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
new file mode 100644
index 0000000..6ed4087
--- /dev/null
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.sqoop.common.PrefixContext;
+import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
+import org.apache.sqoop.etl.io.DataWriter;
+import org.apache.sqoop.job.etl.Extractor;
+import org.apache.sqoop.job.etl.ExtractorContext;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.SEQUENCE_FILE;
+import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.TEXT_FILE;
+
+@RunWith(Parameterized.class)
+public class TestExtractor extends TestHdfsBase {
+ private static final String INPUT_ROOT = System.getProperty("maven.build.directory", "/tmp") + "/sqoop/warehouse/";
+ private static final int NUMBER_OF_FILES = 5;
+ private static final int NUMBER_OF_ROWS_PER_FILE = 1000;
+
+ private OutputFormat outputFileType;
+ private Class<? extends CompressionCodec> compressionClass;
+ private final String inputDirectory;
+ private Extractor extractor;
+
+ public TestExtractor(OutputFormat outputFileType,
+ Class<? extends CompressionCodec> compressionClass)
+ throws Exception {
+ this.inputDirectory = INPUT_ROOT + getClass().getSimpleName();
+ this.outputFileType = outputFileType;
+ this.compressionClass = compressionClass;
+ this.extractor = new HdfsExtractor();
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ List<Object[]> parameters = new ArrayList<Object[]>();
+ for (Class<?> compressionClass : new Class<?>[]{null, DefaultCodec.class, BZip2Codec.class}) {
+ for (Object outputFileType : new Object[]{TEXT_FILE, SEQUENCE_FILE}) {
+ parameters.add(new Object[]{outputFileType, compressionClass});
+ }
+ }
+ return parameters;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ FileUtils.mkdirs(inputDirectory);
+ switch (this.outputFileType) {
+ case TEXT_FILE:
+ createTextInput(inputDirectory, this.compressionClass, NUMBER_OF_FILES, NUMBER_OF_ROWS_PER_FILE);
+ break;
+
+ case SEQUENCE_FILE:
+ createSequenceInput(inputDirectory, this.compressionClass, NUMBER_OF_FILES, NUMBER_OF_ROWS_PER_FILE);
+ break;
+ }
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ FileUtils.delete(inputDirectory);
+ }
+
+ @Test
+ public void testExtractor() throws Exception {
+ Configuration conf = new Configuration();
+ PrefixContext prefixContext = new PrefixContext(conf, "org.apache.sqoop.job.connector.from.context.");
+ ExtractorContext context = new ExtractorContext(prefixContext, new DataWriter() {
+ private long index = 1L;
+
+ @Override
+ public void writeArrayRecord(Object[] array) {
+ throw new AssertionError("Should not be writing array.");
+ }
+
+ @Override
+ public void writeStringRecord(String text) {
+ Assert.assertEquals(index + "," + index + ".0,'" + index++ + "'", text);
+ }
+
+ @Override
+ public void writeRecord(Object obj) {
+ throw new AssertionError("Should not be writing object.");
+ }
+ }, null);
+ ConnectionConfiguration connConf = new ConnectionConfiguration();
+ FromJobConfiguration jobConf = new FromJobConfiguration();
+
+ HdfsPartition partition = createPartition(FileUtils.listDir(inputDirectory));
+
+ extractor.extract(context, connConf, jobConf, partition);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java
new file mode 100644
index 0000000..0cc2b8b
--- /dev/null
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.CompressionCodec;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+public class TestHdfsBase {
+
+ protected HdfsPartition createPartition(Path[] paths) throws IOException {
+ long[] offsets = new long[paths.length];
+ long[] lengths = new long[paths.length];
+ String[] locations = new String[paths.length];
+ FileSystem fs = FileSystem.get(new Configuration());
+
+ for (int i = 0; i < offsets.length; ++i) {
+ locations[i] = paths[i].getName();
+ lengths[i] = fs.getFileStatus(paths[i]).getLen();
+ }
+
+ return new HdfsPartition(paths, offsets, lengths, locations);
+ }
+
+ protected void createTextInput(String indir,
+ Class<? extends CompressionCodec> clz,
+ int numberOfFiles,
+ int numberOfRows)
+ throws IOException, InstantiationException, IllegalAccessException {
+ Configuration conf = new Configuration();
+
+ CompressionCodec codec = null;
+ String extension = "";
+ if (clz != null) {
+ codec = clz.newInstance();
+ if (codec instanceof Configurable) {
+ ((Configurable) codec).setConf(conf);
+ }
+ extension = codec.getDefaultExtension();
+ }
+
+ int index = 1;
+ for (int fi = 0; fi < numberOfFiles; fi++) {
+ String fileName = indir + "/" + "part-r-" + padZeros(fi, 5) + extension;
+ OutputStream filestream = FileUtils.create(fileName);
+ BufferedWriter filewriter;
+ if (codec != null) {
+ filewriter = new BufferedWriter(new OutputStreamWriter(
+ codec.createOutputStream(filestream, codec.createCompressor()),
+ "UTF-8"));
+ } else {
+ filewriter = new BufferedWriter(new OutputStreamWriter(
+ filestream, "UTF-8"));
+ }
+
+ for (int ri = 0; ri < numberOfRows; ri++) {
+ String row = index + "," + (double)index + ",'" + index + "'";
+ filewriter.write(row + HdfsConstants.DEFAULT_RECORD_DELIMITER);
+ index++;
+ }
+
+ filewriter.close();
+ }
+ }
+
+ protected void createSequenceInput(String indir,
+ Class<? extends CompressionCodec> clz,
+ int numberOfFiles,
+ int numberOfRows)
+ throws IOException, InstantiationException, IllegalAccessException {
+ Configuration conf = new Configuration();
+
+ CompressionCodec codec = null;
+ if (clz != null) {
+ codec = clz.newInstance();
+ if (codec instanceof Configurable) {
+ ((Configurable) codec).setConf(conf);
+ }
+ }
+
+ int index = 1;
+ for (int fi = 0; fi < numberOfFiles; fi++) {
+ Path filepath = new Path(indir,
+ "part-r-" + padZeros(fi, 5) + ".seq");
+ SequenceFile.Writer filewriter;
+ if (codec != null) {
+ filewriter = SequenceFile.createWriter(filepath.getFileSystem(conf),
+ conf, filepath, Text.class, NullWritable.class,
+ SequenceFile.CompressionType.BLOCK, codec);
+ } else {
+ filewriter = SequenceFile.createWriter(filepath.getFileSystem(conf),
+ conf, filepath, Text.class, NullWritable.class, SequenceFile.CompressionType.NONE);
+ }
+
+ Text text = new Text();
+ for (int ri = 0; ri < numberOfRows; ri++) {
+ String row = index + "," + (double)index + ",'" + index + "'";
+ text.set(row);
+ filewriter.append(text, NullWritable.get());
+ index++;
+ }
+
+ filewriter.close();
+ }
+ }
+
+ private String padZeros(int number, int digits) {
+ String string = String.valueOf(number);
+ for (int i = (digits - string.length()); i > 0; i--) {
+ string = "0" + string;
+ }
+ return string;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
new file mode 100644
index 0000000..79cf1f1
--- /dev/null
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLoader.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.sqoop.common.PrefixContext;
+import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.OutputCompression;
+import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
+import org.apache.sqoop.connector.hdfs.configuration.ToJobConfiguration;
+import org.apache.sqoop.etl.io.DataReader;
+import org.apache.sqoop.job.etl.Loader;
+import org.apache.sqoop.job.etl.LoaderContext;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.SEQUENCE_FILE;
+import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.TEXT_FILE;
+
+@RunWith(Parameterized.class)
+public class TestLoader extends TestHdfsBase {
+ private static final String INPUT_ROOT = System.getProperty("maven.build.directory", "/tmp") + "/sqoop/warehouse/";
+ private static final int NUMBER_OF_ROWS_PER_FILE = 1000;
+
+ private OutputFormat outputFormat;
+ private OutputCompression compression;
+ private final String outputDirectory;
+ private Loader loader;
+
+ public TestLoader(OutputFormat outputFormat,
+ OutputCompression compression)
+ throws Exception {
+ this.outputDirectory = INPUT_ROOT + getClass().getSimpleName();
+ this.outputFormat = outputFormat;
+ this.compression = compression;
+ this.loader = new HdfsLoader();
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ List<Object[]> parameters = new ArrayList<Object[]>();
+ for (OutputCompression compression : new OutputCompression[]{
+ OutputCompression.DEFAULT,
+ OutputCompression.BZIP2,
+ OutputCompression.NONE
+ }) {
+ for (Object outputFileType : new Object[]{TEXT_FILE, SEQUENCE_FILE}) {
+ parameters.add(new Object[]{outputFileType, compression});
+ }
+ }
+ return parameters;
+ }
+
+ @Before
+ public void setUp() throws Exception {}
+
+ @After
+ public void tearDown() throws IOException {
+ FileUtils.delete(outputDirectory);
+ }
+
+ @Test
+ public void testLoader() throws Exception {
+ FileSystem fs = FileSystem.get(new Configuration());
+
+ Configuration conf = new Configuration();
+ PrefixContext prefixContext = new PrefixContext(conf, "org.apache.sqoop.job.connector.from.context.");
+ LoaderContext context = new LoaderContext(prefixContext, new DataReader() {
+ private long index = 0L;
+
+ @Override
+ public Object[] readArrayRecord() {
+ return null;
+ }
+
+ @Override
+ public String readTextRecord() {
+ if (index++ < NUMBER_OF_ROWS_PER_FILE) {
+ return index + "," + (double)index + ",'" + index + "'";
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Object readContent() {
+ return null;
+ }
+ }, null);
+ ConnectionConfiguration connConf = new ConnectionConfiguration();
+ ToJobConfiguration jobConf = new ToJobConfiguration();
+ jobConf.output.outputDirectory = outputDirectory;
+ jobConf.output.compression = compression;
+ jobConf.output.outputFormat = outputFormat;
+ Path outputPath = new Path(outputDirectory);
+
+ loader.load(context, connConf, jobConf);
+ Assert.assertEquals(1, fs.listStatus(outputPath).length);
+
+ for (FileStatus status : fs.listStatus(outputPath)) {
+ verifyOutput(fs, status.getPath());
+ }
+
+ loader.load(context, connConf, jobConf);
+ Assert.assertEquals(2, fs.listStatus(outputPath).length);
+ loader.load(context, connConf, jobConf);
+ loader.load(context, connConf, jobConf);
+ loader.load(context, connConf, jobConf);
+ Assert.assertEquals(5, fs.listStatus(outputPath).length);
+ }
+
+ private void verifyOutput(FileSystem fs, Path file) throws IOException {
+ Configuration conf = new Configuration();
+ FSDataInputStream fsin = fs.open(file);
+ CompressionCodec codec;
+
+ switch(outputFormat) {
+ case TEXT_FILE:
+ codec = (new CompressionCodecFactory(conf)).getCodec(file);
+
+ // Verify compression
+ switch(compression) {
+ case BZIP2:
+ Assert.assertTrue(codec.getClass().getCanonicalName().indexOf("BZip2") != -1);
+ break;
+
+ case DEFAULT:
+ Assert.assertTrue(codec.getClass().getCanonicalName().indexOf("Deflate") != -1);
+ break;
+
+ case NONE:
+ default:
+ Assert.assertNull(codec);
+ break;
+ }
+
+ InputStreamReader in;
+ if (codec == null) {
+ in = new InputStreamReader(fsin);
+ } else {
+ in = new InputStreamReader(codec.createInputStream(fsin, codec.createDecompressor()));
+ }
+ BufferedReader textReader = new BufferedReader(in);
+
+ for (int i = 1; i <= NUMBER_OF_ROWS_PER_FILE; ++i) {
+ Assert.assertEquals(i + "," + (double)i + ",'" + i + "'", textReader.readLine());
+ }
+ break;
+
+ case SEQUENCE_FILE:
+ SequenceFile.Reader sequenceReader = new SequenceFile.Reader(fs, file, conf);
+ codec = sequenceReader.getCompressionCodec();
+
+ // Verify compression
+ switch(compression) {
+ case BZIP2:
+ Assert.assertTrue(codec.getClass().getCanonicalName().indexOf("BZip2") != -1);
+ break;
+
+ case DEFAULT:
+ Assert.assertTrue(codec.getClass().getCanonicalName().indexOf("Default") != -1);
+ break;
+
+ case NONE:
+ default:
+ Assert.assertNull(codec);
+ break;
+ }
+
+ Text line = new Text();
+ int index = 1;
+ while (sequenceReader.next(line)) {
+ Assert.assertEquals(index + "," + (double)index + ",'" + index++ + "'", line.toString());
+ line = new Text();
+ }
+ break;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java
new file mode 100644
index 0000000..ae93b0a
--- /dev/null
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestPartitioner.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.sqoop.common.PrefixContext;
+import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.OutputFormat;
+import org.apache.sqoop.job.etl.Partition;
+import org.apache.sqoop.job.etl.Partitioner;
+import org.apache.sqoop.job.etl.PartitionerContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.sqoop.connector.hdfs.configuration.OutputFormat.*;
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Parameterized.class)
+public class TestPartitioner extends TestHdfsBase {
+ private static final String INPUT_ROOT = System.getProperty("maven.build.directory", "/tmp") + "/sqoop/warehouse/";
+ private static final int NUMBER_OF_FILES = 5;
+ private static final int NUMBER_OF_ROWS_PER_FILE = 1000;
+
+ private OutputFormat outputFileType;
+ private Class<? extends CompressionCodec> compressionClass;
+ private Partitioner partitioner;
+
+ private final String inputDirectory;
+
+ public TestPartitioner(OutputFormat outputFileType, Class<? extends CompressionCodec> compressionClass) {
+ this.inputDirectory = INPUT_ROOT + getClass().getSimpleName();
+ this.outputFileType = outputFileType;
+ this.compressionClass = compressionClass;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ partitioner = new HdfsPartitioner();
+ FileUtils.mkdirs(inputDirectory);
+
+ switch (this.outputFileType) {
+ case TEXT_FILE:
+ createTextInput(inputDirectory, this.compressionClass, NUMBER_OF_FILES, NUMBER_OF_ROWS_PER_FILE);
+ break;
+
+ case SEQUENCE_FILE:
+ createSequenceInput(inputDirectory, this.compressionClass, NUMBER_OF_FILES, NUMBER_OF_ROWS_PER_FILE);
+ break;
+ }
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ FileUtils.delete(inputDirectory);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ List<Object[]> parameters = new ArrayList<Object[]>();
+ for (Class<?> compressionClass : new Class<?>[]{null, DefaultCodec.class, BZip2Codec.class}) {
+ for (Object outputFileType : new Object[]{TEXT_FILE, SEQUENCE_FILE}) {
+ parameters.add(new Object[]{outputFileType, compressionClass});
+ }
+ }
+ return parameters;
+ }
+
+ @Test
+ public void testPartitioner() {
+ Configuration conf = new Configuration();
+ PrefixContext prefixContext = new PrefixContext(conf, "org.apache.sqoop.job.connector.from.context.");
+ PartitionerContext context = new PartitionerContext(prefixContext, 5, null);
+ ConnectionConfiguration connConf = new ConnectionConfiguration();
+ FromJobConfiguration jobConf = new FromJobConfiguration();
+
+ jobConf.input.inputDirectory = inputDirectory;
+
+ List<Partition> partitions = partitioner.getPartitions(context, connConf, jobConf);
+
+ if (this.compressionClass == null) {
+ assertEquals(5, partitions.size());
+ } else {
+ assertEquals(3, partitions.size());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/connector/connector-hdfs/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/resources/log4j.properties b/connector/connector-hdfs/src/test/resources/log4j.properties
new file mode 100644
index 0000000..44ffced
--- /dev/null
+++ b/connector/connector-hdfs/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=DEBUG, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/execution/mapreduce/src/test/java/org/apache/sqoop/job/FileUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/FileUtils.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/FileUtils.java
deleted file mode 100644
index e685883..0000000
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/FileUtils.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.job;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class FileUtils {
-
- public static boolean exists(String file) throws IOException {
- Path path = new Path(file);
- FileSystem fs = path.getFileSystem(new Configuration());
- return fs.exists(path);
- }
-
- public static void delete(String file) throws IOException {
- Path path = new Path(file);
- FileSystem fs = path.getFileSystem(new Configuration());
- if (fs.exists(path)) {
- fs.delete(path, true);
- }
- }
-
- public static void mkdirs(String directory) throws IOException {
- Path path = new Path(directory);
- FileSystem fs = path.getFileSystem(new Configuration());
- if (!fs.exists(path)) {
- fs.mkdirs(path);
- }
- }
-
- public static InputStream open(String fileName)
- throws IOException, ClassNotFoundException {
- Path filepath = new Path(fileName);
- FileSystem fs = filepath.getFileSystem(new Configuration());
- return fs.open(filepath);
- }
-
- public static OutputStream create(String fileName) throws IOException {
- Path filepath = new Path(fileName);
- FileSystem fs = filepath.getFileSystem(new Configuration());
- return fs.create(filepath, false);
- }
-
- private FileUtils() {
- // Disable explicit object creation
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java
deleted file mode 100644
index 2accf77..0000000
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsExtract.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.job;
-
-import junit.framework.TestCase;
-
-//import org.apache.sqoop.connector.idf.CSVIntermediateDataFormat;
-//import org.apache.sqoop.job.etl.HdfsExportExtractor;
-//import org.apache.sqoop.job.etl.HdfsExportPartitioner;
-
-public class TestHdfsExtract extends TestCase {
-
-// private static final String INPUT_ROOT = System.getProperty("maven.build.directory", "/tmp") + "/sqoop/warehouse/";
-// private static final int NUMBER_OF_FILES = 5;
-// private static final int NUMBER_OF_ROWS_PER_FILE = 1000;
-//
-// private final String indir;
-//
-// public TestHdfsExtract() {
-// indir = INPUT_ROOT + getClass().getSimpleName();
-// }
-//
-// @Override
-// public void setUp() throws IOException {
-// FileUtils.mkdirs(indir);
-// }
-//
-// @Override
-// public void tearDown() throws IOException {
-// FileUtils.delete(indir);
-// }
-//
-// /**
-// * Test case for validating the number of partitions creation
-// * based on input.
-// * Success if the partitions list size is less or equal to
-// * given max partition.
-// * @throws Exception
-// */
-// @Test
-// public void testHdfsExportPartitioner() throws Exception {
-// createTextInput(null);
-// Configuration conf = new Configuration();
-// conf.set(JobConstants.HADOOP_INPUTDIR, indir);
-//
-// conf.set(JobConstants.INTERMEDIATE_DATA_FORMAT,
-// CSVIntermediateDataFormat.class.getName());
-// HdfsExportPartitioner partitioner = new HdfsExportPartitioner();
-// PrefixContext prefixContext = new PrefixContext(conf, "");
-// int[] partitionValues = {2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 17};
-//
-// for(int maxPartitions : partitionValues) {
-// PartitionerContext partCont = new PartitionerContext(prefixContext, maxPartitions, null);
-// List<Partition> partitionList = partitioner.getPartitions(partCont, null, null);
-// assertTrue(partitionList.size()<=maxPartitions);
-// }
-// }
-//
-// @Test
-// public void testUncompressedText() throws Exception {
-// createTextInput(null);
-//
-// JobUtils.runJob(createJob(createConf(), createSchema()).getConfiguration());
-// }
-//
-// @Test
-// public void testDefaultCompressedText() throws Exception {
-// createTextInput(SqoopFileOutputFormat.DEFAULT_CODEC);
-//
-// JobUtils.runJob(createJob(createConf(), createSchema()).getConfiguration());
-// }
-//
-// @Test
-// public void testBZip2CompressedText() throws Exception {
-// createTextInput(BZip2Codec.class);
-//
-// JobUtils.runJob(createJob(createConf(), createSchema()).getConfiguration());
-// }
-//
-// @Test
-// public void testDefaultCompressedSequence() throws Exception {
-// createSequenceInput(SqoopFileOutputFormat.DEFAULT_CODEC);
-//
-// JobUtils.runJob(createJob(createConf(), createSchema()).getConfiguration());
-// }
-//
-// @Test
-// public void testUncompressedSequence() throws Exception {
-// createSequenceInput(null);
-//
-// JobUtils.runJob(createJob(createConf(), createSchema()).getConfiguration());
-// }
-//
-// private Schema createSchema() {
-// Schema schema = new Schema("Test");
-// schema.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"))
-// .addColumn(new org.apache.sqoop.schema.type.Text("3"));
-// return schema;
-// }
-//
-// private Configuration createConf() {
-// Configuration conf = new Configuration();
-// ConfigurationUtils.setJobType(conf, MJob.Type.EXPORT);
-// conf.setIfUnset(JobConstants.JOB_ETL_PARTITIONER,
-// HdfsExportPartitioner.class.getName());
-// conf.setIfUnset(JobConstants.JOB_ETL_EXTRACTOR,
-// HdfsExportExtractor.class.getName());
-// conf.setIfUnset(JobConstants.JOB_ETL_LOADER, DummyLoader.class.getName());
-// conf.setIfUnset(Constants.JOB_ETL_NUMBER_PARTITIONS, "4");
-// conf.setIfUnset(JobConstants.INTERMEDIATE_DATA_FORMAT,
-// CSVIntermediateDataFormat.class.getName());
-// conf.setIfUnset(JobConstants.HADOOP_INPUTDIR, indir);
-// return conf;
-// }
-//
-// private Job createJob(Configuration conf, Schema schema) throws Exception {
-// Job job = new Job(conf);
-// ConfigurationUtils.setConnectorSchema(job, schema);
-// job.getConfiguration().set(JobConstants.INTERMEDIATE_DATA_FORMAT,
-// CSVIntermediateDataFormat.class.getName());
-// return job;
-// }
-//
-// private void createTextInput(Class<? extends CompressionCodec> clz)
-// throws IOException, InstantiationException, IllegalAccessException {
-// Configuration conf = new Configuration();
-//
-// CompressionCodec codec = null;
-// String extension = "";
-// if (clz != null) {
-// codec = clz.newInstance();
-// if (codec instanceof Configurable) {
-// ((Configurable) codec).setConf(conf);
-// }
-// extension = codec.getDefaultExtension();
-// }
-//
-// int index = 1;
-// for (int fi=0; fi<NUMBER_OF_FILES; fi++) {
-// String fileName = indir + "/" + "part-r-" + padZeros(fi, 5) + extension;
-// OutputStream filestream = FileUtils.create(fileName);
-// BufferedWriter filewriter;
-// if (codec != null) {
-// filewriter = new BufferedWriter(new OutputStreamWriter(
-// codec.createOutputStream(filestream, codec.createCompressor()),
-// Data.CHARSET_NAME));
-// } else {
-// filewriter = new BufferedWriter(new OutputStreamWriter(
-// filestream, Data.CHARSET_NAME));
-// }
-//
-// for (int ri=0; ri<NUMBER_OF_ROWS_PER_FILE; ri++) {
-// String row = index + "," + (double)index + ",'" + index + "'";
-// filewriter.write(row + Data.DEFAULT_RECORD_DELIMITER);
-// index++;
-// }
-//
-// filewriter.close();
-// }
-// }
-//
-// private void createSequenceInput(Class<? extends CompressionCodec> clz)
-// throws IOException, InstantiationException, IllegalAccessException {
-// Configuration conf = new Configuration();
-//
-// CompressionCodec codec = null;
-// if (clz != null) {
-// codec = clz.newInstance();
-// if (codec instanceof Configurable) {
-// ((Configurable) codec).setConf(conf);
-// }
-// }
-//
-// int index = 1;
-// for (int fi=0; fi<NUMBER_OF_FILES; fi++) {
-// Path filepath = new Path(indir,
-// "part-r-" + padZeros(fi, 5) + HdfsSequenceImportLoader.EXTENSION);
-// SequenceFile.Writer filewriter;
-// if (codec != null) {
-// filewriter = SequenceFile.createWriter(filepath.getFileSystem(conf),
-// conf, filepath, Text.class, NullWritable.class,
-// CompressionType.BLOCK, codec);
-// } else {
-// filewriter = SequenceFile.createWriter(filepath.getFileSystem(conf),
-// conf, filepath, Text.class, NullWritable.class, CompressionType.NONE);
-// }
-//
-// Text text = new Text();
-// for (int ri=0; ri<NUMBER_OF_ROWS_PER_FILE; ri++) {
-// String row = index + "," + (double)index + ",'" + index + "'";
-// text.set(row);
-// filewriter.append(text, NullWritable.get());
-// index++;
-// }
-//
-// filewriter.close();
-// }
-// }
-//
-// private String padZeros(int number, int digits) {
-// String string = String.valueOf(number);
-// for (int i=(digits-string.length()); i>0; i--) {
-// string = "0" + string;
-// }
-// return string;
-// }
-//
-// public static class DummyLoader extends Loader {
-// @Override
-// public void load(LoaderContext context, Object oc, Object oj) throws Exception {
-// int index = 1;
-// int sum = 0;
-// Object[] array;
-// while ((array = context.getDataReader().readArrayRecord()) != null) {
-// sum += Integer.valueOf(array[0].toString());
-// index++;
-// };
-//
-// int numbers = NUMBER_OF_FILES*NUMBER_OF_ROWS_PER_FILE;
-// assertEquals((1+numbers)*numbers/2, sum);
-//
-// assertEquals(NUMBER_OF_FILES*NUMBER_OF_ROWS_PER_FILE, index-1);
-// }
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c0b22b1d/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java
deleted file mode 100644
index 8eba049..0000000
--- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestHdfsLoad.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.job;
-
-import junit.framework.TestCase;
-
-//import org.apache.sqoop.connector.idf.CSVIntermediateDataFormat;
-//import org.apache.sqoop.job.etl.HdfsSequenceImportLoader;
-//import org.apache.sqoop.job.etl.HdfsTextImportLoader;
-
-public class TestHdfsLoad extends TestCase {
-
-// private static final String OUTPUT_ROOT = System.getProperty("maven.build.directory", "/tmp") + "/sqoop/warehouse/";
-// private static final String OUTPUT_FILE = "part-r-00000";
-// private static final int START_ID = 1;
-// private static final int NUMBER_OF_IDS = 9;
-// private static final int NUMBER_OF_ROWS_PER_ID = 10;
-//
-// private String outdir;
-//
-// public TestHdfsLoad() {
-// outdir = OUTPUT_ROOT + "/" + getClass().getSimpleName();
-// }
-//
-// public void testUncompressedText() throws Exception {
-// FileUtils.delete(outdir);
-//
-// Configuration conf = new Configuration();
-// ConfigurationUtils.setJobType(conf, MJob.Type.IMPORT);
-// conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
-// conf.set(JobConstants.JOB_ETL_EXTRACTOR, DummyExtractor.class.getName());
-// conf.set(JobConstants.JOB_ETL_LOADER, HdfsTextImportLoader.class.getName());
-// conf.set(JobConstants.INTERMEDIATE_DATA_FORMAT,
-// CSVIntermediateDataFormat.class.getName());
-// conf.set(JobConstants.HADOOP_OUTDIR, outdir);
-// Schema schema = new Schema("Test");
-// schema.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"))
-// .addColumn(new org.apache.sqoop.schema.type.Text("3"));
-//
-// Job job = new Job(conf);
-// ConfigurationUtils.setConnectorSchema(job, schema);
-// JobUtils.runJob(job.getConfiguration());
-//
-// String fileName = outdir + "/" + OUTPUT_FILE;
-// InputStream filestream = FileUtils.open(fileName);
-// BufferedReader filereader = new BufferedReader(new InputStreamReader(
-// filestream, Charsets.UTF_8));
-// verifyOutputText(filereader);
-// }
-//
-// public void testCompressedText() throws Exception {
-// FileUtils.delete(outdir);
-//
-// Configuration conf = new Configuration();
-// ConfigurationUtils.setJobType(conf, MJob.Type.IMPORT);
-// conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
-// conf.set(JobConstants.JOB_ETL_EXTRACTOR, DummyExtractor.class.getName());
-// conf.set(JobConstants.JOB_ETL_LOADER, HdfsTextImportLoader.class.getName());
-// conf.set(JobConstants.INTERMEDIATE_DATA_FORMAT,
-// CSVIntermediateDataFormat.class.getName());
-// conf.set(JobConstants.HADOOP_OUTDIR, outdir);
-// conf.setBoolean(JobConstants.HADOOP_COMPRESS, true);
-//
-// Schema schema = new Schema("Test");
-// schema.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"))
-// .addColumn(new org.apache.sqoop.schema.type.Text("3"));
-//
-// Job job = new Job(conf);
-// ConfigurationUtils.setConnectorSchema(job, schema);
-// JobUtils.runJob(job.getConfiguration());
-//
-// Class<? extends CompressionCodec> codecClass = conf.getClass(
-// JobConstants.HADOOP_COMPRESS_CODEC, SqoopFileOutputFormat.DEFAULT_CODEC)
-// .asSubclass(CompressionCodec.class);
-// CompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf);
-// String fileName = outdir + "/" + OUTPUT_FILE + codec.getDefaultExtension();
-// InputStream filestream = codec.createInputStream(FileUtils.open(fileName));
-// BufferedReader filereader = new BufferedReader(new InputStreamReader(
-// filestream, Charsets.UTF_8));
-// verifyOutputText(filereader);
-// }
-//
-// private void verifyOutputText(BufferedReader reader) throws IOException {
-// String actual = null;
-// String expected;
-// Data data = new Data();
-// int index = START_ID*NUMBER_OF_ROWS_PER_ID;
-// while ((actual = reader.readLine()) != null){
-// data.setContent(new Object[] {
-// index, (double) index, new String(new byte[] {(byte)(index + 127)}, Charsets.ISO_8859_1) },
-// Data.ARRAY_RECORD);
-// expected = data.toString();
-// index++;
-//
-// assertEquals(expected, actual);
-// }
-// reader.close();
-//
-// assertEquals(NUMBER_OF_IDS*NUMBER_OF_ROWS_PER_ID,
-// index-START_ID*NUMBER_OF_ROWS_PER_ID);
-// }
-//
-// public void testUncompressedSequence() throws Exception {
-// FileUtils.delete(outdir);
-//
-// Configuration conf = new Configuration();
-// ConfigurationUtils.setJobType(conf, MJob.Type.IMPORT);
-// conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
-// conf.set(JobConstants.JOB_ETL_EXTRACTOR, DummyExtractor.class.getName());
-// conf.set(JobConstants.JOB_ETL_LOADER, HdfsSequenceImportLoader.class.getName());
-// conf.set(JobConstants.INTERMEDIATE_DATA_FORMAT,
-// CSVIntermediateDataFormat.class.getName());
-// conf.set(JobConstants.HADOOP_OUTDIR, outdir);
-//
-// Schema schema = new Schema("Test");
-// schema.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"))
-// .addColumn(new org.apache.sqoop.schema.type.Text("3"));
-//
-// Job job = new Job(conf);
-// ConfigurationUtils.setConnectorSchema(job, schema);
-// JobUtils.runJob(job.getConfiguration());
-//
-// Path filepath = new Path(outdir,
-// OUTPUT_FILE + HdfsSequenceImportLoader.EXTENSION);
-// SequenceFile.Reader filereader = new SequenceFile.Reader(
-// filepath.getFileSystem(conf), filepath, conf);
-// verifyOutputSequence(filereader);
-// }
-//
-// public void testCompressedSequence() throws Exception {
-// FileUtils.delete(outdir);
-//
-// Configuration conf = new Configuration();
-// ConfigurationUtils.setJobType(conf, MJob.Type.IMPORT);
-// conf.set(JobConstants.JOB_ETL_PARTITIONER, DummyPartitioner.class.getName());
-// conf.set(JobConstants.JOB_ETL_EXTRACTOR, DummyExtractor.class.getName());
-// conf.set(JobConstants.JOB_ETL_LOADER, HdfsSequenceImportLoader.class.getName());
-// conf.set(JobConstants.INTERMEDIATE_DATA_FORMAT,
-// CSVIntermediateDataFormat.class.getName());
-// conf.set(JobConstants.HADOOP_OUTDIR, outdir);
-// conf.setBoolean(JobConstants.HADOOP_COMPRESS, true);
-//
-// Schema schema = new Schema("Test");
-// schema.addColumn(new FixedPoint("1")).addColumn(new FloatingPoint("2"))
-// .addColumn(new org.apache.sqoop.schema.type.Text("3"));
-//
-// Job job = new Job(conf);
-// ConfigurationUtils.setConnectorSchema(job, schema);
-// JobUtils.runJob(job.getConfiguration());
-// Path filepath = new Path(outdir,
-// OUTPUT_FILE + HdfsSequenceImportLoader.EXTENSION);
-// SequenceFile.Reader filereader = new SequenceFile.Reader(filepath.getFileSystem(conf), filepath, conf);
-// verifyOutputSequence(filereader);
-// }
-//
-// private void verifyOutputSequence(SequenceFile.Reader reader) throws IOException {
-// int index = START_ID*NUMBER_OF_ROWS_PER_ID;
-// Text actual = new Text();
-// Text expected = new Text();
-// Data data = new Data();
-// while (reader.next(actual)){
-// data.setContent(new Object[] {
-// index, (double) index, new String(new byte[] {(byte)(index + 127)}, Charsets.ISO_8859_1) },
-// Data.ARRAY_RECORD);
-// expected.set(data.toString());
-// index++;
-//
-// assertEquals(expected.toString(), actual.toString());
-// }
-// reader.close();
-//
-// assertEquals(NUMBER_OF_IDS*NUMBER_OF_ROWS_PER_ID,
-// index-START_ID*NUMBER_OF_ROWS_PER_ID);
-// }
-//
-// public static class DummyPartition extends Partition {
-// private int id;
-//
-// public void setId(int id) {
-// this.id = id;
-// }
-//
-// public int getId() {
-// return id;
-// }
-//
-// @Override
-// public void readFields(DataInput in) throws IOException {
-// id = in.readInt();
-// }
-//
-// @Override
-// public void write(DataOutput out) throws IOException {
-// out.writeInt(id);
-// }
-//
-// @Override
-// public String toString() {
-// return Integer.toString(id);
-// }
-// }
-//
-// public static class DummyPartitioner extends Partitioner {
-// @Override
-// public List<Partition> getPartitions(PartitionerContext context, Object oc, Object oj) {
-// List<Partition> partitions = new LinkedList<Partition>();
-// for (int id = START_ID; id <= NUMBER_OF_IDS; id++) {
-// DummyPartition partition = new DummyPartition();
-// partition.setId(id);
-// partitions.add(partition);
-// }
-// return partitions;
-// }
-// }
-//
-// public static class DummyExtractor extends Extractor {
-// @Override
-// public void extract(ExtractorContext context, Object oc, Object oj, Object partition) {
-// int id = ((DummyPartition)partition).getId();
-// for (int row = 0; row < NUMBER_OF_ROWS_PER_ID; row++) {
-// Object[] array = new Object[] {
-// id * NUMBER_OF_ROWS_PER_ID + row,
-// (double) (id * NUMBER_OF_ROWS_PER_ID + row),
-// new String(new byte[]{(byte)(id * NUMBER_OF_ROWS_PER_ID + row + 127)}, Charsets.ISO_8859_1)
-// };
-// context.getDataWriter().writeArrayRecord(array);
-// }
-// }
-//
-// @Override
-// public long getRowsRead() {
-// return NUMBER_OF_ROWS_PER_ID;
-// }
-// }
-}
[47/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java
new file mode 100644
index 0000000..cec46f6
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.model.MBooleanInput;
+import org.apache.sqoop.model.MEnumInput;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConfigType;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MInputType;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MMapInput;
+import org.apache.sqoop.model.MStringInput;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Convenient static methods for serializing config objects.
+ */
+public final class ConfigSerialization {
+
+ public static final String ALL = "all";
+ public static final String ID = "id";
+ public static final String NAME = "name";
+ public static final String VERSION = "version";
+ public static final String CLASS = "class";
+ public static final String ENABLED = "enabled";
+ public static final String CREATION_USER = "creation-user";
+ public static final String CREATION_DATE = "creation-date";
+ public static final String UPDATE_USER = "update-user";
+ public static final String UPDATE_DATE = "update-date";
+ // TODO(VB): Move these constants to connector bean
+ public static final String CONNECTOR_LINK_CONFIG = "link-config";
+ public static final String CONNECTOR_JOB_CONFIG = "job-config";
+ // TODO:move these configs to driver bean
+ public static final String DRIVER_VERSION = "driver-version";
+ public static final String DRIVER_CONFIG = "driver-config";
+
+ public static final String CONFIG_NAME = "name";
+ public static final String CONFIG_TYPE = "type";
+ public static final String CONFIG_INPUTS = "inputs";
+ public static final String CONFIG_INPUT_NAME = "name";
+ public static final String CONFIG_INPUT_TYPE = "type";
+ public static final String CONFIG_INPUT_SENSITIVE = "sensitive";
+ public static final String CONFIG_INPUT_SIZE = "size";
+ public static final String CONFIG_INPUT_VALUE = "value";
+ public static final String CONFIG_INPUT_VALUES = "values";
+
+ /**
+ * Transform given list of configs to JSON Array object.
+ *
+ * @param mConfigs List of configs.
+ * @return JSON object with serialized config of the list.
+ */
+ @SuppressWarnings("unchecked")
+ public static JSONArray extractConfigList(List<MConfig> mConfigs, boolean skipSensitive) {
+ JSONArray configs = new JSONArray();
+
+ for (MConfig mConfig : mConfigs) {
+ configs.add(extractConfig(mConfig, skipSensitive));
+ }
+
+ return configs;
+ }
+
+ /**
+ * Transform given config to JSON Object.
+ *
+ * @param mConfig Given MConfig instance
+ * @param skipSensitive conditionally add sensitive input values
+ * @return Serialized JSON object.
+ */
+ @SuppressWarnings("unchecked")
+ static JSONObject extractConfig(MConfig mConfig, boolean skipSensitive) {
+ JSONObject config = new JSONObject();
+ config.put(ID, mConfig.getPersistenceId());
+ config.put(CONFIG_NAME, mConfig.getName());
+ config.put(CONFIG_TYPE, MConfigType.LINK.toString());
+ JSONArray mInputs = new JSONArray();
+ config.put(CONFIG_INPUTS, mInputs);
+
+ for (MInput<?> mInput : mConfig.getInputs()) {
+ JSONObject input = new JSONObject();
+ input.put(ID, mInput.getPersistenceId());
+ input.put(CONFIG_INPUT_NAME, mInput.getName());
+ input.put(CONFIG_INPUT_TYPE, mInput.getType().toString());
+ input.put(CONFIG_INPUT_SENSITIVE, mInput.isSensitive());
+
+ // String specific serialization
+ if (mInput.getType() == MInputType.STRING) {
+ input.put(CONFIG_INPUT_SIZE,
+ ((MStringInput)mInput).getMaxLength());
+ }
+
+ // Enum specific serialization
+ if(mInput.getType() == MInputType.ENUM) {
+ input.put(CONFIG_INPUT_VALUES,
+ StringUtils.join(((MEnumInput)mInput).getValues(), ","));
+ }
+
+ // Serialize value if is there
+ // Skip if sensitive
+ if (!mInput.isEmpty() && !(skipSensitive && mInput.isSensitive())) {
+ if (mInput.getType() == MInputType.MAP) {
+ input.put(CONFIG_INPUT_VALUE, mInput.getValue());
+ } else {
+ input.put(CONFIG_INPUT_VALUE, mInput.getUrlSafeValueString());
+ }
+ }
+
+ mInputs.add(input);
+ }
+
+ return config;
+ }
+
+ /**
+ * Restore List of MConfigs from JSON Array.
+ *
+ * @param configs JSON array representing list of MConfigs
+ * @return Restored list of MConfigs
+ */
+ public static List<MConfig> restoreConfigList(JSONArray configs) {
+ List<MConfig> mConfigs = new ArrayList<MConfig>();
+
+ for (int i = 0; i < configs.size(); i++) {
+ mConfigs.add(restoreConfig((JSONObject) configs.get(i)));
+ }
+
+ return mConfigs;
+ }
+
+ /**
+ * Restore one MConfig from JSON Object.
+ *
+ * @param config JSON representation of the MConfig.
+ * @return Restored MConfig.
+ */
+ static MConfig restoreConfig(JSONObject config) {
+ JSONArray inputs = (JSONArray) config.get(CONFIG_INPUTS);
+
+ List<MInput<?>> mInputs = new ArrayList<MInput<?>>();
+ for (int i = 0; i < inputs.size(); i++) {
+ JSONObject input = (JSONObject) inputs.get(i);
+ MInputType type =
+ MInputType.valueOf((String) input.get(CONFIG_INPUT_TYPE));
+ String name = (String) input.get(CONFIG_INPUT_NAME);
+ Boolean sensitive = (Boolean) input.get(CONFIG_INPUT_SENSITIVE);
+ MInput mInput = null;
+ switch (type) {
+ case STRING: {
+ long size = (Long) input.get(CONFIG_INPUT_SIZE);
+ mInput = new MStringInput(name, sensitive.booleanValue(), (short) size);
+ break;
+ }
+ case MAP: {
+ mInput = new MMapInput(name, sensitive.booleanValue());
+ break;
+ }
+ case INTEGER: {
+ mInput = new MIntegerInput(name, sensitive.booleanValue());
+ break;
+ }
+ case BOOLEAN: {
+ mInput = new MBooleanInput(name, sensitive.booleanValue());
+ break;
+ }
+ case ENUM: {
+ String values = (String) input.get(CONFIG_INPUT_VALUES);
+ mInput = new MEnumInput(name, sensitive.booleanValue(), values.split(","));
+ break;
+ }
+ }
+
+ // Propagate config ID
+ mInput.setPersistenceId((Long)input.get(ID));
+
+ // Propagate config optional value
+ if(input.containsKey(CONFIG_INPUT_VALUE)) {
+ switch (type) {
+ case MAP:
+ try {
+ mInput.setValue((Map<String, String>)input.get(CONFIG_INPUT_VALUE));
+ } catch (ClassCastException e) {
+ throw new SqoopException(SerializationError.SERIALIZATION_001, name + " requires a 'map' value.");
+ }
+ break;
+ default:
+ mInput.restoreFromUrlSafeValueString(
+ (String) input.get(CONFIG_INPUT_VALUE));
+ break;
+ }
+ }
+ mInputs.add(mInput);
+ }
+
+ MConfig mConfig = new MConfig((String) config.get(CONFIG_NAME), mInputs);
+ mConfig.setPersistenceId((Long) config.get(ID));
+ return mConfig;
+ }
+
+ private ConfigSerialization() {
+ // Do not instantiate
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java
deleted file mode 100644
index 77f6191..0000000
--- a/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json.util;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MEnumInput;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFormType;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MInputType;
-import org.apache.sqoop.model.MIntegerInput;
-import org.apache.sqoop.model.MMapInput;
-import org.apache.sqoop.model.MStringInput;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Convenient static methods for serializing forms.
- */
-public final class FormSerialization {
-
- public static final String ALL = "all";
- public static final String ID = "id";
- public static final String NAME = "name";
- public static final String VERSION = "version";
- public static final String DRIVER_VERSION = "driver-version";
- public static final String CLASS = "class";
- public static final String ENABLED = "enabled";
- public static final String CREATION_USER = "creation-user";
- public static final String CREATION_DATE = "creation-date";
- public static final String UPDATE_USER = "update-user";
- public static final String UPDATE_DATE = "update-date";
- public static final String CON_FORMS = "con-forms";
- public static final String JOB_FORMS = "job-forms";
-
- public static final String FORM_NAME = "name";
- public static final String FORM_TYPE = "type";
- public static final String FORM_INPUTS = "inputs";
- public static final String FORM_INPUT_NAME = "name";
- public static final String FORM_INPUT_TYPE = "type";
- public static final String FORM_INPUT_SENSITIVE = "sensitive";
- public static final String FORM_INPUT_SIZE = "size";
- public static final String FORM_INPUT_VALUE = "value";
- public static final String FORM_INPUT_VALUES = "values";
-
- /**
- * Transform given list of forms to JSON Array object.
- *
- * @param mForms List of forms.
- * @return JSON object with serialized form of the list.
- */
- @SuppressWarnings("unchecked")
- public static JSONArray extractForms(List<MForm> mForms, boolean skipSensitive) {
- JSONArray forms = new JSONArray();
-
- for (MForm mForm : mForms) {
- forms.add(extractForm(mForm, skipSensitive));
- }
-
- return forms;
- }
-
- /**
- * Transform given form to JSON Object.
- *
- * @param mForm Given MForm instance
- * @param skipSensitive conditionally add sensitive input values
- * @return Serialized JSON object.
- */
- @SuppressWarnings("unchecked")
- public static JSONObject extractForm(MForm mForm, boolean skipSensitive) {
- JSONObject form = new JSONObject();
- form.put(ID, mForm.getPersistenceId());
- form.put(FORM_NAME, mForm.getName());
- form.put(FORM_TYPE, MFormType.CONNECTION.toString());
- JSONArray mInputs = new JSONArray();
- form.put(FORM_INPUTS, mInputs);
-
- for (MInput<?> mInput : mForm.getInputs()) {
- JSONObject input = new JSONObject();
- input.put(ID, mInput.getPersistenceId());
- input.put(FORM_INPUT_NAME, mInput.getName());
- input.put(FORM_INPUT_TYPE, mInput.getType().toString());
- input.put(FORM_INPUT_SENSITIVE, mInput.isSensitive());
-
- // String specific serialization
- if (mInput.getType() == MInputType.STRING) {
- input.put(FORM_INPUT_SIZE,
- ((MStringInput)mInput).getMaxLength());
- }
-
- // Enum specific serialization
- if(mInput.getType() == MInputType.ENUM) {
- input.put(FORM_INPUT_VALUES,
- StringUtils.join(((MEnumInput)mInput).getValues(), ","));
- }
-
- // Serialize value if is there
- // Skip if sensitive
- if (!mInput.isEmpty() && !(skipSensitive && mInput.isSensitive())) {
- if (mInput.getType() == MInputType.MAP) {
- input.put(FORM_INPUT_VALUE, mInput.getValue());
- } else {
- input.put(FORM_INPUT_VALUE, mInput.getUrlSafeValueString());
- }
- }
-
- mInputs.add(input);
- }
-
- return form;
- }
-
- /**
- * Restore List of MForms from JSON Array.
- *
- * @param forms JSON array representing list of MForms
- * @return Restored list of MForms
- */
- public static List<MForm> restoreForms(JSONArray forms) {
- List<MForm> mForms = new ArrayList<MForm>();
-
- for (int i = 0; i < forms.size(); i++) {
- mForms.add(restoreForm((JSONObject) forms.get(i)));
- }
-
- return mForms;
- }
-
- /**
- * Restore one MForm from JSON Object.
- *
- * @param form JSON representation of the MForm.
- * @return Restored MForm.
- */
- public static MForm restoreForm(JSONObject form) {
- JSONArray inputs = (JSONArray) form.get(FORM_INPUTS);
-
- List<MInput<?>> mInputs = new ArrayList<MInput<?>>();
- for (int i = 0; i < inputs.size(); i++) {
- JSONObject input = (JSONObject) inputs.get(i);
- MInputType type =
- MInputType.valueOf((String) input.get(FORM_INPUT_TYPE));
- String name = (String) input.get(FORM_INPUT_NAME);
- Boolean sensitive = (Boolean) input.get(FORM_INPUT_SENSITIVE);
- MInput mInput = null;
- switch (type) {
- case STRING: {
- long size = (Long) input.get(FORM_INPUT_SIZE);
- mInput = new MStringInput(name, sensitive.booleanValue(), (short) size);
- break;
- }
- case MAP: {
- mInput = new MMapInput(name, sensitive.booleanValue());
- break;
- }
- case INTEGER: {
- mInput = new MIntegerInput(name, sensitive.booleanValue());
- break;
- }
- case BOOLEAN: {
- mInput = new MBooleanInput(name, sensitive.booleanValue());
- break;
- }
- case ENUM: {
- String values = (String) input.get(FORM_INPUT_VALUES);
- mInput = new MEnumInput(name, sensitive.booleanValue(), values.split(","));
- break;
- }
- }
-
- // Propagate form ID
- mInput.setPersistenceId((Long)input.get(ID));
-
- // Propagate form optional value
- if(input.containsKey(FORM_INPUT_VALUE)) {
- switch (type) {
- case MAP:
- try {
- mInput.setValue((Map<String, String>)input.get(FORM_INPUT_VALUE));
- } catch (ClassCastException e) {
- throw new SqoopException(SerializationError.SERIALIZATION_001, name + " requires a 'map' value.");
- }
- break;
- default:
- mInput.restoreFromUrlSafeValueString(
- (String) input.get(FORM_INPUT_VALUE));
- break;
- }
- }
- mInputs.add(mInput);
- }
-
- MForm mForm = new MForm((String) form.get(FORM_NAME), mInputs);
- mForm.setPersistenceId((Long) form.get(ID));
- return mForm;
- }
-
- private FormSerialization() {
- // Do not instantiate
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
index 20ee5f3..f27d81d 100644
--- a/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
+++ b/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
@@ -43,7 +43,6 @@ public final class ResourceBundleSerialization {
for (ResourceBundle bundle : bundles) {
array.add(extractResourceBundle(bundle));
}
-
return array;
}
@@ -58,7 +57,6 @@ public final class ResourceBundleSerialization {
return json;
}
- @SuppressWarnings("unchecked")
public static List<ResourceBundle> restoreResourceBundles(JSONArray array) {
List<ResourceBundle> bundles = new LinkedList<ResourceBundle>();
for (Object item : array) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/Config.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/Config.java b/common/src/main/java/org/apache/sqoop/model/Config.java
new file mode 100644
index 0000000..b4d847d
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/Config.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Denote config in Configuration class
+ */
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Config {
+ /**
+ * Optional name for the form object
+ *
+ * @return
+ */
+ String name() default "";
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/ConfigClass.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/ConfigClass.java b/common/src/main/java/org/apache/sqoop/model/ConfigClass.java
new file mode 100644
index 0000000..f925759
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/ConfigClass.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Denote configuration class
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface ConfigClass {
+
+ /**
+ * Default size for Inputs in this config.
+ *
+ * @return
+ */
+ short defaultSize() default -1;
+
+ /**
+ * List of validators associated with this config.
+ *
+ * @return
+ */
+ Validator[] validators() default {};
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java b/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java
new file mode 100644
index 0000000..9e762dc
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java
@@ -0,0 +1,637 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.utils.ClassUtils;
+import org.apache.sqoop.validation.Message;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.ConfigValidator;
+import org.apache.sqoop.validation.ConfigValidationResult;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * Util class for transforming data from correctly annotated configuration
+ * objects to different structures and vice-versa.
+ *
+ * TODO: This class should see some overhaul into more reusable code, especially expose and re-use the methods at the end.
+ */
+public class ConfigUtils {
+
+ /**
+ * Transform correctly annotated configuration object to corresponding
+ * list of configs.
+ *
+ * Configs will be order according to the occurrence in the configuration
+ * class. Inputs will be also ordered based on occurrence.
+ *
+ * @param configuration Annotated arbitrary configuration object
+ * @return Corresponding list of configs
+ */
+ public static List<MConfig> toConfigs(Object configuration) {
+ return toConfigs(configuration.getClass(), configuration);
+ }
+
+ public static List<MConfig> toConfigs(Class klass) {
+ return toConfigs(klass, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static List<MConfig> toConfigs(Class klass, Object configuration) {
+ Set<String> formNames = new HashSet<String>();
+
+ ConfigurationClass configurationClass =
+ (ConfigurationClass)klass.getAnnotation(ConfigurationClass.class);
+
+ // Each configuration object must have this class annotation
+ if(configurationClass == null) {
+ throw new SqoopException(ModelError.MODEL_003,
+ "Missing annotation ConfigurationClass on class " + klass.getName());
+ }
+
+ List<MConfig> configs = new LinkedList<MConfig>();
+
+ // Iterate over all declared fields
+ for (Field field : klass.getDeclaredFields()) {
+ field.setAccessible(true);
+
+ // Each field that should be part of user input should have Input
+ // annotation.
+ Config formAnnotation = field.getAnnotation(Config.class);
+
+ if (formAnnotation != null) {
+ String formName = getFormName(field, formAnnotation, formNames);
+
+ Class type = field.getType();
+
+ Object value = null;
+ if(configuration != null) {
+ try {
+ value = field.get(configuration);
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005,
+ "Can't retrieve value from " + field.getName(), e);
+ }
+ }
+
+ configs.add(toConfig(formName, type, value));
+ }
+ }
+
+ return configs;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static MConfig toConfig(String formName, Class klass, Object object) {
+ ConfigClass global =
+ (ConfigClass)klass.getAnnotation(ConfigClass.class);
+
+ // Each configuration object must have this class annotation
+ if(global == null) {
+ throw new SqoopException(ModelError.MODEL_003,
+ "Missing annotation ConfigClass on class " + klass.getName());
+ }
+
+ // Intermediate list of inputs
+ List<MInput<?>> inputs = new LinkedList<MInput<?>>();
+
+ // Iterate over all declared fields
+ for (Field field : klass.getDeclaredFields()) {
+ field.setAccessible(true);
+
+ String fieldName = field.getName();
+ String inputName = formName + "." + fieldName;
+
+ // Each field that should be part of user input should have Input
+ // annotation.
+ Input inputAnnotation = field.getAnnotation(Input.class);
+
+ if(inputAnnotation != null) {
+ boolean sensitive = inputAnnotation.sensitive();
+ short maxLen = inputAnnotation.size();
+ Class<?> type = field.getType();
+
+ MInput input;
+
+ // We need to support NULL, so we do not support primitive types
+ if(type.isPrimitive()) {
+ throw new SqoopException(ModelError.MODEL_007,
+ "Detected primitive type " + type + " for field " + fieldName);
+ }
+
+ // Instantiate corresponding MInput<?> structure
+ if (type == String.class) {
+ input = new MStringInput(inputName, sensitive, maxLen);
+ } else if (type.isAssignableFrom(Map.class)) {
+ input = new MMapInput(inputName, sensitive);
+ } else if (type == Integer.class) {
+ input = new MIntegerInput(inputName, sensitive);
+ } else if (type == Boolean.class) {
+ input = new MBooleanInput(inputName, sensitive);
+ } else if (type.isEnum()) {
+ input = new MEnumInput(inputName, sensitive,
+ ClassUtils.getEnumStrings(type));
+ } else {
+ throw new SqoopException(ModelError.MODEL_004, "Unsupported type "
+ + type.getName() + " for input " + fieldName);
+ }
+
+ // Move value if it's present in original configuration object
+ if(object != null) {
+ Object value;
+ try {
+ value = field.get(object);
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005,
+ "Can't retrieve value from " + field.getName(), e);
+ }
+ if(value == null) {
+ input.setEmpty();
+ } else {
+ input.setValue(value);
+ }
+ }
+
+ inputs.add(input);
+ }
+ }
+
+ return new MConfig(formName, inputs);
+ }
+
+ private static Field getFieldFromName(Class<?> klass, String name) {
+ Field formField;
+ try {
+ formField = klass.getDeclaredField(name);
+ } catch (NoSuchFieldException e) {
+ // reverse lookup form field from custom form name
+ if (name != null) {
+ for (Field field : klass.getDeclaredFields()) {
+ Config formAnnotation = field.getAnnotation(Config.class);
+ if (formAnnotation == null) {
+ continue;
+ }
+ if (!StringUtils.isEmpty(formAnnotation.name()) && name.equals(formAnnotation.name())) {
+ return field;
+ }
+ }
+ }
+ throw new SqoopException(ModelError.MODEL_006, "Missing field " + name + " on form class "
+ + klass.getCanonicalName(), e);
+ }
+ return formField;
+ }
+
+ /**
+ * Move config values from config list into corresponding configuration object.
+ *
+ * @param configs Input config list
+ * @param configuration Output configuration object
+ */
+ public static void fromConfigs(List<MConfig> configs, Object configuration) {
+ Class klass = configuration.getClass();
+
+ for(MConfig config : configs) {
+ Field configField;
+ try {
+ configField = klass.getDeclaredField(config.getName());
+ } catch (NoSuchFieldException e) {
+ throw new SqoopException(ModelError.MODEL_006,
+ "Missing field " + config.getName() + " on config class " + klass.getCanonicalName(), e);
+ }
+
+ Field formField = getFieldFromName(klass, config.getName());
+ // We need to access this field even if it would be declared as private
+ configField.setAccessible(true);
+ Class configClass = configField.getType();
+ Object newValue = ClassUtils.instantiate(configClass);
+
+ if (newValue == null) {
+ throw new SqoopException(ModelError.MODEL_006,
+ "Can't instantiate new config " + configClass);
+ }
+
+ for(MInput input : config.getInputs()) {
+ String[] splitNames = input.getName().split("\\.");
+ if (splitNames.length != 2) {
+ throw new SqoopException(ModelError.MODEL_009, "Invalid name: "
+ + input.getName());
+ }
+
+ String inputName = splitNames[1];
+ // TODO(jarcec): Names structures fix, handle error cases
+ Field inputField;
+ try {
+ inputField = configClass.getDeclaredField(inputName);
+ } catch (NoSuchFieldException e) {
+ throw new SqoopException(ModelError.MODEL_006, "Missing field "
+ + input.getName(), e);
+ }
+
+ // We need to access this field even if it would be declared as private
+ inputField.setAccessible(true);
+
+ try {
+ if (input.isEmpty()) {
+ inputField.set(newValue, null);
+ } else {
+ if (input.getType() == MInputType.ENUM) {
+ inputField.set(newValue, Enum.valueOf(
+ (Class<? extends Enum>) inputField.getType(),
+ (String) input.getValue()));
+ } else {
+ inputField.set(newValue, input.getValue());
+ }
+ }
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005, "Issue with field "
+ + inputField.getName(), e);
+ }
+ }
+
+ try {
+ configField.set(configuration, newValue);
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005,
+ "Issue with field " + configField.getName(), e);
+ }
+ }
+ }
+
+ /**
+ * Apply validations on the configs.
+ *
+ * @param configs Configs that should be updated
+ * @param validation Validation that we should apply
+ */
+ public static void applyValidation(List<MConfig> configs, ConfigValidator validation) {
+ Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages = validation.getMessages();
+
+ for(MConfig config : configs) {
+ applyValidation(config, messages);
+
+ for(MInput input : config.getInputs()) {
+ applyValidation(input, messages);
+ }
+ }
+ }
+
+ /**
+ * Apply validation on given validated element.
+ *
+ * @param element
+ * Element on what we're applying the validations
+ * @param messages
+ * Map of all validation messages
+ */
+ public static void applyValidation(MValidatedElement element, Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages) {
+ ConfigValidator.ConfigInput name = new ConfigValidator.ConfigInput(element.getName());
+
+ if(messages.containsKey(name)) {
+ ConfigValidator.Message message = messages.get(name);
+ element.addValidationMessage(new Message(message.getStatus(), message.getMessage()));
+ } else {
+ element.addValidationMessage(new Message(Status.getDefault(), null));
+ }
+ }
+
+
+ /**
+ * Apply given validations on list of configs.
+ *
+ * @param configs
+ * @param result
+ */
+ public static void applyValidation(List<MConfig> configs, ConfigValidationResult result) {
+ for(MConfig config : configs) {
+ applyValidation(config, result);
+
+ for(MInput input : config.getInputs()) {
+ applyValidation(input, result);
+ }
+ }
+ }
+
+ /**
+ * Apply validation messages on given element.
+ *
+ * Element's state will be set to default if there are no associated messages.
+ *
+ * @param element
+ * @param result
+ */
+ public static void applyValidation(MValidatedElement element, ConfigValidationResult result) {
+ List<Message> messages = result.getMessages().get(element.getName());
+
+ if(messages != null) {
+ element.setValidationMessages(messages);
+ } else {
+ element.resetValidationMessages();
+ }
+ }
+
+ /**
+ * Convert configuration object to JSON. Only filled properties are serialized,
+ * properties with null value are skipped.
+ *
+ * @param configuration Correctly annotated configuration object
+ * @return String of JSON representation
+ */
+ @SuppressWarnings("unchecked")
+ public static String toJson(Object configuration) {
+ Class klass = configuration.getClass();
+ Set<String> formNames = new HashSet<String>();
+ ConfigurationClass configurationClass =
+ (ConfigurationClass)klass.getAnnotation(ConfigurationClass.class);
+
+ // Each configuration object must have this class annotation
+ if(configurationClass == null) {
+ throw new SqoopException(ModelError.MODEL_003,
+ "Missing annotation ConfigurationGroup on class " + klass.getName());
+ }
+
+ JSONObject jsonOutput = new JSONObject();
+
+ // Iterate over all declared fields
+ for (Field formField : klass.getDeclaredFields()) {
+ formField.setAccessible(true);
+
+ // We're processing only config validations
+ Config formAnnotation = formField.getAnnotation(Config.class);
+ if(formAnnotation == null) {
+ continue;
+ }
+ String formName = getFormName(formField, formAnnotation, formNames);
+
+ Object formValue;
+ try {
+ formValue = formField.get(configuration);
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005, "Issue with field "
+ + formName, e);
+ }
+
+ JSONObject jsonConfig = new JSONObject();
+
+ // Now process each input on the config
+ for(Field inputField : formField.getType().getDeclaredFields()) {
+ inputField.setAccessible(true);
+ String inputName = inputField.getName();
+
+ Object value;
+ try {
+ value = inputField.get(formValue);
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005, "Issue with field "
+ + formName + "." + inputName, e);
+ }
+
+ Input inputAnnotation = inputField.getAnnotation(Input.class);
+
+ // Do not serialize all values
+ if (inputAnnotation != null && value != null) {
+ Class<?> type = inputField.getType();
+
+ // We need to support NULL, so we do not support primitive types
+ if (type.isPrimitive()) {
+ throw new SqoopException(ModelError.MODEL_007,
+ "Detected primitive type " + type + " for field " + formName
+ + "." + inputName);
+ }
+
+ if(type == String.class) {
+ jsonConfig.put(inputName, value);
+ } else if (type.isAssignableFrom(Map.class)) {
+ JSONObject map = new JSONObject();
+ for (Object key : ((Map) value).keySet()) {
+ map.put(key, ((Map) value).get(key));
+ }
+ jsonConfig.put(inputName, map);
+ } else if(type == Integer.class) {
+ jsonConfig.put(inputName, value);
+ } else if(type.isEnum()) {
+ jsonConfig.put(inputName, value.toString());
+ } else if(type == Boolean.class) {
+ jsonConfig.put(inputName, value);
+ }else {
+ throw new SqoopException(ModelError.MODEL_004,
+ "Unsupported type " + type.getName() + " for input " + formName + "." + inputName);
+ }
+ }
+ }
+
+ jsonOutput.put(formName, jsonConfig);
+ }
+ return jsonOutput.toJSONString();
+ }
+
+ /**
+ * Parse given input JSON string and move it's values to given configuration
+ * object.
+ *
+ * @param json JSON representation of the configuration object
+ * @param configuration ConfigurationGroup object to be filled
+ */
+ public static void fillValues(String json, Object configuration) {
+ Class klass = configuration.getClass();
+
+ Set<String> formNames = new HashSet<String>();
+ JSONObject jsonConfigs = (JSONObject) JSONValue.parse(json);
+
+ for(Field configField : klass.getDeclaredFields()) {
+ configField.setAccessible(true);
+ String configName = configField.getName();
+
+ // We're processing only config validations
+ Config formAnnotation = configField.getAnnotation(Config.class);
+ if(formAnnotation == null) {
+ continue;
+ }
+ String formName = getFormName(configField, formAnnotation, formNames);
+
+ try {
+ configField.set(configuration, configField.getType().newInstance());
+ } catch (Exception e) {
+ throw new SqoopException(ModelError.MODEL_005,
+ "Issue with field " + configName, e);
+ }
+
+ JSONObject jsonInputs = (JSONObject) jsonConfigs.get(configField.getName());
+ if(jsonInputs == null) {
+ continue;
+ }
+
+ Object configValue;
+ try {
+ configValue = configField.get(configuration);
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005,
+ "Issue with field " + configName, e);
+ }
+
+ for(Field inputField : configField.getType().getDeclaredFields()) {
+ inputField.setAccessible(true);
+ String inputName = inputField.getName();
+
+ Input inputAnnotation = inputField.getAnnotation(Input.class);
+
+ if(inputAnnotation == null || jsonInputs.get(inputName) == null) {
+ try {
+ inputField.set(configValue, null);
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005,
+ "Issue with field " + configName + "." + inputName, e);
+ }
+ continue;
+ }
+
+ Class type = inputField.getType();
+
+ try {
+ if(type == String.class) {
+ inputField.set(configValue, jsonInputs.get(inputName));
+ } else if (type.isAssignableFrom(Map.class)) {
+ Map<String, String> map = new HashMap<String, String>();
+ JSONObject jsonObject = (JSONObject) jsonInputs.get(inputName);
+ for(Object key : jsonObject.keySet()) {
+ map.put((String)key, (String)jsonObject.get(key));
+ }
+ inputField.set(configValue, map);
+ } else if(type == Integer.class) {
+ inputField.set(configValue, ((Long)jsonInputs.get(inputName)).intValue());
+ } else if(type.isEnum()) {
+ inputField.set(configValue, Enum.valueOf((Class<? extends Enum>) inputField.getType(), (String) jsonInputs.get(inputName)));
+ } else if(type == Boolean.class) {
+ inputField.set(configValue, (Boolean) jsonInputs.get(inputName));
+ }else {
+ throw new SqoopException(ModelError.MODEL_004,
+ "Unsupported type " + type.getName() + " for input " + configName + "." + inputName);
+ }
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_005,
+ "Issue with field " + configName + "." + inputName, e);
+ }
+ }
+ }
+ }
+
+ private static String getFormName(Field member, Config annotation, Set<String> existingFormNames) {
+ if (StringUtils.isEmpty(annotation.name())) {
+ return member.getName();
+ } else {
+ checkForValidFormName(existingFormNames, annotation.name());
+ existingFormNames.add(annotation.name());
+ return annotation.name();
+ }
+ }
+
+ private static void checkForValidFormName(Set<String> existingFormNames,
+ String customFormName) {
+ // uniqueness across fields check
+ if (existingFormNames.contains(customFormName)) {
+ throw new SqoopException(ModelError.MODEL_012,
+ "Issue with field form name " + customFormName);
+ }
+
+ if (!Character.isJavaIdentifierStart(customFormName.toCharArray()[0])) {
+ throw new SqoopException(ModelError.MODEL_013,
+ "Issue with field form name " + customFormName);
+ }
+ for (Character c : customFormName.toCharArray()) {
+ if (Character.isJavaIdentifierPart(c))
+ continue;
+ throw new SqoopException(ModelError.MODEL_013,
+ "Issue with field form name " + customFormName);
+ }
+
+ if (customFormName.length() > 30) {
+ throw new SqoopException(ModelError.MODEL_014,
+ "Issue with field form name " + customFormName);
+
+ }
+ }
+
+ public static String getName(Field input, Input annotation) {
+ return input.getName();
+ }
+
+ public static String getName(Field config, Config annotation) {
+ return config.getName();
+ }
+
+ public static ConfigurationClass getConfigurationClassAnnotation(Object object, boolean strict) {
+ ConfigurationClass annotation = object.getClass().getAnnotation(ConfigurationClass.class);
+
+ if(strict && annotation == null) {
+ throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationGroupClass on class " + object.getClass().getName());
+ }
+
+ return annotation;
+ }
+
+ public static ConfigClass getConfigClassAnnotation(Object object, boolean strict) {
+ ConfigClass annotation = object.getClass().getAnnotation(ConfigClass.class);
+
+ if(strict && annotation == null) {
+ throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationGroupClass on class " + object.getClass().getName());
+ }
+
+ return annotation;
+ }
+
+ public static Config getConfigAnnotation(Field field, boolean strict) {
+ Config annotation = field.getAnnotation(Config.class);
+
+ if(strict && annotation == null) {
+ throw new SqoopException(ModelError.MODEL_003, "Missing annotation Config on Field " + field.getName() + " on class " + field.getDeclaringClass().getName());
+ }
+
+ return annotation;
+ }
+
+ public static Input getInputAnnotation(Field field, boolean strict) {
+ Input annotation = field.getAnnotation(Input.class);
+
+ if(strict && annotation == null) {
+ throw new SqoopException(ModelError.MODEL_003, "Missing annotation Input on Field " + field.getName() + " on class " + field.getDeclaringClass().getName());
+ }
+
+ return annotation;
+ }
+
+ public static Object getFieldValue(Field field, Object object) {
+ try {
+ field.setAccessible(true);
+ return field.get(object);
+ } catch (IllegalAccessException e) {
+ throw new SqoopException(ModelError.MODEL_015, e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java b/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
index 73374d8..c65c478 100644
--- a/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
+++ b/common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
@@ -23,15 +23,16 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
- * Class annotation. Each class that is used a configuration object where user
- * is expected to provide input need to have this annotation.
+ * Class annotation to represent configuration for the connectors
+ * Each class that is used a configuration group object, the connector developer
+ * is expected to provide the inputs needed for this annotation
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface ConfigurationClass {
/**
- * List of validators associated with this Configuration class.
+ * List of validators associated with this Configuration group class.
*
* @return
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/Form.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/Form.java b/common/src/main/java/org/apache/sqoop/model/Form.java
deleted file mode 100644
index 27a0831..0000000
--- a/common/src/main/java/org/apache/sqoop/model/Form.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-
-/**
- * Denote form in Configuration class
- */
-@Retention(RetentionPolicy.RUNTIME)
-public @interface Form {
-
- /**
- * Optional name for the form object
- *
- * @return
- */
- String name() default "";
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/FormClass.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/FormClass.java b/common/src/main/java/org/apache/sqoop/model/FormClass.java
deleted file mode 100644
index 6048d03..0000000
--- a/common/src/main/java/org/apache/sqoop/model/FormClass.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * Denote configuration class
- */
-@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.TYPE)
-public @interface FormClass {
-
- /**
- * Default size for Inputs in this form.
- *
- * @return
- */
- short defaultSize() default -1;
-
- /**
- * List of validators associated with this form.
- *
- * @return
- */
- Validator[] validators() default {};
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/FormUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/FormUtils.java b/common/src/main/java/org/apache/sqoop/model/FormUtils.java
deleted file mode 100644
index 586e0fe..0000000
--- a/common/src/main/java/org/apache/sqoop/model/FormUtils.java
+++ /dev/null
@@ -1,634 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.utils.ClassUtils;
-import org.apache.sqoop.validation.Message;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
-import org.apache.sqoop.validation.ValidationResult;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-import java.lang.reflect.Field;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-
-/**
- * Util class for transforming data from correctly annotated configuration
- * objects to different structures and vice-versa.
- *
- * TODO: This class should see some overhaul into more reusable code, especially expose and re-use the methods at the end.
- */
-public class FormUtils {
-
- /**
- * Transform correctly annotated configuration object to corresponding
- * list of forms.
- *
- * Forms will be order according to the occurrence in the configuration
- * class. Inputs will be also ordered based on occurrence.
- *
- * @param configuration Annotated arbitrary configuration object
- * @return Corresponding list of forms
- */
- public static List<MForm> toForms(Object configuration) {
- return toForms(configuration.getClass(), configuration);
- }
-
- public static List<MForm> toForms(Class klass) {
- return toForms(klass, null);
- }
-
- @SuppressWarnings("unchecked")
- public static List<MForm> toForms(Class klass, Object configuration) {
-
- Set<String> formNames = new HashSet<String>();
-
- ConfigurationClass global =
- (ConfigurationClass)klass.getAnnotation(ConfigurationClass.class);
-
- // Each configuration object must have this class annotation
- if(global == null) {
- throw new SqoopException(ModelError.MODEL_003,
- "Missing annotation ConfigurationClass on class " + klass.getName());
- }
-
- List<MForm> forms = new LinkedList<MForm>();
-
- // Iterate over all declared fields
- for (Field formField : klass.getDeclaredFields()) {
- formField.setAccessible(true);
-
- // Each field that should be part of user input should have Input
- // annotation.
- Form formAnnotation = formField.getAnnotation(Form.class);
-
- if (formAnnotation != null) {
- String formName = getFormName(formField, formAnnotation, formNames);
-
- Class type = formField.getType();
-
- Object value = null;
- if(configuration != null) {
- try {
- value = formField.get(configuration);
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005,
- "Can't retrieve value from " + formField.getName(), e);
- }
- }
-
- forms.add(toForm(formName, type, value));
- }
- }
-
- return forms;
- }
-
- @SuppressWarnings("unchecked")
- private static MForm toForm(String formName, Class<?> klass, Object object) {
- FormClass global =
- (FormClass)klass.getAnnotation(FormClass.class);
-
- // Each configuration object must have this class annotation
- if(global == null) {
- throw new SqoopException(ModelError.MODEL_003,
- "Missing annotation FormClass on class " + klass.getName());
- }
-
- // Intermediate list of inputs
- List<MInput<?>> inputs = new LinkedList<MInput<?>>();
-
- // Iterate over all declared fields
- for (Field field : klass.getDeclaredFields()) {
- field.setAccessible(true);
-
- String fieldName = field.getName();
- String inputName = formName + "." + fieldName;
-
- // Each field that should be part of user input should have Input
- // annotation.
- Input inputAnnotation = field.getAnnotation(Input.class);
-
- if(inputAnnotation != null) {
- boolean sensitive = inputAnnotation.sensitive();
- short maxLen = inputAnnotation.size();
- Class<?> type = field.getType();
-
- MInput input;
-
- // We need to support NULL, so we do not support primitive types
- if(type.isPrimitive()) {
- throw new SqoopException(ModelError.MODEL_007,
- "Detected primitive type " + type + " for field " + fieldName);
- }
-
- // Instantiate corresponding MInput<?> structure
- if (type == String.class) {
- input = new MStringInput(inputName, sensitive, maxLen);
- } else if (type.isAssignableFrom(Map.class)) {
- input = new MMapInput(inputName, sensitive);
- } else if (type == Integer.class) {
- input = new MIntegerInput(inputName, sensitive);
- } else if (type == Boolean.class) {
- input = new MBooleanInput(inputName, sensitive);
- } else if (type.isEnum()) {
- input = new MEnumInput(inputName, sensitive,
- ClassUtils.getEnumStrings(type));
- } else {
- throw new SqoopException(ModelError.MODEL_004, "Unsupported type "
- + type.getName() + " for input " + fieldName);
- }
-
- // Move value if it's present in original configuration object
- if(object != null) {
- Object value;
- try {
- value = field.get(object);
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005,
- "Can't retrieve value from " + field.getName(), e);
- }
- if(value == null) {
- input.setEmpty();
- } else {
- input.setValue(value);
- }
- }
-
- inputs.add(input);
- }
- }
-
- return new MForm(formName, inputs);
- }
-
- private static Field getFieldFromName(Class<?> klass, String name) {
- Field formField;
- try {
- formField = klass.getDeclaredField(name);
- } catch (NoSuchFieldException e) {
- // reverse lookup form field from custom form name
- if (name != null) {
- for (Field field : klass.getDeclaredFields()) {
- Form formAnnotation = field.getAnnotation(Form.class);
- if (formAnnotation == null) {
- continue;
- }
- if (!StringUtils.isEmpty(formAnnotation.name()) && name.equals(formAnnotation.name())) {
- return field;
- }
- }
- }
- throw new SqoopException(ModelError.MODEL_006, "Missing field " + name + " on form class "
- + klass.getCanonicalName(), e);
- }
- return formField;
- }
-
- /**
- * Move form values from form list into corresponding configuration object.
- *
- * @param forms
- * Input form list
- * @param configuration
- * Output configuration object
- */
- public static void fromForms(List<MForm> forms, Object configuration) {
- Class<?> klass = configuration.getClass();
-
- for (MForm form : forms) {
- Field formField = getFieldFromName(klass, form.getName());
- // We need to access this field even if it would be declared as private
- formField.setAccessible(true);
- Class<?> formClass = formField.getType();
- Object newValue = ClassUtils.instantiate(formClass);
-
- if (newValue == null) {
- throw new SqoopException(ModelError.MODEL_006,
- "Can't instantiate new form " + formClass);
- }
-
- for (MInput input : form.getInputs()) {
- String[] splitNames = input.getName().split("\\.");
- if (splitNames.length != 2) {
- throw new SqoopException(ModelError.MODEL_009, "Invalid name: "
- + input.getName());
- }
-
- String inputName = splitNames[1];
- // TODO(jarcec): Names structures fix, handle error cases
- Field inputField;
- try {
- inputField = formClass.getDeclaredField(inputName);
- } catch (NoSuchFieldException e) {
- throw new SqoopException(ModelError.MODEL_006, "Missing field "
- + input.getName(), e);
- }
-
- // We need to access this field even if it would be declared as private
- inputField.setAccessible(true);
-
- try {
- if (input.isEmpty()) {
- inputField.set(newValue, null);
- } else {
- if (input.getType() == MInputType.ENUM) {
- inputField.set(newValue, Enum.valueOf(
- (Class<? extends Enum>) inputField.getType(),
- (String) input.getValue()));
- } else {
- inputField.set(newValue, input.getValue());
- }
- }
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005, "Issue with field "
- + inputField.getName(), e);
- }
- }
-
- try {
- formField.set(configuration, newValue);
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005, "Issue with field "
- + formField.getName(), e);
- }
- }
- }
-
- /**
- * Apply validations on the forms.
- *
- * @param forms
- * Forms that should be updated
- * @param validation
- * Validation that we should apply
- */
- public static void applyValidation(List<MForm> forms, Validation validation) {
- Map<Validation.FormInput, Validation.Message> messages = validation
- .getMessages();
-
- for (MForm form : forms) {
- applyValidation(form, messages);
-
- for (MInput input : form.getInputs()) {
- applyValidation(input, messages);
- }
- }
- }
-
- /**
- * Apply validation on given validated element.
- *
- * @param element
- * Element on what we're applying the validations
- * @param messages
- * Map of all validation messages
- */
- public static void applyValidation(MValidatedElement element,
- Map<Validation.FormInput, Validation.Message> messages) {
- Validation.FormInput name = new Validation.FormInput(element.getName());
-
- if (messages.containsKey(name)) {
- Validation.Message message = messages.get(name);
- element.addValidationMessage(new Message(message.getStatus(), message.getMessage()));
- } else {
- element.addValidationMessage(new Message(Status.getDefault(), null));
- }
- }
-
-
- /**
- * Apply given validations on list of forms.
- *
- * @param forms
- * @param result
- */
- public static void applyValidation(List<MForm> forms, ValidationResult result) {
- for(MForm form : forms) {
- applyValidation(form, result);
-
- for(MInput input : form.getInputs()) {
- applyValidation(input, result);
- }
- }
- }
-
- /**
- * Apply validation messages on given element.
- *
- * Element's state will be set to default if there are no associated messages.
- *
- * @param element
- * @param result
- */
- public static void applyValidation(MValidatedElement element, ValidationResult result) {
- List<Message> messages = result.getMessages().get(element.getName());
-
- if(messages != null) {
- element.setValidationMessages(messages);
- } else {
- element.resetValidationMessages();
- }
- }
-
- /**
- * Convert configuration object to JSON. Only filled properties are serialized,
- * properties with null value are skipped.
- *
- * @param configuration Correctly annotated configuration object
- * @return String of JSON representation
- */
- @SuppressWarnings("unchecked")
- public static String toJson(Object configuration) {
- Class klass = configuration.getClass();
- Set<String> formNames = new HashSet<String>();
- ConfigurationClass global = (ConfigurationClass) klass
- .getAnnotation(ConfigurationClass.class);
-
- // Each configuration object must have this class annotation
- if (global == null) {
- throw new SqoopException(ModelError.MODEL_003,
- "Missing annotation Configuration on class " + klass.getName());
- }
-
- JSONObject jsonOutput = new JSONObject();
-
- // Iterate over all declared fields
- for (Field formField : klass.getDeclaredFields()) {
- formField.setAccessible(true);
-
- // We're processing only form validations
- Form formAnnotation = formField.getAnnotation(Form.class);
- if (formAnnotation == null) {
- continue;
- }
- String formName = getFormName(formField, formAnnotation, formNames);
-
- Object formValue;
- try {
- formValue = formField.get(configuration);
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005, "Issue with field "
- + formName, e);
- }
-
- JSONObject jsonForm = new JSONObject();
-
- // Now process each input on the form
- for (Field inputField : formField.getType().getDeclaredFields()) {
- inputField.setAccessible(true);
- String inputName = inputField.getName();
-
- Object value;
- try {
- value = inputField.get(formValue);
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005, "Issue with field "
- + formName + "." + inputName, e);
- }
-
- Input inputAnnotation = inputField.getAnnotation(Input.class);
-
- // Do not serialize all values
- if (inputAnnotation != null && value != null) {
- Class<?> type = inputField.getType();
-
- // We need to support NULL, so we do not support primitive types
- if (type.isPrimitive()) {
- throw new SqoopException(ModelError.MODEL_007,
- "Detected primitive type " + type + " for field " + formName
- + "." + inputName);
- }
-
- if (type == String.class) {
- jsonForm.put(inputName, value);
- } else if (type.isAssignableFrom(Map.class)) {
- JSONObject map = new JSONObject();
- for (Object key : ((Map) value).keySet()) {
- map.put(key, ((Map) value).get(key));
- }
- jsonForm.put(inputName, map);
- } else if (type == Integer.class) {
- jsonForm.put(inputName, value);
- } else if (type.isEnum()) {
- jsonForm.put(inputName, value.toString());
- } else if (type == Boolean.class) {
- jsonForm.put(inputName, value);
- } else {
- throw new SqoopException(ModelError.MODEL_004, "Unsupported type "
- + type.getName() + " for input " + formName + "." + inputName);
- }
- }
- }
- jsonOutput.put(formName, jsonForm);
- }
- return jsonOutput.toJSONString();
- }
-
- /**
- * Parse given input JSON string and move it's values to given configuration
- * object.
- *
- * @param json JSON representation of the configuration object
- * @param configuration Configuration object to be filled
- */
- public static void fillValues(String json, Object configuration) {
- Class klass = configuration.getClass();
-
- JSONObject jsonForms = (JSONObject) JSONValue.parse(json);
- Set<String> formNames = new HashSet<String>();
-
- for(Field formField : klass.getDeclaredFields()) {
- formField.setAccessible(true);
-
- // We're processing only form validations
- Form formAnnotation = formField.getAnnotation(Form.class);
- if(formAnnotation == null) {
- continue;
- }
- String formName = getFormName(formField, formAnnotation, formNames);
-
- try {
- formField.set(configuration, formField.getType().newInstance());
- } catch (Exception e) {
- throw new SqoopException(ModelError.MODEL_005,
- "Issue with field " + formName, e);
- }
-
- JSONObject jsonInputs = (JSONObject) jsonForms.get(formName);
- if(jsonInputs == null) {
- continue;
- }
-
- Object formValue;
- try {
- formValue = formField.get(configuration);
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005,
- "Issue with field " + formName, e);
- }
-
- for(Field inputField : formField.getType().getDeclaredFields()) {
- inputField.setAccessible(true);
- String inputName = inputField.getName();
-
- Input inputAnnotation = inputField.getAnnotation(Input.class);
-
- if(inputAnnotation == null || jsonInputs.get(inputName) == null) {
- try {
- inputField.set(formValue, null);
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005,
- "Issue with field " + formName + "." + inputName, e);
- }
- continue;
- }
-
- Class type = inputField.getType();
-
- try {
- if(type == String.class) {
- inputField.set(formValue, jsonInputs.get(inputName));
- } else if (type.isAssignableFrom(Map.class)) {
- Map<String, String> map = new HashMap<String, String>();
- JSONObject jsonObject = (JSONObject) jsonInputs.get(inputName);
- for(Object key : jsonObject.keySet()) {
- map.put((String)key, (String)jsonObject.get(key));
- }
- inputField.set(formValue, map);
- } else if(type == Integer.class) {
- inputField.set(formValue, ((Long)jsonInputs.get(inputName)).intValue());
- } else if(type.isEnum()) {
- inputField.set(formValue, Enum.valueOf((Class<? extends Enum>) inputField.getType(), (String) jsonInputs.get(inputName)));
- } else if(type == Boolean.class) {
- inputField.set(formValue, (Boolean) jsonInputs.get(inputName));
- }else {
- throw new SqoopException(ModelError.MODEL_004,
- "Unsupported type " + type.getName() + " for input " + formName + "." + inputName);
- }
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_005,
- "Issue with field " + formName + "." + inputName, e);
- }
- }
- }
- }
-
- private static String getFormName(Field member, Form annotation, Set<String> existingFormNames) {
- if (StringUtils.isEmpty(annotation.name())) {
- return member.getName();
- } else {
- checkForValidFormName(existingFormNames, annotation.name());
- existingFormNames.add(annotation.name());
- return annotation.name();
- }
- }
-
- private static void checkForValidFormName(Set<String> existingFormNames,
- String customFormName) {
- // uniqueness across fields check
- if (existingFormNames.contains(customFormName)) {
- throw new SqoopException(ModelError.MODEL_012,
- "Issue with field form name " + customFormName);
- }
-
- if (!Character.isJavaIdentifierStart(customFormName.toCharArray()[0])) {
- throw new SqoopException(ModelError.MODEL_013,
- "Issue with field form name " + customFormName);
- }
- for (Character c : customFormName.toCharArray()) {
- if (Character.isJavaIdentifierPart(c))
- continue;
- throw new SqoopException(ModelError.MODEL_013,
- "Issue with field form name " + customFormName);
- }
-
- if (customFormName.length() > 30) {
- throw new SqoopException(ModelError.MODEL_014,
- "Issue with field form name " + customFormName);
-
- }
- }
-
- public static String getName(Field input, Input annotation) {
- return input.getName();
- }
-
- public static String getName(Field form, Form annotation) {
- return form.getName();
- }
-
- public static ConfigurationClass getConfigurationClassAnnotation(Object object, boolean strict) {
- ConfigurationClass annotation = object.getClass().getAnnotation(ConfigurationClass.class);
-
- if(strict && annotation == null) {
- throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationClass on class " + object.getClass().getName());
- }
-
- return annotation;
- }
-
- public static FormClass getFormClassAnnotation(Object object, boolean strict) {
- FormClass annotation = object.getClass().getAnnotation(FormClass.class);
-
- if(strict && annotation == null) {
- throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationClass on class " + object.getClass().getName());
- }
-
- return annotation;
- }
-
- public static Form getFormAnnotation(Field field, boolean strict) {
- Form annotation = field.getAnnotation(Form.class);
-
- if(strict && annotation == null) {
- throw new SqoopException(ModelError.MODEL_003, "Missing annotation Form on Field " + field.getName() + " on class " + field.getDeclaringClass().getName());
- }
-
- return annotation;
- }
-
- public static Input getInputAnnotation(Field field, boolean strict) {
- Input annotation = field.getAnnotation(Input.class);
-
- if(strict && annotation == null) {
- throw new SqoopException(ModelError.MODEL_003, "Missing annotation Input on Field " + field.getName() + " on class " + field.getDeclaringClass().getName());
- }
-
- return annotation;
- }
-
- public static Object getFieldValue(Field field, Object object) {
- try {
- field.setAccessible(true);
- return field.get(object);
- } catch (IllegalAccessException e) {
- throw new SqoopException(ModelError.MODEL_015, e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConfig.java b/common/src/main/java/org/apache/sqoop/model/MConfig.java
new file mode 100644
index 0000000..b5d2afd
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MConfig.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import org.apache.sqoop.common.SqoopException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents a group of inputs that are processed together. This allows the
+ * input gathering process to be broken down into multiple steps that can be
+ * then paged through by the user interface.
+ */
+public final class MConfig extends MValidatedElement implements MClonable {
+
+ private final List<MInput<?>> inputs;
+
+ public MConfig(String name, List<MInput<?>> inputs) {
+ super(name);
+
+ this.inputs = inputs;
+ }
+
+ public List<MInput<?>> getInputs() {
+ return inputs;
+ }
+
+ public MInput<?> getInput(String inputName) {
+ for(MInput<?> input: inputs) {
+ if(inputName.equals(input.getName())) {
+ return input;
+ }
+ }
+
+ throw new SqoopException(ModelError.MODEL_011, "Input name: " + inputName);
+ }
+
+ public MStringInput getStringInput(String inputName) {
+ return (MStringInput)getInput(inputName);
+ }
+
+ public MEnumInput getEnumInput(String inputName) {
+ return (MEnumInput)getInput(inputName);
+ }
+
+ public MIntegerInput getIntegerInput(String inputName) {
+ return (MIntegerInput)getInput(inputName);
+ }
+
+ public MBooleanInput getBooleanInput(String inputName) {
+ return (MBooleanInput)getInput(inputName);
+ }
+
+ public MMapInput getMapInput(String inputName) {
+ return (MMapInput)getInput(inputName);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("config-").append(getName());
+ sb.append(":").append(getPersistenceId()).append(":").append(inputs);
+
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (!(other instanceof MConfig)) {
+ return false;
+ }
+
+ MConfig mf = (MConfig) other;
+ return getName().equals(mf.getName())
+ && inputs.equals(mf.inputs);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ result = 31 * result + getName().hashCode();
+ for (MInput<?> mi : inputs) {
+ result = 31 * result + mi.hashCode();
+ }
+
+ return result;
+ }
+
+ @Override
+ public MConfig clone(boolean cloneWithValue) {
+ List<MInput<?>> copyInputs = new ArrayList<MInput<?>>();
+ for(MInput<?> itr : this.getInputs()) {
+ copyInputs.add((MInput<?>)itr.clone(cloneWithValue));
+ }
+ MConfig copyConfig = new MConfig(this.getName(), copyInputs);
+ return copyConfig;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MConfigList.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConfigList.java b/common/src/main/java/org/apache/sqoop/model/MConfigList.java
new file mode 100644
index 0000000..8747b55
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MConfigList.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import org.apache.sqoop.common.SqoopException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Arbitrary list of config objects.
+ */
+public class MConfigList implements MClonable {
+
+ private final List<MConfig> configObjects;
+
+ public MConfigList(List<MConfig> configObjects) {
+ this.configObjects = configObjects;
+ }
+
+ public List<MConfig> getConfigs() {
+ return configObjects;
+ }
+
+ public MConfig getConfig(String configName) {
+ for(MConfig config: configObjects) {
+ if(configName.equals(config.getName())) {
+ return config;
+ }
+ }
+
+ throw new SqoopException(ModelError.MODEL_010, "config name: " + configName);
+ }
+
+ public MInput getInput(String name) {
+ String []parts = name.split("\\.");
+ if(parts.length != 2) {
+ throw new SqoopException(ModelError.MODEL_009, name);
+ }
+
+ return getConfig(parts[0]).getInput(name);
+ }
+
+ public MStringInput getStringInput(String name) {
+ return (MStringInput)getInput(name);
+ }
+
+ public MEnumInput getEnumInput(String name) {
+ return (MEnumInput)getInput(name);
+ }
+
+ public MIntegerInput getIntegerInput(String name) {
+ return (MIntegerInput)getInput(name);
+ }
+
+ public MMapInput getMapInput(String name) {
+ return (MMapInput)getInput(name);
+ }
+
+ public MBooleanInput getBooleanInput(String name) {
+ return (MBooleanInput)getInput(name);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof MConfigList)) return false;
+
+ MConfigList mConfigList = (MConfigList) o;
+
+ if (!configObjects.equals(mConfigList.configObjects)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ for(MConfig config : configObjects) {
+ result = 31 * result + config.hashCode();
+ }
+
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("Configs: ");
+ for(MConfig config : configObjects) {
+ sb.append(config.toString());
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public MConfigList clone(boolean cloneWithValue) {
+ List<MConfig> copyConfigs = null;
+ if(this.getConfigs() != null) {
+ copyConfigs = new ArrayList<MConfig>();
+ for(MConfig itr : this.getConfigs()) {
+ MConfig newConfig = itr.clone(cloneWithValue);
+ newConfig.setPersistenceId(itr.getPersistenceId());
+ copyConfigs.add(newConfig);
+ }
+ }
+ MConfigList copyConfigList = new MConfigList(copyConfigs);
+ return copyConfigList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MConfigType.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConfigType.java b/common/src/main/java/org/apache/sqoop/model/MConfigType.java
new file mode 100644
index 0000000..de05332
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MConfigType.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+/**
+ * Represents the various config types supported by the system.
+ */
+public enum MConfigType {
+
+ /** Unknown config type */
+ OTHER,
+
+ /** link config type */
+ LINK,
+
+ /** Job config type */
+ JOB;
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/main/java/org/apache/sqoop/model/MConnectionForms.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConnectionForms.java b/common/src/main/java/org/apache/sqoop/model/MConnectionForms.java
deleted file mode 100644
index 457ccdb..0000000
--- a/common/src/main/java/org/apache/sqoop/model/MConnectionForms.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import java.util.List;
-
-/**
- * Metadata describing all required information to build up an connection
- * object for one part. Both connector and framework need to supply this object
- * to build up entire connection.
- */
-public class MConnectionForms extends MFormList {
-
- public MConnectionForms(List<MForm> forms) {
- super(forms);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("Connection: ");
- sb.append(super.toString());
- return sb.toString();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == this) {
- return true;
- }
-
- return super.equals(other);
- }
-
- @Override
- public MConnectionForms clone(boolean cloneWithValue) {
- MConnectionForms copy = new MConnectionForms(super.clone(cloneWithValue).getForms());
- return copy;
- }
-}
[19/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
----------------------------------------------------------------------
diff --git a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
index 731f6cc..54ea3d2 100644
--- a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
+++ b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.client;
-import org.apache.sqoop.client.request.SqoopRequests;
+import org.apache.sqoop.client.request.SqoopResourceRequests;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.json.ConnectorBean;
-import org.apache.sqoop.json.FrameworkBean;
+import org.apache.sqoop.json.DriverConfigBean;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.utils.MapResourceBundle;
import org.junit.Before;
@@ -32,7 +32,6 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
@@ -43,14 +42,14 @@ import static org.mockito.Mockito.*;
public class TestSqoopClient {
- SqoopRequests requests;
+ SqoopResourceRequests resourceRequests;
SqoopClient client;
@Before
public void setUp() {
- requests = mock(SqoopRequests.class);
+ resourceRequests = mock(SqoopResourceRequests.class);
client = new SqoopClient("my-cool-server");
- client.setSqoopRequests(requests);
+ client.setSqoopRequests(resourceRequests);
}
/**
@@ -59,26 +58,26 @@ public class TestSqoopClient {
*/
@Test
public void testGetConnector() {
- when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
+ when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
MConnector connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
- client.getResourceBundle(1L);
+ client.getConnectorConfigResourceBundle(1L);
- verify(requests, times(1)).readConnector(1L);
+ verify(resourceRequests, times(1)).readConnector(1L);
}
@Test
public void testGetConnectorByString() {
- when(requests.readConnector(null)).thenReturn(connectorBean(connector(1)));
+ when(resourceRequests.readConnector(null)).thenReturn(connectorBean(connector(1)));
MConnector connector = client.getConnector("A1");
assertEquals(1, connector.getPersistenceId());
assertEquals("A1", connector.getUniqueName());
- client.getResourceBundle(1L);
+ client.getConnectorConfigResourceBundle(1L);
- verify(requests, times(0)).readConnector(1L);
- verify(requests, times(1)).readConnector(null);
+ verify(resourceRequests, times(0)).readConnector(1L);
+ verify(resourceRequests, times(1)).readConnector(null);
}
/**
@@ -87,41 +86,41 @@ public class TestSqoopClient {
*/
@Test
public void testGetConnectorBundle() {
- when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
- client.getResourceBundle(1L);
+ when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
+ client.getConnectorConfigResourceBundle(1L);
MConnector connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
- verify(requests, times(1)).readConnector(1L);
+ verify(resourceRequests, times(1)).readConnector(1L);
}
/**
- * Retrieve framework information, request to framework bundle should not
+ * Retrieve driverConfig information, request to driverConfig bundle should not
* require additional HTTP request.
*/
@Test
- public void testGetFramework() {
- when(requests.readFramework()).thenReturn(frameworkBean(framework()));
+ public void testGetDriverConfig() {
+ when(resourceRequests.readDriverConfig()).thenReturn(driverConfigBean(driverConfig()));
- client.getFramework();
- client.getFrameworkResourceBundle();
+ client.getDriverConfig();
+ client.getDriverConfigBundle();
- verify(requests, times(1)).readFramework();
+ verify(resourceRequests, times(1)).readDriverConfig();
}
/**
- * Retrieve framework bundle, request to framework metadata should not
+ * Retrieve driverConfig bundle, request to driverConfig metadata should not
* require additional HTTP request.
*/
@Test
- public void testGetFrameworkBundle() {
- when(requests.readFramework()).thenReturn(frameworkBean(framework()));
+ public void testGetDriverConfigBundle() {
+ when(resourceRequests.readDriverConfig()).thenReturn(driverConfigBean(driverConfig()));
- client.getFrameworkResourceBundle();
- client.getFramework();
+ client.getDriverConfigBundle();
+ client.getDriverConfig();
- verify(requests, times(1)).readFramework();
+ verify(resourceRequests, times(1)).readDriverConfig();
}
/**
@@ -132,16 +131,16 @@ public class TestSqoopClient {
public void testGetConnectors() {
MConnector connector;
- when(requests.readConnector(null)).thenReturn(connectorBean(connector(1), connector(2)));
+ when(resourceRequests.readConnector(null)).thenReturn(connectorBean(connector(1), connector(2)));
Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size());
- client.getResourceBundle(1);
+ client.getConnectorConfigResourceBundle(1);
connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
connector = client.getConnector(2);
- client.getResourceBundle(2);
+ client.getConnectorConfigResourceBundle(2);
assertEquals(2, connector.getPersistenceId());
connectors = client.getConnectors();
@@ -158,8 +157,8 @@ public class TestSqoopClient {
connector = client.getConnector("A3");
assertNull(connector);
- verify(requests, times(1)).readConnector(null);
- verifyNoMoreInteractions(requests);
+ verify(resourceRequests, times(1)).readConnector(null);
+ verifyNoMoreInteractions(resourceRequests);
}
@@ -170,32 +169,32 @@ public class TestSqoopClient {
@Test
public void testGetConnectorOneByOne() {
ConnectorBean bean = connectorBean(connector(1), connector(2));
- when(requests.readConnector(null)).thenReturn(bean);
- when(requests.readConnector(1L)).thenReturn(bean);
- when(requests.readConnector(2L)).thenReturn(bean);
+ when(resourceRequests.readConnector(null)).thenReturn(bean);
+ when(resourceRequests.readConnector(1L)).thenReturn(bean);
+ when(resourceRequests.readConnector(2L)).thenReturn(bean);
- client.getResourceBundle(1);
+ client.getConnectorConfigResourceBundle(1);
client.getConnector(1);
client.getConnector(2);
- client.getResourceBundle(2);
+ client.getConnectorConfigResourceBundle(2);
Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size());
- verify(requests, times(1)).readConnector(null);
- verify(requests, times(1)).readConnector(1L);
- verify(requests, times(1)).readConnector(2L);
- verifyNoMoreInteractions(requests);
+ verify(resourceRequests, times(1)).readConnector(null);
+ verify(resourceRequests, times(1)).readConnector(1L);
+ verify(resourceRequests, times(1)).readConnector(2L);
+ verifyNoMoreInteractions(resourceRequests);
}
/**
- * Connection for non-existing connector can't be created.
+ * Link for non-existing connector can't be created.
*/
@Test(expected = SqoopException.class)
- public void testNewConnection() {
- when(requests.readConnector(null)).thenReturn(connectorBean(connector(1)));
- client.newConnection("non existing connector");
+ public void testCreateLink() {
+ when(resourceRequests.readConnector(null)).thenReturn(connectorBean(connector(1)));
+ client.createLink("non existing connector");
}
private ConnectorBean connectorBean(MConnector...connectors) {
@@ -208,8 +207,8 @@ public class TestSqoopClient {
}
return new ConnectorBean(connectorList, bundles);
}
- private FrameworkBean frameworkBean(MFramework framework) {
- return new FrameworkBean(framework, new MapResourceBundle(null));
+ private DriverConfigBean driverConfigBean(MDriverConfig driverConfig) {
+ return new DriverConfigBean(driverConfig, new MapResourceBundle(null));
}
private MConnector connector(long id) {
@@ -219,10 +218,10 @@ public class TestSqoopClient {
return connector;
}
- private MFramework framework() {
- MFramework framework = new MFramework(new MConnectionForms(null),
+ private MDriverConfig driverConfig() {
+ MDriverConfig driverConfig = new MDriverConfig(new MConnectionForms(null),
new MJobForms(null), "1");
- framework.setPersistenceId(1);
- return framework;
+ driverConfig.setPersistenceId(1);
+ return driverConfig;
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/common/SqoopException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/common/SqoopException.java b/common/src/main/java/org/apache/sqoop/common/SqoopException.java
index 98f9dc7..6c5dc2a 100644
--- a/common/src/main/java/org/apache/sqoop/common/SqoopException.java
+++ b/common/src/main/java/org/apache/sqoop/common/SqoopException.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.common;
/**
- * Base exception for Sqoop framework. This exception requires the specification
+ * Base exception for Sqoop driver. This exception requires the specification
* of an error code for reference purposes. Where necessary the appropriate
* constructor can be used to pass in additional message beyond what is
* specified by the error code and/or the causal exception.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/etl/io/DataReader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/etl/io/DataReader.java b/common/src/main/java/org/apache/sqoop/etl/io/DataReader.java
index a34dfb4..a555431 100644
--- a/common/src/main/java/org/apache/sqoop/etl/io/DataReader.java
+++ b/common/src/main/java/org/apache/sqoop/etl/io/DataReader.java
@@ -18,20 +18,20 @@
package org.apache.sqoop.etl.io;
/**
- * An intermediate layer for passing data from the execution framework
- * to the ETL framework.
+ * An intermediate layer for passing data from the execution engine
+ * to the ETL engine.
*/
public abstract class DataReader {
/**
- * Read data from the execution framework as an object array.
+ * Read data from the execution engine as an object array.
* @return - array of objects with each column represented as an object
* @throws Exception
*/
public abstract Object[] readArrayRecord() throws Exception;
/**
- * Read data from execution framework as text - as a CSV record.
+ * Read data from execution engine as text - as a CSV record.
* public abstract Object readContent(int type) throws Exception;
* @return - CSV formatted data.
* @throws Exception
@@ -39,7 +39,7 @@ public abstract class DataReader {
public abstract String readTextRecord() throws Exception;
/**
- * Read data from execution framework as a native format.
+ * Read data from execution engine as a native format.
* @return - the content in the native format of the intermediate data
* format being used.
* @throws Exception
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/job/etl/ActorContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/job/etl/ActorContext.java b/common/src/main/java/org/apache/sqoop/job/etl/ActorContext.java
deleted file mode 100644
index 98b2f5e..0000000
--- a/common/src/main/java/org/apache/sqoop/job/etl/ActorContext.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.job.etl;
-
-import org.apache.sqoop.common.ImmutableContext;
-
-/**
- * Basic context class for each actor containing only the connector/framework
- * context object.
- */
-public abstract class ActorContext {
-
- ImmutableContext context;
-
- public ActorContext(ImmutableContext context) {
- this.context = context;
- }
-
- /**
- * Context object associated with the particular actor
- *
- * @return
- */
- public ImmutableContext getContext() {
- return context;
- }
-
- /**
- * Convenience method that will return value from wrapped context class.
- */
- public String getString(String key) {
- return context.getString(key);
- }
-
- /**
- * Convenience method that will return value from wrapped context class.
- */
- public String getString(String key, String defaultValue) {
- return context.getString(key, defaultValue);
- }
-
- /**
- * Convenience method that will return value from wrapped context class.
- */
- public long getLong(String key, long defaultValue) {
- return context.getLong(key, defaultValue);
- }
-
- /**
- * Convenience method that will return value from wrapped context class.
- */
- public int getInt(String key, int defaultValue) {
- return context.getInt(key, defaultValue);
- }
-
- /**
- * Convenience method that will return value from wrapped context class.
- */
- public boolean getBoolean(String key, boolean defaultValue) {
- return context.getBoolean(key, defaultValue);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/job/etl/DestroyerContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/job/etl/DestroyerContext.java b/common/src/main/java/org/apache/sqoop/job/etl/DestroyerContext.java
index 2f29de4..7afb120 100644
--- a/common/src/main/java/org/apache/sqoop/job/etl/DestroyerContext.java
+++ b/common/src/main/java/org/apache/sqoop/job/etl/DestroyerContext.java
@@ -25,7 +25,7 @@ import org.apache.sqoop.schema.Schema;
*
* This class is wrapping information if the run was successful or not.
*/
-public class DestroyerContext extends ActorContext {
+public class DestroyerContext extends TransferableContext {
private boolean success;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java b/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
index af03f0a..fd73890 100644
--- a/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
+++ b/common/src/main/java/org/apache/sqoop/job/etl/ExtractorContext.java
@@ -26,7 +26,7 @@ import org.apache.sqoop.schema.Schema;
*
* This class is wrapping writer object.
*/
-public class ExtractorContext extends ActorContext {
+public class ExtractorContext extends TransferableContext {
private DataWriter writer;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/job/etl/InitializerContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/job/etl/InitializerContext.java b/common/src/main/java/org/apache/sqoop/job/etl/InitializerContext.java
index d2e2dfc..4de00a6 100644
--- a/common/src/main/java/org/apache/sqoop/job/etl/InitializerContext.java
+++ b/common/src/main/java/org/apache/sqoop/job/etl/InitializerContext.java
@@ -25,7 +25,7 @@ import org.apache.sqoop.common.MutableContext;
*
* This class is returning mutable context instead of immutable.
*/
-public class InitializerContext extends ActorContext {
+public class InitializerContext extends TransferableContext {
public InitializerContext(MutableContext context) {
super(context);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/job/etl/LoaderContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/job/etl/LoaderContext.java b/common/src/main/java/org/apache/sqoop/job/etl/LoaderContext.java
index f2e6b97..563b9ad 100644
--- a/common/src/main/java/org/apache/sqoop/job/etl/LoaderContext.java
+++ b/common/src/main/java/org/apache/sqoop/job/etl/LoaderContext.java
@@ -26,7 +26,7 @@ import org.apache.sqoop.schema.Schema;
*
* This class is also wrapping reader object.
*/
-public class LoaderContext extends ActorContext {
+public class LoaderContext extends TransferableContext {
private DataReader reader;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/job/etl/PartitionerContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/job/etl/PartitionerContext.java b/common/src/main/java/org/apache/sqoop/job/etl/PartitionerContext.java
index e7daeee..79901fd 100644
--- a/common/src/main/java/org/apache/sqoop/job/etl/PartitionerContext.java
+++ b/common/src/main/java/org/apache/sqoop/job/etl/PartitionerContext.java
@@ -25,7 +25,7 @@ import org.apache.sqoop.schema.Schema;
*
* This class is also wrapping number of maximal allowed partitions.
*/
-public class PartitionerContext extends ActorContext {
+public class PartitionerContext extends TransferableContext {
private long maxPartitions;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/job/etl/TransferableContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/job/etl/TransferableContext.java b/common/src/main/java/org/apache/sqoop/job/etl/TransferableContext.java
new file mode 100644
index 0000000..e57bf45
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/job/etl/TransferableContext.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.job.etl;
+
+import org.apache.sqoop.common.ImmutableContext;
+
+/**
+ * Base context class for the {@link Transferable} components
+ */
+public abstract class TransferableContext {
+
+ ImmutableContext context;
+
+ public TransferableContext(ImmutableContext context) {
+ this.context = context;
+ }
+
+ /**
+ * Context object associated with the particular actor
+ *
+ * @return
+ */
+ public ImmutableContext getContext() {
+ return context;
+ }
+
+ /**
+ * Convenience method that will return value from wrapped context class.
+ */
+ public String getString(String key) {
+ return context.getString(key);
+ }
+
+ /**
+ * Convenience method that will return value from wrapped context class.
+ */
+ public String getString(String key, String defaultValue) {
+ return context.getString(key, defaultValue);
+ }
+
+ /**
+ * Convenience method that will return value from wrapped context class.
+ */
+ public long getLong(String key, long defaultValue) {
+ return context.getLong(key, defaultValue);
+ }
+
+ /**
+ * Convenience method that will return value from wrapped context class.
+ */
+ public int getInt(String key, int defaultValue) {
+ return context.getInt(key, defaultValue);
+ }
+
+ /**
+ * Convenience method that will return value from wrapped context class.
+ */
+ public boolean getBoolean(String key, boolean defaultValue) {
+ return context.getBoolean(key, defaultValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/ConnectionBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectionBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectionBean.java
deleted file mode 100644
index 688bb64..0000000
--- a/common/src/main/java/org/apache/sqoop/json/ConnectionBean.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json;
-
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.ResourceBundle;
-import java.util.Set;
-
-import static org.apache.sqoop.json.util.FormSerialization.*;
-import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
-
-/**
- * Connection representation that is being send across the network between
- * Sqoop server and client. Server might optionally send resource bundles
- * associated with the connections to spare client of sending another HTTP
- * requests to obtain them.
- */
-public class ConnectionBean implements JsonBean {
-
- private static final String CONNECTOR_ID = "connector-id";
- private static final String CONNECTOR_PART = "connector";
- private static final String FRAMEWORK_PART = "framework";
-
- // Compulsory
- private List<MConnection> connections;
-
- // Optional
- private Map<Long, ResourceBundle> connectorBundles;
- private ResourceBundle frameworkBundle;
-
- // For "extract"
- public ConnectionBean(MConnection connection) {
- this();
- this.connections = new ArrayList<MConnection>();
- this.connections.add(connection);
- }
-
- public ConnectionBean(List<MConnection> connections) {
- this();
- this.connections = connections;
- }
-
- // For "restore"
- public ConnectionBean() {
- connectorBundles = new HashMap<Long, ResourceBundle>();
- }
-
- public void setFrameworkBundle(ResourceBundle frameworkBundle) {
- this.frameworkBundle = frameworkBundle;
- }
-
- public void addConnectorBundle(Long id, ResourceBundle connectorBundle) {
- connectorBundles.put(id, connectorBundle);
- }
-
- public boolean hasConnectorBundle(Long id) {
- return connectorBundles.containsKey(id);
- }
-
- public List<MConnection> getConnections() {
- return connections;
- }
-
- public ResourceBundle getConnectorBundle(Long id) {
- return connectorBundles.get(id);
- }
-
- public ResourceBundle getFrameworkBundle() {
- return frameworkBundle;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public JSONObject extract(boolean skipSensitive) {
- JSONArray array = new JSONArray();
-
- for(MConnection connection : connections) {
- JSONObject object = new JSONObject();
-
- object.put(ID, connection.getPersistenceId());
- object.put(NAME, connection.getName());
- object.put(ENABLED, connection.getEnabled());
- object.put(CREATION_USER, connection.getCreationUser());
- object.put(CREATION_DATE, connection.getCreationDate().getTime());
- object.put(UPDATE_USER, connection.getLastUpdateUser());
- object.put(UPDATE_DATE, connection.getLastUpdateDate().getTime());
- object.put(CONNECTOR_ID, connection.getConnectorId());
- object.put(CONNECTOR_PART,
- extractForms(connection.getConnectorPart().getForms(), skipSensitive));
- object.put(FRAMEWORK_PART,
- extractForms(connection.getFrameworkPart().getForms(), skipSensitive));
-
- array.add(object);
- }
-
- JSONObject all = new JSONObject();
- all.put(ALL, array);
-
- if(!connectorBundles.isEmpty()) {
- JSONObject bundles = new JSONObject();
-
- for(Map.Entry<Long, ResourceBundle> entry : connectorBundles.entrySet()) {
- bundles.put(entry.getKey().toString(),
- extractResourceBundle(entry.getValue()));
- }
-
- all.put(CONNECTOR_RESOURCES, bundles);
- }
- if(frameworkBundle != null) {
- all.put(FRAMEWORK_RESOURCES,extractResourceBundle(frameworkBundle));
- }
- return all;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void restore(JSONObject jsonObject) {
- connections = new ArrayList<MConnection>();
-
- JSONArray array = (JSONArray) jsonObject.get(ALL);
-
- for (Object obj : array) {
- JSONObject object = (JSONObject) obj;
-
- long connectorId = (Long) object.get(CONNECTOR_ID);
- JSONArray connectorPart = (JSONArray) object.get(CONNECTOR_PART);
- JSONArray frameworkPart = (JSONArray) object.get(FRAMEWORK_PART);
-
- List<MForm> connectorForms = restoreForms(connectorPart);
- List<MForm> frameworkForms = restoreForms(frameworkPart);
-
- MConnection connection = new MConnection(connectorId,
- new MConnectionForms(connectorForms),
- new MConnectionForms(frameworkForms));
-
- connection.setPersistenceId((Long) object.get(ID));
- connection.setName((String) object.get(NAME));
- connection.setEnabled((Boolean) object.get(ENABLED));
- connection.setCreationUser((String) object.get(CREATION_USER));
- connection.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
- connection.setLastUpdateUser((String) object.get(UPDATE_USER));
- connection.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
-
- connections.add(connection);
- }
-
- if(jsonObject.containsKey(CONNECTOR_RESOURCES)) {
- JSONObject bundles = (JSONObject) jsonObject.get(CONNECTOR_RESOURCES);
- Set<Map.Entry<String, JSONObject>> entrySet = bundles.entrySet();
- for (Map.Entry<String, JSONObject> entry : entrySet) {
- connectorBundles.put(Long.parseLong(entry.getKey()),
- restoreResourceBundle(entry.getValue()));
- }
- }
- if(jsonObject.containsKey(FRAMEWORK_RESOURCES)) {
- frameworkBundle = restoreResourceBundle(
- (JSONObject) jsonObject.get(FRAMEWORK_RESOURCES));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/ConnectionValidationBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectionValidationBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectionValidationBean.java
deleted file mode 100644
index ffdd13e..0000000
--- a/common/src/main/java/org/apache/sqoop/json/ConnectionValidationBean.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json;
-
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
-import org.json.simple.JSONObject;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Bean for sending validations across network. This bean will move two
- * validation objects at one time - one for connector and second for framework
- * part of validated entity. Optionally validation bean can also transfer
- * created persistent id in case that new entity was created.
- */
-public class ConnectionValidationBean implements JsonBean {
-
- private static final String ID = "id";
- private static final String FRAMEWORK = "framework";
- private static final String CONNECTOR = "connector";
- private static final String STATUS = "status";
- private static final String MESSAGE = "message";
- private static final String MESSAGES = "messages";
-
- private Long id;
- private Validation connectorValidation;
- private Validation frameworkValidation;
-
- // For "extract"
- public ConnectionValidationBean(Validation connector, Validation framework) {
- this();
-
- this.connectorValidation = connector;
- this.frameworkValidation = framework;
- }
-
- // For "restore"
- public ConnectionValidationBean() {
- id = null;
- }
-
- public Validation getConnectorValidation() {
- return connectorValidation;
- }
-
- public Validation getFrameworkValidation() {
- return frameworkValidation;
- }
-
- public void setId(Long id) {
- this.id = id;
- }
-
- public Long getId() {
- return id;
- }
-
- @SuppressWarnings("unchecked")
- public JSONObject extract(boolean skipSensitive) {
- JSONObject object = new JSONObject();
-
- // Optionally transfer id
- if(id != null) {
- object.put(ID, id);
- }
-
- object.put(CONNECTOR, extractValidation(connectorValidation));
- object.put(FRAMEWORK, extractValidation(frameworkValidation));
-
- return object;
- }
-
- @SuppressWarnings("unchecked")
- private JSONObject extractValidation(Validation validation) {
- JSONObject object = new JSONObject();
-
- object.put(STATUS, validation.getStatus().name());
-
- JSONObject jsonMessages = new JSONObject();
- Map<Validation.FormInput, Validation.Message> messages = validation.getMessages();
-
- for(Map.Entry<Validation.FormInput, Validation.Message> entry : messages.entrySet()) {
- JSONObject jsonEntry = new JSONObject();
- jsonEntry.put(STATUS, entry.getValue().getStatus().name());
- jsonEntry.put(MESSAGE, entry.getValue().getMessage());
- jsonMessages.put(entry.getKey(), jsonEntry);
- }
-
- object.put(MESSAGES, jsonMessages);
-
- return object;
- }
-
- @Override
- public void restore(JSONObject jsonObject) {
- // Optional and accepting NULLs
- id = (Long) jsonObject.get(ID);
-
- connectorValidation = restoreValidation(
- (JSONObject)jsonObject.get(CONNECTOR));
- frameworkValidation = restoreValidation(
- (JSONObject)jsonObject.get(FRAMEWORK));
- }
-
- public Validation restoreValidation(JSONObject jsonObject) {
- JSONObject jsonMessages = (JSONObject) jsonObject.get(MESSAGES);
- Map<Validation.FormInput, Validation.Message> messages
- = new HashMap<Validation.FormInput, Validation.Message>();
-
- for(Object key : jsonMessages.keySet()) {
- JSONObject jsonMessage = (JSONObject) jsonMessages.get(key);
-
- Status status = Status.valueOf((String) jsonMessage.get(STATUS));
- String stringMessage = (String) jsonMessage.get(MESSAGE);
-
- Validation.Message message
- = new Validation.Message(status, stringMessage);
-
- messages.put(new Validation.FormInput((String)key), message);
- }
-
- Status status = Status.valueOf((String) jsonObject.get(STATUS));
-
- return new Validation(status, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
index 5069f0b..465dd69 100644
--- a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
@@ -17,6 +17,19 @@
*/
package org.apache.sqoop.json;
+import static org.apache.sqoop.json.util.FormSerialization.ALL;
+import static org.apache.sqoop.json.util.FormSerialization.CLASS;
+import static org.apache.sqoop.json.util.FormSerialization.CON_FORMS;
+import static org.apache.sqoop.json.util.FormSerialization.ID;
+import static org.apache.sqoop.json.util.FormSerialization.JOB_FORMS;
+import static org.apache.sqoop.json.util.FormSerialization.NAME;
+import static org.apache.sqoop.json.util.FormSerialization.VERSION;
+import static org.apache.sqoop.json.util.FormSerialization.extractForms;
+import static org.apache.sqoop.json.util.FormSerialization.restoreForms;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONNECTOR_CONFIGS;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -26,15 +39,12 @@ import java.util.Set;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MJobForms;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
-import static org.apache.sqoop.json.util.FormSerialization.*;
-import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
-
public class ConnectorBean implements JsonBean {
private List<MConnector> connectors;
@@ -93,8 +103,7 @@ public class ConnectorBean implements JsonBean {
jsonBundles.put(entry.getKey().toString(),
extractResourceBundle(entry.getValue()));
}
-
- all.put(CONNECTOR_RESOURCES, jsonBundles);
+ all.put(CONNECTOR_CONFIGS, jsonBundles);
}
return all;
@@ -116,28 +125,24 @@ public class ConnectorBean implements JsonBean {
String version = (String) object.get(VERSION);
List<MForm> connForms = restoreForms((JSONArray) object.get(CON_FORMS));
-
JSONObject jobJson = (JSONObject) object.get(JOB_FORMS);
JSONArray fromJobJson = (JSONArray)jobJson.get(Direction.FROM.name());
JSONArray toJobJson = (JSONArray)jobJson.get(Direction.TO.name());
- List<MForm> fromJobForms =
- restoreForms(fromJobJson);
- List<MForm> toJobForms =
- restoreForms(toJobJson);
+ List<MForm> fromJobForms = restoreForms(fromJobJson);
+ List<MForm> toJobForms = restoreForms(toJobJson);
MJobForms fromJob = new MJobForms(fromJobForms);
MJobForms toJob = new MJobForms(toJobForms);
MConnectionForms connection = new MConnectionForms(connForms);
-
- MConnector connector = new MConnector(uniqueName, className, version, connection, fromJob, toJob);
+ MConnector connector = new MConnector(uniqueName, className, version, connection, fromJob,
+ toJob);
connector.setPersistenceId(connectorId);
-
connectors.add(connector);
}
- if(jsonObject.containsKey(CONNECTOR_RESOURCES)) {
+ if(jsonObject.containsKey(CONNECTOR_CONFIGS)) {
bundles = new HashMap<Long, ResourceBundle>();
- JSONObject jsonBundles = (JSONObject) jsonObject.get(CONNECTOR_RESOURCES);
+ JSONObject jsonBundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS);
Set<Map.Entry<String, JSONObject>> entrySet = jsonBundles.entrySet();
for (Map.Entry<String, JSONObject> entry : entrySet) {
bundles.put(Long.parseLong(entry.getKey()),
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java b/common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java
new file mode 100644
index 0000000..a2cc8b8
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json;
+
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MJobForms;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+import java.util.List;
+import java.util.ResourceBundle;
+
+import static org.apache.sqoop.json.util.FormSerialization.*;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
+
+public class DriverConfigBean implements JsonBean {
+
+ private MDriverConfig driverConfig;
+
+ private ResourceBundle bundle;
+
+ // for "extract"
+ public DriverConfigBean(MDriverConfig driverConfig, ResourceBundle bundle) {
+ this.driverConfig = driverConfig;
+ this.bundle = bundle;
+ }
+
+ // for "restore"
+ public DriverConfigBean() {
+ }
+
+ public MDriverConfig getDriverConfig() {
+ return driverConfig;
+ }
+
+ public ResourceBundle getResourceBundle() {
+ return bundle;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public JSONObject extract(boolean skipSensitive) {
+ // TODO(Abe): Add From/To connection forms.
+ JSONArray conForms =
+ extractForms(driverConfig.getConnectionForms().getForms(), skipSensitive);
+ JSONArray jobForms = extractForms(driverConfig.getJobForms().getForms(), skipSensitive);
+
+ JSONObject result = new JSONObject();
+ result.put(ID, driverConfig.getPersistenceId());
+ result.put(DRIVER_VERSION, driverConfig.getVersion());
+ result.put(CON_FORMS, conForms);
+ result.put(JOB_FORMS, jobForms);
+ result.put(CONFIGS, extractResourceBundle(bundle));
+ return result;
+ }
+
+ @Override
+ public void restore(JSONObject jsonObject) {
+ long id = (Long) jsonObject.get(ID);
+ String driverVersion = (String) jsonObject.get(DRIVER_VERSION);
+
+ List<MForm> connForms = restoreForms((JSONArray) jsonObject.get(CON_FORMS));
+ List<MForm> jobForms = restoreForms((JSONArray) jsonObject.get(JOB_FORMS));
+
+ // TODO(Abe): Get From/To connection forms.
+ driverConfig = new MDriverConfig(
+ new MConnectionForms(connForms),
+ new MJobForms(jobForms),
+ driverVersion);
+ driverConfig.setPersistenceId(id);
+
+ bundle = restoreResourceBundle((JSONObject) jsonObject.get(CONFIGS));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/FrameworkBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/FrameworkBean.java b/common/src/main/java/org/apache/sqoop/json/FrameworkBean.java
deleted file mode 100644
index abbdcc6..0000000
--- a/common/src/main/java/org/apache/sqoop/json/FrameworkBean.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json;
-
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFramework;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.ResourceBundle;
-import java.util.Set;
-
-import static org.apache.sqoop.json.util.FormSerialization.*;
-import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
-
-/**
- *
- */
-public class FrameworkBean implements JsonBean {
-
-
- private MFramework framework;
-
- private ResourceBundle bundle;
-
- // for "extract"
- public FrameworkBean(MFramework framework, ResourceBundle bundle) {
- this.framework = framework;
- this.bundle = bundle;
- }
-
- // for "restore"
- public FrameworkBean() {
- }
-
- public MFramework getFramework() {
- return framework;
- }
-
- public ResourceBundle getResourceBundle() {
- return bundle;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public JSONObject extract(boolean skipSensitive) {
- // @TODO(Abe): Add From/To connection forms.
- JSONArray conForms =
- extractForms(framework.getConnectionForms().getForms(), skipSensitive);
- JSONArray jobForms = extractForms(framework.getJobForms().getForms(), skipSensitive);
-
- JSONObject result = new JSONObject();
- result.put(ID, framework.getPersistenceId());
- result.put(FRAMEWORK_VERSION, framework.getVersion());
- result.put(CON_FORMS, conForms);
- result.put(JOB_FORMS, jobForms);
- result.put(RESOURCES, extractResourceBundle(bundle));
- return result;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void restore(JSONObject jsonObject) {
- long id = (Long) jsonObject.get(ID);
- String frameworkVersion = (String) jsonObject.get(FRAMEWORK_VERSION);
-
- List<MForm> connForms = restoreForms((JSONArray) jsonObject.get(CON_FORMS));
- List<MForm> jobForms = restoreForms((JSONArray) jsonObject.get(JOB_FORMS));
-
- // @TODO(Abe): Get From/To connection forms.
- framework = new MFramework(
- new MConnectionForms(connForms),
- new MJobForms(jobForms),
- frameworkVersion);
- framework.setPersistenceId(id);
-
- bundle = restoreResourceBundle((JSONObject) jsonObject.get(RESOURCES));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/JobBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/JobBean.java b/common/src/main/java/org/apache/sqoop/json/JobBean.java
index 91edcfc..a3e6071 100644
--- a/common/src/main/java/org/apache/sqoop/json/JobBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/JobBean.java
@@ -43,20 +43,20 @@ public class JobBean implements JsonBean {
private static final String ALL = "all";
private static final String ID = "id";
private static final String NAME = "name";
- private static final String FROM_CONNECTION_ID = "from-connection-id";
- private static final String TO_CONNECTION_ID = "to-connection-id";
+ private static final String FROM_LINK_ID = "from-link-id";
+ private static final String TO_LINK_ID = "to-link-id";
private static final String FROM_CONNECTOR_ID = "from-connector-id";
private static final String TO_CONNECTOR_ID = "to-connector-id";
private static final String FROM_CONNECTOR_PART = "from-connector";
private static final String TO_CONNECTOR_PART = "to-connector";
private static final String FRAMEWORK_PART = "framework";
- // Compulsory
+ // Required
private List<MJob> jobs;
// Optional
- private Map<Long, ResourceBundle> connectorBundles;
- private ResourceBundle frameworkBundle;
+ private Map<Long, ResourceBundle> connectorConfigBundles;
+ private ResourceBundle driverConfigBundle;
// For "extract"
public JobBean(MJob job) {
@@ -72,31 +72,31 @@ public class JobBean implements JsonBean {
// For "restore"
public JobBean() {
- connectorBundles = new HashMap<Long, ResourceBundle>();
+ connectorConfigBundles = new HashMap<Long, ResourceBundle>();
}
- public void setFrameworkBundle(ResourceBundle frameworkBundle) {
- this.frameworkBundle = frameworkBundle;
+ public void setDriverConfigBundle(ResourceBundle driverConfigBundle) {
+ this.driverConfigBundle = driverConfigBundle;
}
- public void addConnectorBundle(Long id, ResourceBundle connectorBundle) {
- connectorBundles.put(id, connectorBundle);
+ public void addConnectorConfigBundle(Long id, ResourceBundle connectorConfigBundle) {
+ connectorConfigBundles.put(id, connectorConfigBundle);
}
- public boolean hasConnectorBundle(Long id) {
- return connectorBundles.containsKey(id);
+ public boolean hasConnectorConfigBundle(Long id) {
+ return connectorConfigBundles.containsKey(id);
}
public List<MJob> getJobs() {
return jobs;
}
- public ResourceBundle getConnectorBundle(Long id) {
- return connectorBundles.get(id);
+ public ResourceBundle getConnectorConfigBundle(Long id) {
+ return connectorConfigBundles.get(id);
}
- public ResourceBundle getFrameworkBundle() {
- return frameworkBundle;
+ public ResourceBundle getDriverConfigBundle() {
+ return driverConfigBundle;
}
@Override
@@ -114,8 +114,8 @@ public class JobBean implements JsonBean {
object.put(CREATION_DATE, job.getCreationDate().getTime());
object.put(UPDATE_USER, job.getLastUpdateUser());
object.put(UPDATE_DATE, job.getLastUpdateDate().getTime());
- object.put(FROM_CONNECTION_ID, job.getConnectionId(Direction.FROM));
- object.put(TO_CONNECTION_ID, job.getConnectionId(Direction.TO));
+ object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
+ object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
object.put(FROM_CONNECTOR_ID, job.getConnectorId(Direction.FROM));
object.put(TO_CONNECTOR_ID, job.getConnectorId(Direction.TO));
object.put(FROM_CONNECTOR_PART,
@@ -131,18 +131,17 @@ public class JobBean implements JsonBean {
JSONObject all = new JSONObject();
all.put(ALL, array);
- if(!connectorBundles.isEmpty()) {
+ if(!connectorConfigBundles.isEmpty()) {
JSONObject bundles = new JSONObject();
- for(Map.Entry<Long, ResourceBundle> entry : connectorBundles.entrySet()) {
+ for(Map.Entry<Long, ResourceBundle> entry : connectorConfigBundles.entrySet()) {
bundles.put(entry.getKey().toString(),
extractResourceBundle(entry.getValue()));
}
-
- all.put(CONNECTOR_RESOURCES, bundles);
+ all.put(CONNECTOR_CONFIGS, bundles);
}
- if(frameworkBundle != null) {
- all.put(FRAMEWORK_RESOURCES,extractResourceBundle(frameworkBundle));
+ if(driverConfigBundle != null) {
+ all.put(DRIVER_CONFIGS,extractResourceBundle(driverConfigBundle));
}
return all;
}
@@ -159,8 +158,8 @@ public class JobBean implements JsonBean {
long fromConnectorId = (Long) object.get(FROM_CONNECTOR_ID);
long toConnectorId = (Long) object.get(TO_CONNECTOR_ID);
- long fromConnectionId = (Long) object.get(FROM_CONNECTION_ID);
- long toConnectionId = (Long) object.get(TO_CONNECTION_ID);
+ long fromConnectionId = (Long) object.get(FROM_LINK_ID);
+ long toConnectionId = (Long) object.get(TO_LINK_ID);
JSONArray fromConnectorPart = (JSONArray) object.get(FROM_CONNECTOR_PART);
JSONArray toConnectorPart = (JSONArray) object.get(TO_CONNECTOR_PART);
JSONArray frameworkPart = (JSONArray) object.get(FRAMEWORK_PART);
@@ -190,17 +189,17 @@ public class JobBean implements JsonBean {
jobs.add(job);
}
- if(jsonObject.containsKey(CONNECTOR_RESOURCES)) {
- JSONObject bundles = (JSONObject) jsonObject.get(CONNECTOR_RESOURCES);
+ if(jsonObject.containsKey(CONNECTOR_CONFIGS)) {
+ JSONObject bundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS);
Set<Map.Entry<String, JSONObject>> entrySet = bundles.entrySet();
for (Map.Entry<String, JSONObject> entry : entrySet) {
- connectorBundles.put(Long.parseLong(entry.getKey()),
+ connectorConfigBundles.put(Long.parseLong(entry.getKey()),
restoreResourceBundle(entry.getValue()));
}
}
- if(jsonObject.containsKey(FRAMEWORK_RESOURCES)) {
- frameworkBundle = restoreResourceBundle(
- (JSONObject) jsonObject.get(FRAMEWORK_RESOURCES));
+ if(jsonObject.containsKey(DRIVER_CONFIGS)) {
+ driverConfigBundle = restoreResourceBundle(
+ (JSONObject) jsonObject.get(DRIVER_CONFIGS));
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/LinkBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/LinkBean.java b/common/src/main/java/org/apache/sqoop/json/LinkBean.java
new file mode 100644
index 0000000..8981ea7
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/json/LinkBean.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json;
+
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MForm;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.ResourceBundle;
+import java.util.Set;
+
+import static org.apache.sqoop.json.util.FormSerialization.*;
+import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
+
+/**
+ * Link representation that is being send across the network between
+ * Sqoop server and client. Server might optionally send configs
+ * associated with the links to spare client of sending another HTTP
+ * requests to obtain them.
+ */
+public class LinkBean implements JsonBean {
+
+ private static final String CONNECTOR_ID = "connector-id";
+ private static final String CONNECTOR_PART = "connector";
+ private static final String FRAMEWORK_PART = "framework";
+
+ // Required
+ private List<MLink> links;
+
+ // Optional
+ private Map<Long, ResourceBundle> connectorConfigBundles;
+ private ResourceBundle driverConfigBundle;
+
+ // For "extract"
+ public LinkBean(MLink link) {
+ this();
+ this.links = new ArrayList<MLink>();
+ this.links.add(link);
+ }
+
+ public LinkBean(List<MLink> links) {
+ this();
+ this.links = links;
+ }
+
+ // For "restore"
+ public LinkBean() {
+ connectorConfigBundles = new HashMap<Long, ResourceBundle>();
+ }
+
+ public void setDriverConfigBundle(ResourceBundle driverConfigBundle) {
+ this.driverConfigBundle = driverConfigBundle;
+ }
+
+ public void addConnectorConfigBundle(Long id, ResourceBundle connectorConfigBundle) {
+ connectorConfigBundles.put(id, connectorConfigBundle);
+ }
+
+ public boolean hasConnectorBundle(Long id) {
+ return connectorConfigBundles.containsKey(id);
+ }
+
+ public List<MLink> getLinks() {
+ return links;
+ }
+
+ public ResourceBundle getConnectorBundle(Long id) {
+ return connectorConfigBundles.get(id);
+ }
+
+ public ResourceBundle getFrameworkBundle() {
+ return driverConfigBundle;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public JSONObject extract(boolean skipSensitive) {
+ JSONArray array = new JSONArray();
+
+ for(MLink link : links) {
+ JSONObject object = new JSONObject();
+
+ object.put(ID, link.getPersistenceId());
+ object.put(NAME, link.getName());
+ object.put(ENABLED, link.getEnabled());
+ object.put(CREATION_USER, link.getCreationUser());
+ object.put(CREATION_DATE, link.getCreationDate().getTime());
+ object.put(UPDATE_USER, link.getLastUpdateUser());
+ object.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
+ object.put(CONNECTOR_ID, link.getConnectorId());
+ object.put(CONNECTOR_PART,
+ extractForms(link.getConnectorPart().getForms(), skipSensitive));
+ object.put(FRAMEWORK_PART,
+ extractForms(link.getFrameworkPart().getForms(), skipSensitive));
+
+ array.add(object);
+ }
+
+ JSONObject all = new JSONObject();
+ all.put(ALL, array);
+
+ if(!connectorConfigBundles.isEmpty()) {
+ JSONObject bundles = new JSONObject();
+
+ for(Map.Entry<Long, ResourceBundle> entry : connectorConfigBundles.entrySet()) {
+ bundles.put(entry.getKey().toString(),
+ extractResourceBundle(entry.getValue()));
+ }
+ all.put(CONNECTOR_CONFIGS, bundles);
+ }
+ if(driverConfigBundle != null) {
+ all.put(DRIVER_CONFIGS,extractResourceBundle(driverConfigBundle));
+ }
+ return all;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void restore(JSONObject jsonObject) {
+ links = new ArrayList<MLink>();
+
+ JSONArray array = (JSONArray) jsonObject.get(ALL);
+
+ for (Object obj : array) {
+ JSONObject object = (JSONObject) obj;
+
+ long connectorId = (Long) object.get(CONNECTOR_ID);
+ JSONArray connectorPart = (JSONArray) object.get(CONNECTOR_PART);
+ JSONArray frameworkPart = (JSONArray) object.get(FRAMEWORK_PART);
+
+ List<MForm> connectorForms = restoreForms(connectorPart);
+ List<MForm> frameworkForms = restoreForms(frameworkPart);
+
+ MLink link = new MLink(connectorId,
+ new MConnectionForms(connectorForms),
+ new MConnectionForms(frameworkForms));
+
+ link.setPersistenceId((Long) object.get(ID));
+ link.setName((String) object.get(NAME));
+ link.setEnabled((Boolean) object.get(ENABLED));
+ link.setCreationUser((String) object.get(CREATION_USER));
+ link.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
+ link.setLastUpdateUser((String) object.get(UPDATE_USER));
+ link.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
+
+ links.add(link);
+ }
+
+ if(jsonObject.containsKey(CONNECTOR_CONFIGS)) {
+ JSONObject bundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS);
+ Set<Map.Entry<String, JSONObject>> entrySet = bundles.entrySet();
+ for (Map.Entry<String, JSONObject> entry : entrySet) {
+ connectorConfigBundles.put(Long.parseLong(entry.getKey()),
+ restoreResourceBundle(entry.getValue()));
+ }
+ }
+ if(jsonObject.containsKey(DRIVER_CONFIGS)) {
+ driverConfigBundle = restoreResourceBundle(
+ (JSONObject) jsonObject.get(DRIVER_CONFIGS));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java b/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java
new file mode 100644
index 0000000..a6a6b65
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json;
+
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.Validation;
+import org.json.simple.JSONObject;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Bean for sending validations across network. This bean will move two
+ * validation objects at one time - one for connector and second for framework
+ * part of validated entity. Optionally validation bean can also transfer
+ * created persistent id in case that new entity was created.
+ */
+public class LinkValidationBean implements JsonBean {
+
+ private static final String ID = "id";
+ private static final String FRAMEWORK = "framework";
+ private static final String CONNECTOR = "connector";
+ private static final String STATUS = "status";
+ private static final String MESSAGE = "message";
+ private static final String MESSAGES = "messages";
+
+ private Long id;
+ private Validation connectorValidation;
+ private Validation frameworkValidation;
+
+ // For "extract"
+ public LinkValidationBean(Validation connector, Validation framework) {
+ this();
+
+ this.connectorValidation = connector;
+ this.frameworkValidation = framework;
+ }
+
+ // For "restore"
+ public LinkValidationBean() {
+ id = null;
+ }
+
+ public Validation getConnectorValidation() {
+ return connectorValidation;
+ }
+
+ public Validation getFrameworkValidation() {
+ return frameworkValidation;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public Long getId() {
+ return id;
+ }
+
+ @SuppressWarnings("unchecked")
+ public JSONObject extract(boolean skipSensitive) {
+ JSONObject object = new JSONObject();
+
+ // Optionally transfer id
+ if(id != null) {
+ object.put(ID, id);
+ }
+
+ object.put(CONNECTOR, extractValidation(connectorValidation));
+ object.put(FRAMEWORK, extractValidation(frameworkValidation));
+
+ return object;
+ }
+
+ @SuppressWarnings("unchecked")
+ private JSONObject extractValidation(Validation validation) {
+ JSONObject object = new JSONObject();
+
+ object.put(STATUS, validation.getStatus().name());
+
+ JSONObject jsonMessages = new JSONObject();
+ Map<Validation.FormInput, Validation.Message> messages = validation.getMessages();
+
+ for(Map.Entry<Validation.FormInput, Validation.Message> entry : messages.entrySet()) {
+ JSONObject jsonEntry = new JSONObject();
+ jsonEntry.put(STATUS, entry.getValue().getStatus().name());
+ jsonEntry.put(MESSAGE, entry.getValue().getMessage());
+ jsonMessages.put(entry.getKey(), jsonEntry);
+ }
+
+ object.put(MESSAGES, jsonMessages);
+
+ return object;
+ }
+
+ @Override
+ public void restore(JSONObject jsonObject) {
+ // Optional and accepting NULLs
+ id = (Long) jsonObject.get(ID);
+
+ connectorValidation = restoreValidation(
+ (JSONObject)jsonObject.get(CONNECTOR));
+ frameworkValidation = restoreValidation(
+ (JSONObject)jsonObject.get(FRAMEWORK));
+ }
+
+ public Validation restoreValidation(JSONObject jsonObject) {
+ JSONObject jsonMessages = (JSONObject) jsonObject.get(MESSAGES);
+ Map<Validation.FormInput, Validation.Message> messages
+ = new HashMap<Validation.FormInput, Validation.Message>();
+
+ for(Object key : jsonMessages.keySet()) {
+ JSONObject jsonMessage = (JSONObject) jsonMessages.get(key);
+
+ Status status = Status.valueOf((String) jsonMessage.get(STATUS));
+ String stringMessage = (String) jsonMessage.get(MESSAGE);
+
+ Validation.Message message
+ = new Validation.Message(status, stringMessage);
+
+ messages.put(new Validation.FormInput((String)key), message);
+ }
+
+ Status status = Status.valueOf((String) jsonObject.get(STATUS));
+
+ return new Validation(status, messages);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java
index 9558941..77f6191 100644
--- a/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java
+++ b/common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java
@@ -44,7 +44,7 @@ public final class FormSerialization {
public static final String ID = "id";
public static final String NAME = "name";
public static final String VERSION = "version";
- public static final String FRAMEWORK_VERSION = "framework-version";
+ public static final String DRIVER_VERSION = "driver-version";
public static final String CLASS = "class";
public static final String ENABLED = "enabled";
public static final String CREATION_USER = "creation-user";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
index 9f3ad62..20ee5f3 100644
--- a/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
+++ b/common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
@@ -33,9 +33,9 @@ import java.util.ResourceBundle;
*/
public final class ResourceBundleSerialization {
- public static final String RESOURCES = "resources";
- public static final String CONNECTOR_RESOURCES = "resources-connector";
- public static final String FRAMEWORK_RESOURCES = "resources-framework";
+ public static final String CONFIGS = "configs";
+ public static final String CONNECTOR_CONFIGS = "connector-configs";
+ public static final String DRIVER_CONFIGS = "driver-configs";
@SuppressWarnings("unchecked")
public static JSONArray extractResourceBundles(List<ResourceBundle> bundles) {
@@ -50,27 +50,20 @@ public final class ResourceBundleSerialization {
@SuppressWarnings("unchecked")
public static JSONObject extractResourceBundle(ResourceBundle bundle) {
JSONObject json = new JSONObject();
-
Enumeration<String> keys = bundle.getKeys();
-
while(keys.hasMoreElements()) {
String key = keys.nextElement();
-
json.put(key, bundle.getString(key));
-
}
-
return json;
}
@SuppressWarnings("unchecked")
public static List<ResourceBundle> restoreResourceBundles(JSONArray array) {
List<ResourceBundle> bundles = new LinkedList<ResourceBundle>();
-
for (Object item : array) {
bundles.add(restoreResourceBundle((JSONObject) item));
}
-
return bundles;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/model/Input.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/Input.java b/common/src/main/java/org/apache/sqoop/model/Input.java
index 2b32e43..db3e0e6 100644
--- a/common/src/main/java/org/apache/sqoop/model/Input.java
+++ b/common/src/main/java/org/apache/sqoop/model/Input.java
@@ -30,7 +30,7 @@ import java.lang.annotation.Target;
@Target(ElementType.FIELD)
public @interface Input {
/**
- * Sqoop framework will ensure that sensitive information will not be easily
+ * Sqoop will ensure that sensitive information will not be easily
* accessible.
*
* @return True if field is sensitive
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/model/MConnection.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConnection.java b/common/src/main/java/org/apache/sqoop/model/MConnection.java
deleted file mode 100644
index f84abbf..0000000
--- a/common/src/main/java/org/apache/sqoop/model/MConnection.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-/**
- * Model describing entire connection object including both connector and
- * framework part.
- */
-public class MConnection extends MAccountableEntity implements MClonable {
- private long connectorId;
-
- private final MConnectionForms connectorPart;
- private final MConnectionForms frameworkPart;
-
- /**
- * Default constructor to build new MConnection model.
- *
- * @param connectorId Connector id
- * @param connectorPart Connector forms
- * @param frameworkPart Framework forms
- */
- public MConnection(long connectorId,
- MConnectionForms connectorPart,
- MConnectionForms frameworkPart) {
- this.connectorId = connectorId;
- this.connectorPart = connectorPart;
- this.frameworkPart = frameworkPart;
- }
-
- /**
- * Constructor to create deep copy of another MConnection model.
- *
- * @param other MConnection model to copy
- */
- public MConnection(MConnection other) {
- this(other, other.connectorPart.clone(true), other.frameworkPart.clone(true));
- }
-
- /**
- * Construct new MConnection model as a copy of another with replaced forms.
- *
- * This method is suitable only for metadata upgrade path and should not be
- * used otherwise.
- *
- * @param other MConnection model to copy
- * @param connectorPart Connector forms
- * @param frameworkPart Framework forms
- */
- public MConnection(MConnection other, MConnectionForms connectorPart, MConnectionForms frameworkPart) {
- super(other);
- this.connectorId = other.connectorId;
- this.connectorPart = connectorPart;
- this.frameworkPart = frameworkPart;
- this.setPersistenceId(other.getPersistenceId());
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("connection: ").append(getName());
- sb.append(" connector-part: ").append(connectorPart);
- sb.append(", framework-part: ").append(frameworkPart);
-
- return sb.toString();
- }
-
- public long getConnectorId() {
- return connectorId;
- }
-
- public void setConnectorId(long connectorId) {
- this.connectorId = connectorId;
- }
-
- public MConnectionForms getConnectorPart() {
- return connectorPart;
- }
-
- public MConnectionForms getFrameworkPart() {
- return frameworkPart;
- }
-
- public MForm getConnectorForm(String formName) {
- return connectorPart.getForm(formName);
- }
-
- public MForm getFrameworkForm(String formName) {
- return frameworkPart.getForm(formName);
- }
-
- @Override
- public MConnection clone(boolean cloneWithValue) {
- if(cloneWithValue) {
- return new MConnection(this);
- } else {
- return new MConnection(connectorId, connectorPart.clone(false), frameworkPart.clone(false));
- }
- }
-
- @Override
- public boolean equals(Object object) {
- if(object == this) {
- return true;
- }
-
- if(!(object instanceof MConnection)) {
- return false;
- }
-
- MConnection mc = (MConnection)object;
- return (mc.connectorId == this.connectorId)
- && (mc.getPersistenceId() == this.getPersistenceId())
- && (mc.connectorPart.equals(this.connectorPart))
- && (mc.frameworkPart.equals(this.frameworkPart));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java b/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java
new file mode 100644
index 0000000..62eb197
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MDriverConfig.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+/**
+ * Describes the configs associated with the {@link Driver} for executing sqoop jobs.
+ */
+public class MDriverConfig extends MPersistableEntity implements MClonable {
+
+ private final MConnectionForms connectionForms;
+ private final MJobForms jobForms;
+ String version;
+
+ public MDriverConfig(MConnectionForms connectionForms, MJobForms jobForms, String version) {
+ this.connectionForms = connectionForms;
+ this.jobForms = jobForms;
+ this.version = version;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("driver-");
+ sb.append(getPersistenceId()).append(":");
+ sb.append("version = " + version);
+ sb.append(", ").append(connectionForms.toString());
+ sb.append(jobForms.toString());
+
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (!(other instanceof MDriverConfig)) {
+ return false;
+ }
+
+ MDriverConfig mo = (MDriverConfig) other;
+ return version.equals(mo.getVersion()) &&
+ connectionForms.equals(mo.connectionForms) &&
+ jobForms.equals(mo.jobForms);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = connectionForms.hashCode();
+ result = 31 * result + jobForms.hashCode();
+ result = 31 * result + version.hashCode();
+ return result;
+ }
+
+ public MConnectionForms getConnectionForms() {
+ return connectionForms;
+ }
+
+ public MJobForms getJobForms() {
+ return jobForms;
+ }
+
+ @Override
+ public MDriverConfig clone(boolean cloneWithValue) {
+ //Framework never have any values filled
+ cloneWithValue = false;
+ MDriverConfig copy = new MDriverConfig(this.getConnectionForms().clone(cloneWithValue),
+ this.getJobForms().clone(cloneWithValue), this.version);
+ copy.setPersistenceId(this.getPersistenceId());
+ return copy;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/model/MFramework.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MFramework.java b/common/src/main/java/org/apache/sqoop/model/MFramework.java
deleted file mode 100644
index 580db9c..0000000
--- a/common/src/main/java/org/apache/sqoop/model/MFramework.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-/**
- * Metadata describing framework options for connection and jobForms for each
- * supported jobForms type.
- */
-public class MFramework extends MPersistableEntity implements MClonable {
-
- private final MConnectionForms connectionForms;
- private final MJobForms jobForms;
- String version;
-
- public MFramework(MConnectionForms connectionForms, MJobForms jobForms,
- String version) {
- this.version = version;
- this.connectionForms = connectionForms;
- this.jobForms = jobForms;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("framework-");
- sb.append(getPersistenceId()).append(":");
- sb.append("version = " + version);
- sb.append(", ").append(connectionForms.toString());
- sb.append(jobForms.toString());
-
- return sb.toString();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == this) {
- return true;
- }
-
- if (!(other instanceof MFramework)) {
- return false;
- }
-
- MFramework mo = (MFramework) other;
- return version.equals(mo.getVersion()) &&
- connectionForms.equals(mo.connectionForms) &&
- jobForms.equals(mo.jobForms);
- }
-
- @Override
- public int hashCode() {
- int result = connectionForms.hashCode();
- result = 31 * result + jobForms.hashCode();
- result = 31 * result + version.hashCode();
- return result;
- }
-
- public MConnectionForms getConnectionForms() {
- return connectionForms;
- }
-
- public MJobForms getJobForms() {
- return jobForms;
- }
-
- @Override
- public MFramework clone(boolean cloneWithValue) {
- //Framework never have any values filled
- cloneWithValue = false;
- MFramework copy = new MFramework(this.getConnectionForms().clone(cloneWithValue),
- this.getJobForms().clone(cloneWithValue), this.version);
- copy.setPersistenceId(this.getPersistenceId());
- return copy;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/model/MInput.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MInput.java b/common/src/main/java/org/apache/sqoop/model/MInput.java
index 1f4c425..0074267 100644
--- a/common/src/main/java/org/apache/sqoop/model/MInput.java
+++ b/common/src/main/java/org/apache/sqoop/model/MInput.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.model;
/**
- * Represents a parameter input used by the connector for creating a connection
+ * Represents a parameter input used by the connector for creating a link
* or a job object.
* @param <T> the value type associated with this parameter
* @param boolean whether or not the field contains sensitive information
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/model/MJob.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MJob.java b/common/src/main/java/org/apache/sqoop/model/MJob.java
index 182bbfb..c9b45a5 100644
--- a/common/src/main/java/org/apache/sqoop/model/MJob.java
+++ b/common/src/main/java/org/apache/sqoop/model/MJob.java
@@ -30,17 +30,17 @@ public class MJob extends MAccountableEntity implements MClonable {
* Connector reference.
*
* Job object do not immediately depend on connector as there is indirect
- * dependency through connection object, but having this dependency explicitly
+ * dependency through link object, but having this dependency explicitly
* carried along helps a lot.
*/
private final long fromConnectorId;
private final long toConnectorId;
/**
- * Corresponding connection objects for connector.
+ * Corresponding link objects for connector.
*/
- private final long fromConnectionId;
- private final long toConnectionId;
+ private final long fromLinkId;
+ private final long toLinkId;
private final MJobForms fromConnectorPart;
private final MJobForms toConnectorPart;
@@ -51,8 +51,8 @@ public class MJob extends MAccountableEntity implements MClonable {
*
* @param fromConnectorId FROM Connector id
* @param toConnectorId TO Connector id
- * @param fromConnectionId FROM Connection id
- * @param toConnectionId TO Connection id
+ * @param fromLinkId FROM Link id
+ * @param toLinkId TO Link id
* @param fromPart FROM Connector forms
* @param toPart TO Connector forms
* @param frameworkPart Framework forms
@@ -66,8 +66,8 @@ public class MJob extends MAccountableEntity implements MClonable {
MJobForms frameworkPart) {
this.fromConnectorId = fromConnectorId;
this.toConnectorId = toConnectorId;
- this.fromConnectionId = fromConnectionId;
- this.toConnectionId = toConnectionId;
+ this.fromLinkId = fromConnectionId;
+ this.toLinkId = toConnectionId;
this.fromConnectorPart = fromPart;
this.toConnectorPart = toPart;
this.frameworkPart = frameworkPart;
@@ -101,8 +101,8 @@ public class MJob extends MAccountableEntity implements MClonable {
this.fromConnectorId = other.getConnectorId(Direction.FROM);
this.toConnectorId = other.getConnectorId(Direction.TO);
- this.fromConnectionId = other.getConnectionId(Direction.FROM);
- this.toConnectionId = other.getConnectionId(Direction.TO);
+ this.fromLinkId = other.getLinkId(Direction.FROM);
+ this.toLinkId = other.getLinkId(Direction.TO);
this.fromConnectorPart = fromPart;
this.toConnectorPart = toPart;
this.frameworkPart = frameworkPart;
@@ -119,13 +119,13 @@ public class MJob extends MAccountableEntity implements MClonable {
return sb.toString();
}
- public long getConnectionId(Direction type) {
+ public long getLinkId(Direction type) {
switch(type) {
case FROM:
- return fromConnectionId;
+ return fromLinkId;
case TO:
- return toConnectionId;
+ return toLinkId;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
@@ -170,8 +170,8 @@ public class MJob extends MAccountableEntity implements MClonable {
return new MJob(
getConnectorId(Direction.FROM),
getConnectorId(Direction.TO),
- getConnectionId(Direction.FROM),
- getConnectionId(Direction.TO),
+ getLinkId(Direction.FROM),
+ getLinkId(Direction.TO),
getConnectorPart(Direction.FROM).clone(false),
getConnectorPart(Direction.TO).clone(false),
frameworkPart.clone(false));
@@ -191,8 +191,8 @@ public class MJob extends MAccountableEntity implements MClonable {
MJob job = (MJob)object;
return (job.getConnectorId(Direction.FROM) == this.getConnectorId(Direction.FROM))
&& (job.getConnectorId(Direction.TO) == this.getConnectorId(Direction.TO))
- && (job.getConnectionId(Direction.FROM) == this.getConnectionId(Direction.FROM))
- && (job.getConnectionId(Direction.TO) == this.getConnectionId(Direction.TO))
+ && (job.getLinkId(Direction.FROM) == this.getLinkId(Direction.FROM))
+ && (job.getLinkId(Direction.TO) == this.getLinkId(Direction.TO))
&& (job.getPersistenceId() == this.getPersistenceId())
&& (job.getConnectorPart(Direction.FROM).equals(this.getConnectorPart(Direction.FROM)))
&& (job.getConnectorPart(Direction.TO).equals(this.getConnectorPart(Direction.TO)))
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/common/src/main/java/org/apache/sqoop/model/MLink.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MLink.java b/common/src/main/java/org/apache/sqoop/model/MLink.java
new file mode 100644
index 0000000..6a8c424
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/model/MLink.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+/**
+ * Model describing the link object and its corresponding configs
+ */
+public class MLink extends MAccountableEntity implements MClonable {
+ private long connectorId;
+
+ private final MConnectionForms connectorPart;
+ private final MConnectionForms frameworkPart;
+
+ /**
+ * Default constructor to build new MConnection model.
+ *
+ * @param connectorId Connector id
+ * @param connectorPart Connector forms
+ * @param frameworkPart Framework forms
+ */
+ public MLink(long connectorId,
+ MConnectionForms connectorPart,
+ MConnectionForms frameworkPart) {
+ this.connectorId = connectorId;
+ this.connectorPart = connectorPart;
+ this.frameworkPart = frameworkPart;
+ }
+
+ /**
+ * Constructor to create deep copy of another MConnection model.
+ *
+ * @param other MConnection model to copy
+ */
+ public MLink(MLink other) {
+ this(other, other.connectorPart.clone(true), other.frameworkPart.clone(true));
+ }
+
+ /**
+ * Construct new MConnection model as a copy of another with replaced forms.
+ *
+ * This method is suitable only for metadata upgrade path and should not be
+ * used otherwise.
+ *
+ * @param other MConnection model to copy
+ * @param connectorPart Connector forms
+ * @param frameworkPart Framework forms
+ */
+ public MLink(MLink other, MConnectionForms connectorPart, MConnectionForms frameworkPart) {
+ super(other);
+ this.connectorId = other.connectorId;
+ this.connectorPart = connectorPart;
+ this.frameworkPart = frameworkPart;
+ this.setPersistenceId(other.getPersistenceId());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("connection: ").append(getName());
+ sb.append(" connector-part: ").append(connectorPart);
+ sb.append(", framework-part: ").append(frameworkPart);
+
+ return sb.toString();
+ }
+
+ public long getConnectorId() {
+ return connectorId;
+ }
+
+ public void setConnectorId(long connectorId) {
+ this.connectorId = connectorId;
+ }
+
+ public MConnectionForms getConnectorPart() {
+ return connectorPart;
+ }
+
+ public MConnectionForms getFrameworkPart() {
+ return frameworkPart;
+ }
+
+ public MForm getConnectorForm(String formName) {
+ return connectorPart.getForm(formName);
+ }
+
+ public MForm getFrameworkForm(String formName) {
+ return frameworkPart.getForm(formName);
+ }
+
+ @Override
+ public MLink clone(boolean cloneWithValue) {
+ if(cloneWithValue) {
+ return new MLink(this);
+ } else {
+ return new MLink(connectorId, connectorPart.clone(false), frameworkPart.clone(false));
+ }
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if(object == this) {
+ return true;
+ }
+
+ if(!(object instanceof MLink)) {
+ return false;
+ }
+
+ MLink mc = (MLink)object;
+ return (mc.connectorId == this.connectorId)
+ && (mc.getPersistenceId() == this.getPersistenceId())
+ && (mc.connectorPart.equals(this.connectorPart))
+ && (mc.frameworkPart.equals(this.frameworkPart));
+ }
+}
[23/52] [abbrv] git commit: SQOOP-1496: Sqoop2: Revisit/Refactor the
SubmissionEngine/ExecutionEngine APIs
Posted by ab...@apache.org.
SQOOP-1496: Sqoop2: Revisit/Refactor the SubmissionEngine/ExecutionEngine APIs
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3d539dd4
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3d539dd4
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3d539dd4
Branch: refs/heads/SQOOP-1367
Commit: 3d539dd4d7477324dfe62a4e57f684351769b000
Parents: af25bcc
Author: Abraham Elmahrek <ab...@elmahrek.com>
Authored: Fri Sep 19 16:24:59 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:18 2014 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/json/SubmissionBean.java | 20 +-
.../org/apache/sqoop/model/MSubmission.java | 31 +-
.../apache/sqoop/json/TestSubmissionBean.java | 12 +-
.../sqoop/connector/jdbc/TestToInitializer.java | 1 -
.../idf/CSVIntermediateDataFormat.java | 4 -
.../connector/idf/IntermediateDataFormat.java | 4 -
.../idf/CSVIntermediateDataFormatTest.java | 222 ------
.../idf/TestCSVIntermediateDataFormat.java | 222 ++++++
.../sqoop/connector/ConnectorManager.java | 5 +-
.../apache/sqoop/framework/ExecutionEngine.java | 20 +-
.../org/apache/sqoop/framework/JobManager.java | 447 ++++++------
.../org/apache/sqoop/framework/JobRequest.java | 356 ++++++++++
.../sqoop/framework/SubmissionEngine.java | 7 +-
.../sqoop/framework/SubmissionRequest.java | 361 ----------
.../sqoop/framework/TestFrameworkValidator.java | 182 +++--
.../apache/sqoop/framework/TestJobManager.java | 173 +++++
.../apache/sqoop/framework/TestJobRequest.java | 71 ++
.../sqoop/framework/TestSubmissionRequest.java | 71 --
.../sqoop/repository/TestJdbcRepository.java | 694 +++++++++----------
.../sqoop/execution/mapreduce/MRJobRequest.java | 102 +++
.../mapreduce/MRSubmissionRequest.java | 102 ---
.../mapreduce/MapreduceExecutionEngine.java | 51 +-
.../apache/sqoop/job/mr/ConfigurationUtils.java | 3 -
.../apache/sqoop/job/mr/ProgressRunnable.java | 4 +-
.../sqoop/job/mr/SqoopDestroyerExecutor.java | 7 +-
.../sqoop/job/mr/SqoopFileOutputFormat.java | 4 +-
.../org/apache/sqoop/job/mr/SqoopMapper.java | 50 +-
.../job/mr/SqoopOutputFormatLoadExecutor.java | 28 +-
.../sqoop/shell/utils/SubmissionDisplayer.java | 8 +-
.../org/apache/sqoop/job/etl/CallbackBase.java | 49 --
.../java/org/apache/sqoop/job/etl/From.java | 2 +-
.../main/java/org/apache/sqoop/job/etl/To.java | 2 +-
.../org/apache/sqoop/job/etl/Transferable.java | 51 ++
.../org/apache/sqoop/validation/Validator.java | 1 -
.../mapreduce/MapreduceSubmissionEngine.java | 12 +-
35 files changed, 1780 insertions(+), 1599 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java b/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
index 61d6576..9b1ae74 100644
--- a/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
@@ -52,8 +52,8 @@ public class SubmissionBean implements JsonBean {
private static final String EXCEPTION_TRACE = "exception-trace";
private static final String PROGRESS = "progress";
private static final String COUNTERS = "counters";
- private static final String CONNECTOR_SCHEMA = "schema-connector";
- private static final String HIO_SCHEMA = "schema-hio";
+ private static final String FROM_SCHEMA = "schema-from";
+ private static final String TO_SCHEMA = "schema-to";
private List<MSubmission> submissions;
@@ -116,11 +116,11 @@ public class SubmissionBean implements JsonBean {
if(submission.getCounters() != null) {
object.put(COUNTERS, extractCounters(submission.getCounters()));
}
- if(submission.getConnectorSchema() != null) {
- object.put(CONNECTOR_SCHEMA, extractSchema(submission.getConnectorSchema()));
+ if(submission.getFromSchema() != null) {
+ object.put(FROM_SCHEMA, extractSchema(submission.getFromSchema()));
}
- if(submission.getHioSchema() != null) {
- object.put(HIO_SCHEMA, extractSchema(submission.getHioSchema()));
+ if(submission.getToSchema() != null) {
+ object.put(TO_SCHEMA, extractSchema(submission.getToSchema()));
}
array.add(object);
@@ -188,11 +188,11 @@ public class SubmissionBean implements JsonBean {
if(object.containsKey(COUNTERS)) {
submission.setCounters(restoreCounters((JSONObject) object.get(COUNTERS)));
}
- if(object.containsKey(CONNECTOR_SCHEMA)) {
- submission.setConnectorSchema(restoreSchemna((JSONObject) object.get(CONNECTOR_SCHEMA)));
+ if(object.containsKey(FROM_SCHEMA)) {
+ submission.setFromSchema(restoreSchemna((JSONObject) object.get(FROM_SCHEMA)));
}
- if(object.containsKey(HIO_SCHEMA)) {
- submission.setHioSchema(restoreSchemna((JSONObject) object.get(HIO_SCHEMA)));
+ if(object.containsKey(TO_SCHEMA)) {
+ submission.setToSchema(restoreSchemna((JSONObject) object.get(TO_SCHEMA)));
}
this.submissions.add(submission);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/common/src/main/java/org/apache/sqoop/model/MSubmission.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MSubmission.java b/common/src/main/java/org/apache/sqoop/model/MSubmission.java
index 1edd6ee..ca21135 100644
--- a/common/src/main/java/org/apache/sqoop/model/MSubmission.java
+++ b/common/src/main/java/org/apache/sqoop/model/MSubmission.java
@@ -100,20 +100,21 @@ public class MSubmission extends MAccountableEntity {
String exceptionStackTrace;
/**
- * Schema that was reported by the connector.
+ * Schema for the FROM part of the job submission
*
* This property is required.
*/
- Schema connectorSchema;
+ Schema fromSchema;
/**
+ * Schema for the TO part of the job submission
* Optional schema that reported by the underlying I/O implementation. Please
- * note that this property might be empty and in such case the connector
- * schema will use also on Hadoop I/O side.
+ * note that this property might be empty and in such case use the FROM schema
+ * on the TO side.
*
* This property is optional.
*/
- Schema hioSchema;
+ Schema toSchema;
public MSubmission() {
status = SubmissionStatus.UNKNOWN;
@@ -219,20 +220,20 @@ public class MSubmission extends MAccountableEntity {
this.setExceptionStackTrace(writer.toString());
}
- public Schema getConnectorSchema() {
- return connectorSchema;
+ public Schema getFromSchema() {
+ return fromSchema;
}
- public void setConnectorSchema(Schema connectorSchema) {
- this.connectorSchema = connectorSchema;
+ public void setFromSchema(Schema connectorSchema) {
+ this.fromSchema = connectorSchema;
}
- public Schema getHioSchema() {
- return hioSchema;
+ public Schema getToSchema() {
+ return toSchema;
}
- public void setHioSchema(Schema hioSchema) {
- this.hioSchema = hioSchema;
+ public void setToSchema(Schema hioSchema) {
+ this.toSchema = hioSchema;
}
@Override
@@ -248,8 +249,8 @@ public class MSubmission extends MAccountableEntity {
", externalLink='" + externalLink + '\'' +
", exceptionInfo='" + exceptionInfo + '\'' +
", exceptionStackTrace='" + exceptionStackTrace + '\'' +
- ", connectorSchema='" + connectorSchema + '\'' +
- ", hioSchema='" + hioSchema + '\'' +
+ ", fromSchema='" + fromSchema + '\'' +
+ ", toSchema='" + toSchema + '\'' +
'}';
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java b/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
index d87655e..518c9cb 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
@@ -405,20 +405,20 @@ public class TestSubmissionBean extends TestCase {
assertEquals(222222, counter.getValue());
}
- public void testTransferConnectorSchema() {
+ public void testTransferFromSchema() {
MSubmission source = new MSubmission();
- source.setConnectorSchema(getSchema());
+ source.setFromSchema(getSchema());
- Schema target = transfer(source).getConnectorSchema();
+ Schema target = transfer(source).getFromSchema();
assertNotNull(target);
assertEquals(getSchema(), target);
}
- public void testTransferHioSchema() {
+ public void testTransferToSchema() {
MSubmission source = new MSubmission();
- source.setHioSchema(getSchema());
+ source.setToSchema(getSchema());
- Schema target = transfer(source).getHioSchema();
+ Schema target = transfer(source).getToSchema();
assertNotNull(target);
assertEquals(getSchema(), target);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
index eb6fcf1..4767215 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
@@ -26,7 +26,6 @@ import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.job.etl.Initializer;
import org.apache.sqoop.job.etl.InitializerContext;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.ValidationResult;
import org.apache.sqoop.validation.ValidationRunner;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
index 1e8ab52..df5cb9c 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java
@@ -21,7 +21,6 @@ package org.apache.sqoop.connector.idf;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Column;
@@ -38,7 +37,6 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
-import java.util.regex.Pattern;
public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
@@ -46,8 +44,6 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> {
public static final char ESCAPE_CHARACTER = '\\';
public static final char QUOTE_CHARACTER = '\'';
- private static final Logger LOG = Logger.getLogger
- (CSVIntermediateDataFormat.class);
private static final char[] originals = {
0x5C,0x00,0x0A,0x0D,0x1A,0x22,0x27
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
index 91b594e..66d46a3 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java
@@ -19,14 +19,10 @@
package org.apache.sqoop.connector.idf;
import org.apache.sqoop.schema.Schema;
-import org.apache.sqoop.schema.type.Column;
-import org.apache.sqoop.schema.type.Type;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
/**
* Abstract class representing a pluggable intermediate data format the Sqoop
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormatTest.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormatTest.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormatTest.java
deleted file mode 100644
index df6d30f..0000000
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormatTest.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sqoop.connector.idf;
-
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.schema.Schema;
-import org.apache.sqoop.schema.type.Binary;
-import org.apache.sqoop.schema.type.FixedPoint;
-import org.apache.sqoop.schema.type.Text;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-public class CSVIntermediateDataFormatTest {
-
- private final String BYTE_FIELD_ENCODING = "ISO-8859-1";
-
- private IntermediateDataFormat<?> data;
-
- @Before
- public void setUp() {
- data = new CSVIntermediateDataFormat();
- }
-
- private String getByteFieldString(byte[] byteFieldData) {
- try {
- return new StringBuilder("'").append(new String(byteFieldData, BYTE_FIELD_ENCODING)).append("'").toString();
- } catch(UnsupportedEncodingException e) {
- // Should never get to this point because ISO-8859-1 is a standard codec.
- return null;
- }
- }
-
- @Test
- public void testStringInStringOut() {
- String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
- + ",'" + String.valueOf(0x0A) + "'";
- data.setTextData(testData);
- assertEquals(testData, data.getTextData());
- }
-
- @Test
- public void testNullStringInObjectOut() {
- Schema schema = new Schema("test");
- schema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"))
- .addColumn(new Text("3"))
- .addColumn(new Text("4"))
- .addColumn(new Binary("5"))
- .addColumn(new Text("6"));
- data.setSchema(schema);
- data.setTextData(null);
-
- Object[] out = data.getObjectData();
-
- assertNull(out);
- }
-
- @Test(expected=SqoopException.class)
- public void testEmptyStringInObjectOut() {
- Schema schema = new Schema("test");
- schema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"))
- .addColumn(new Text("3"))
- .addColumn(new Text("4"))
- .addColumn(new Binary("5"))
- .addColumn(new Text("6"));
- data.setSchema(schema);
- data.setTextData("");
-
- data.getObjectData();
- }
-
- @Test
- public void testStringInObjectOut() {
-
- //byte[0] = -112, byte[1] = 54 - 2's complements
- String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
- + ",'\\n'";
- Schema schema = new Schema("test");
- schema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"))
- .addColumn(new Text("3"))
- .addColumn(new Text("4"))
- .addColumn(new Binary("5"))
- .addColumn(new Text("6"));
- data.setSchema(schema);
- data.setTextData(testData);
-
- Object[] out = data.getObjectData();
-
- assertEquals(new Long(10),out[0]);
- assertEquals(new Long(34),out[1]);
- assertEquals("54",out[2]);
- assertEquals("random data",out[3]);
- assertEquals(-112, ((byte[])out[4])[0]);
- assertEquals(54, ((byte[])out[4])[1]);
- assertEquals("\n", out[5].toString());
- }
-
- @Test
- public void testObjectInStringOut() {
- Schema schema = new Schema("test");
- schema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"))
- .addColumn(new Text("3"))
- .addColumn(new Text("4"))
- .addColumn(new Binary("5"))
- .addColumn(new Text("6"));
- data.setSchema(schema);
-
- byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54};
- Object[] in = new Object[6];
- in[0] = new Long(10);
- in[1] = new Long(34);
- in[2] = "54";
- in[3] = "random data";
- in[4] = byteFieldData;
- in[5] = new String(new char[] { 0x0A });
-
- data.setObjectData(in);
-
- //byte[0] = \r byte[1] = -112, byte[1] = 54 - 2's complements
- String testData = "10,34,'54','random data'," +
- getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n'";
- assertEquals(testData, data.getTextData());
- }
-
- @Test
- public void testObjectInObjectOut() {
- //Test escapable sequences too.
- //byte[0] = -112, byte[1] = 54 - 2's complements
- Schema schema = new Schema("test");
- schema.addColumn(new FixedPoint("1"))
- .addColumn(new FixedPoint("2"))
- .addColumn(new Text("3"))
- .addColumn(new Text("4"))
- .addColumn(new Binary("5"))
- .addColumn(new Text("6"));
- data.setSchema(schema);
-
- Object[] in = new Object[6];
- in[0] = new Long(10);
- in[1] = new Long(34);
- in[2] = "54";
- in[3] = "random data";
- in[4] = new byte[] { (byte) -112, (byte) 54};
- in[5] = new String(new char[] { 0x0A });
- Object[] inCopy = new Object[6];
- System.arraycopy(in,0,inCopy,0,in.length);
-
- // Modifies the input array, so we use the copy to confirm
- data.setObjectData(in);
-
- assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
- }
-
- @Test
- public void testStringFullRangeOfCharacters() {
- Schema schema = new Schema("test");
- schema.addColumn(new Text("1"));
- data.setSchema(schema);
-
- char[] allCharArr = new char[256];
- for(int i = 0; i < allCharArr.length; ++i) {
- allCharArr[i] = (char)i;
- }
- String strData = new String(allCharArr);
-
- Object[] in = {strData};
- Object[] inCopy = new Object[1];
- System.arraycopy(in,0,inCopy,0,in.length);
-
- // Modifies the input array, so we use the copy to confirm
- data.setObjectData(in);
-
- assertEquals(strData, data.getObjectData()[0]);
- assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
- }
-
- @Test
- public void testByteArrayFullRangeOfCharacters() {
- Schema schema = new Schema("test");
- schema.addColumn(new Binary("1"));
- data.setSchema(schema);
-
- byte[] allCharByteArr = new byte[256];
- for(int i = 0; i < allCharByteArr.length; ++i) {
- allCharByteArr[i] = (byte)i;
- }
-
- Object[] in = {allCharByteArr};
- Object[] inCopy = new Object[1];
- System.arraycopy(in,0,inCopy,0,in.length);
-
- // Modifies the input array, so we use the copy to confirm
- data.setObjectData(in);
- assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
new file mode 100644
index 0000000..8c83a71
--- /dev/null
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.connector.idf;
+
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.schema.Schema;
+import org.apache.sqoop.schema.type.Binary;
+import org.apache.sqoop.schema.type.FixedPoint;
+import org.apache.sqoop.schema.type.Text;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class TestCSVIntermediateDataFormat {
+
+ private final String BYTE_FIELD_ENCODING = "ISO-8859-1";
+
+ private IntermediateDataFormat<?> data;
+
+ @Before
+ public void setUp() {
+ data = new CSVIntermediateDataFormat();
+ }
+
+ private String getByteFieldString(byte[] byteFieldData) {
+ try {
+ return new StringBuilder("'").append(new String(byteFieldData, BYTE_FIELD_ENCODING)).append("'").toString();
+ } catch(UnsupportedEncodingException e) {
+ // Should never get to this point because ISO-8859-1 is a standard codec.
+ return null;
+ }
+ }
+
+ @Test
+ public void testStringInStringOut() {
+ String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
+ + ",'" + String.valueOf(0x0A) + "'";
+ data.setTextData(testData);
+ assertEquals(testData, data.getTextData());
+ }
+
+ @Test
+ public void testNullStringInObjectOut() {
+ Schema schema = new Schema("test");
+ schema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"))
+ .addColumn(new Text("3"))
+ .addColumn(new Text("4"))
+ .addColumn(new Binary("5"))
+ .addColumn(new Text("6"));
+ data.setSchema(schema);
+ data.setTextData(null);
+
+ Object[] out = data.getObjectData();
+
+ assertNull(out);
+ }
+
+ @Test(expected=SqoopException.class)
+ public void testEmptyStringInObjectOut() {
+ Schema schema = new Schema("test");
+ schema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"))
+ .addColumn(new Text("3"))
+ .addColumn(new Text("4"))
+ .addColumn(new Binary("5"))
+ .addColumn(new Text("6"));
+ data.setSchema(schema);
+ data.setTextData("");
+
+ data.getObjectData();
+ }
+
+ @Test
+ public void testStringInObjectOut() {
+
+ //byte[0] = -112, byte[1] = 54 - 2's complements
+ String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54})
+ + ",'\\n'";
+ Schema schema = new Schema("test");
+ schema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"))
+ .addColumn(new Text("3"))
+ .addColumn(new Text("4"))
+ .addColumn(new Binary("5"))
+ .addColumn(new Text("6"));
+ data.setSchema(schema);
+ data.setTextData(testData);
+
+ Object[] out = data.getObjectData();
+
+ assertEquals(new Long(10),out[0]);
+ assertEquals(new Long(34),out[1]);
+ assertEquals("54",out[2]);
+ assertEquals("random data",out[3]);
+ assertEquals(-112, ((byte[])out[4])[0]);
+ assertEquals(54, ((byte[])out[4])[1]);
+ assertEquals("\n", out[5].toString());
+ }
+
+ @Test
+ public void testObjectInStringOut() {
+ Schema schema = new Schema("test");
+ schema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"))
+ .addColumn(new Text("3"))
+ .addColumn(new Text("4"))
+ .addColumn(new Binary("5"))
+ .addColumn(new Text("6"));
+ data.setSchema(schema);
+
+ byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54};
+ Object[] in = new Object[6];
+ in[0] = new Long(10);
+ in[1] = new Long(34);
+ in[2] = "54";
+ in[3] = "random data";
+ in[4] = byteFieldData;
+ in[5] = new String(new char[] { 0x0A });
+
+ data.setObjectData(in);
+
+ //byte[0] = \r byte[1] = -112, byte[1] = 54 - 2's complements
+ String testData = "10,34,'54','random data'," +
+ getByteFieldString(byteFieldData).replaceAll("\r", "\\\\r") + ",'\\n'";
+ assertEquals(testData, data.getTextData());
+ }
+
+ @Test
+ public void testObjectInObjectOut() {
+ //Test escapable sequences too.
+ //byte[0] = -112, byte[1] = 54 - 2's complements
+ Schema schema = new Schema("test");
+ schema.addColumn(new FixedPoint("1"))
+ .addColumn(new FixedPoint("2"))
+ .addColumn(new Text("3"))
+ .addColumn(new Text("4"))
+ .addColumn(new Binary("5"))
+ .addColumn(new Text("6"));
+ data.setSchema(schema);
+
+ Object[] in = new Object[6];
+ in[0] = new Long(10);
+ in[1] = new Long(34);
+ in[2] = "54";
+ in[3] = "random data";
+ in[4] = new byte[] { (byte) -112, (byte) 54};
+ in[5] = new String(new char[] { 0x0A });
+ Object[] inCopy = new Object[6];
+ System.arraycopy(in,0,inCopy,0,in.length);
+
+ // Modifies the input array, so we use the copy to confirm
+ data.setObjectData(in);
+
+ assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
+ }
+
+ @Test
+ public void testStringFullRangeOfCharacters() {
+ Schema schema = new Schema("test");
+ schema.addColumn(new Text("1"));
+ data.setSchema(schema);
+
+ char[] allCharArr = new char[256];
+ for(int i = 0; i < allCharArr.length; ++i) {
+ allCharArr[i] = (char)i;
+ }
+ String strData = new String(allCharArr);
+
+ Object[] in = {strData};
+ Object[] inCopy = new Object[1];
+ System.arraycopy(in,0,inCopy,0,in.length);
+
+ // Modifies the input array, so we use the copy to confirm
+ data.setObjectData(in);
+
+ assertEquals(strData, data.getObjectData()[0]);
+ assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
+ }
+
+ @Test
+ public void testByteArrayFullRangeOfCharacters() {
+ Schema schema = new Schema("test");
+ schema.addColumn(new Binary("1"));
+ data.setSchema(schema);
+
+ byte[] allCharByteArr = new byte[256];
+ for(int i = 0; i < allCharByteArr.length; ++i) {
+ allCharByteArr[i] = (byte)i;
+ }
+
+ Object[] in = {allCharByteArr};
+ Object[] inCopy = new Object[1];
+ System.arraycopy(in,0,inCopy,0,in.length);
+
+ // Modifies the input array, so we use the copy to confirm
+ data.setObjectData(in);
+ assertTrue(Arrays.deepEquals(inCopy, data.getObjectData()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
index db6f579..c87df84 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
@@ -114,10 +114,9 @@ public class ConnectorManager implements Reconfigurable {
return bundles;
}
- public ResourceBundle getResourceBundle(long connectorId,
- Locale locale) {
+ public ResourceBundle getResourceBundle(long connectorId, Locale locale) {
ConnectorHandler handler = handlerMap.get(nameMap.get(connectorId));
- return handler.getConnector().getBundle(locale);
+ return handler.getConnector().getBundle(locale);
}
public MConnector getConnectorMetadata(long connectorId) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java b/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java
index 96ec148..75b570d 100644
--- a/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java
+++ b/core/src/main/java/org/apache/sqoop/framework/ExecutionEngine.java
@@ -18,12 +18,11 @@
package org.apache.sqoop.framework;
import org.apache.sqoop.common.ImmutableContext;
-import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.model.MSubmission;
/**
- * Execution engine drive execution of sqoop submission (job). It's responsible
+ * Execution engine drives execution of sqoop job. It's responsible
* for executing all defined steps in the import/export workflow.
+ * A successful job execution will be recorded in the job submission entity
*/
public abstract class ExecutionEngine {
@@ -31,6 +30,7 @@ public abstract class ExecutionEngine {
* Initialize execution engine
*
* @param context Configuration context
+ * @parma prefix Execution engine prefix
*/
public void initialize(ImmutableContext context, String prefix) {
}
@@ -42,19 +42,19 @@ public abstract class ExecutionEngine {
}
/**
- * Return new SubmissionRequest class or any subclass if it's needed by
+ * Return new JobRequest class or any subclass if it's needed by
* execution and submission engine combination.
*
- * @return New Submission request object
+ * @return new JobRequestobject
*/
- public SubmissionRequest createSubmissionRequest() {
- return new SubmissionRequest();
+ public JobRequest createJobRequest() {
+ return new JobRequest();
}
/**
- * Prepare given submission request.
+ * Prepare given job request.
*
- * @param request Submission request
+ * @param request JobRequest
*/
- public abstract void prepareSubmission(SubmissionRequest request);
+ public abstract void prepareJob(JobRequest request);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/main/java/org/apache/sqoop/framework/JobManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/JobManager.java b/core/src/main/java/org/apache/sqoop/framework/JobManager.java
index b1b37f6..8149d1c 100644
--- a/core/src/main/java/org/apache/sqoop/framework/JobManager.java
+++ b/core/src/main/java/org/apache/sqoop/framework/JobManager.java
@@ -1,3 +1,5 @@
+
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -266,264 +268,228 @@ public class JobManager implements Reconfigurable {
}
public MSubmission submit(long jobId, HttpEventContext ctx) {
- String username = ctx.getUsername();
-
- Repository repository = RepositoryManager.getInstance().getRepository();
-
- MJob job = repository.findJob(jobId);
- if (job == null) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0004,
- "Unknown job id " + jobId);
- }
- if (!job.getEnabled()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0009,
- "Job id: " + job.getPersistenceId());
+ MSubmission mSubmission = createJobSubmission(ctx, jobId);
+ JobRequest jobRequest = createJobRequest(jobId, mSubmission);
+ // Bootstrap job to execute
+ prepareJob(jobRequest);
+ // Make sure that this job id is not currently running and submit the job
+ // only if it's not.
+ synchronized (getClass()) {
+ MSubmission lastSubmission = RepositoryManager.getInstance().getRepository()
+ .findSubmissionLastForJob(jobId);
+ if (lastSubmission != null && lastSubmission.getStatus().isRunning()) {
+ throw new SqoopException(FrameworkError.FRAMEWORK_0002, "Job with id " + jobId);
+ }
+ // TODO(Abe): Call multiple destroyers.
+ // TODO(jarcec): We might need to catch all exceptions here to ensure
+ // that Destroyer will be executed in all cases.
+ // NOTE: the following is a blocking call
+ boolean success = submissionEngine.submit(jobRequest);
+ if (!success) {
+ destroySubmission(jobRequest);
+ mSubmission.setStatus(SubmissionStatus.FAILURE_ON_SUBMIT);
+ }
+ RepositoryManager.getInstance().getRepository().createSubmission(mSubmission);
}
+ return mSubmission;
+ }
- MConnection fromConnection = repository.findConnection(job.getConnectionId(Direction.FROM));
- MConnection toConnection = repository.findConnection(job.getConnectionId(Direction.TO));
+ private JobRequest createJobRequest(long jobId, MSubmission submission) {
+ // get job
+ MJob job = getJob(jobId);
- if (!fromConnection.getEnabled()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0010,
- "Connection id: " + fromConnection.getPersistenceId());
- }
+ // get from/to connections for the job
+ MConnection fromConnection = getConnection(job.getConnectionId(Direction.FROM));
+ MConnection toConnection = getConnection(job.getConnectionId(Direction.TO));
- if (!toConnection.getEnabled()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0010,
- "Connection id: " + toConnection.getPersistenceId());
- }
+ // get from/to connectors for the connection
+ SqoopConnector fromConnector = getConnector(fromConnection.getConnectorId());
+ validateSupportedDirection(fromConnector, Direction.FROM);
+ SqoopConnector toConnector = getConnector(toConnection.getConnectorId());
+ validateSupportedDirection(toConnector, Direction.TO);
- SqoopConnector fromConnector =
- ConnectorManager.getInstance().getConnector(job.getConnectorId(Direction.FROM));
- SqoopConnector toConnector =
- ConnectorManager.getInstance().getConnector(job.getConnectorId(Direction.TO));
+ // Transform config to fromConnector specific classes
+ Object fromConnectionConfig = ClassUtils.instantiate(fromConnector
+ .getConnectionConfigurationClass());
+ FormUtils.fromForms(fromConnection.getConnectorPart().getForms(), fromConnectionConfig);
- // Make sure that connectors support the directions they will be used from.
- if (!fromConnector.getSupportedDirections().contains(Direction.FROM)) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0011,
- "Connector: " + fromConnector.getClass().getCanonicalName());
- }
+ // Transform config to toConnector specific classes
+ Object toConnectorConfig = ClassUtils
+ .instantiate(toConnector.getConnectionConfigurationClass());
+ FormUtils.fromForms(toConnection.getConnectorPart().getForms(), toConnectorConfig);
- if (!toConnector.getSupportedDirections().contains(Direction.TO)) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0011,
- "Connector: " + toConnector.getClass().getCanonicalName());
- }
+ Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
+ FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromJob);
- // Transform forms to fromConnector specific classes
- Object fromConnectorConnection = ClassUtils.instantiate(
- fromConnector.getConnectionConfigurationClass());
- FormUtils.fromForms(fromConnection.getConnectorPart().getForms(),
- fromConnectorConnection);
-
- Object fromJob = ClassUtils.instantiate(
- fromConnector.getJobConfigurationClass(Direction.FROM));
- FormUtils.fromForms(
- job.getConnectorPart(Direction.FROM).getForms(), fromJob);
-
- // Transform forms to toConnector specific classes
- Object toConnectorConnection = ClassUtils.instantiate(
- toConnector.getConnectionConfigurationClass());
- FormUtils.fromForms(toConnection.getConnectorPart().getForms(),
- toConnectorConnection);
-
- Object toJob = ClassUtils.instantiate(
- toConnector.getJobConfigurationClass(Direction.TO));
+ Object toJob = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
FormUtils.fromForms(job.getConnectorPart(Direction.TO).getForms(), toJob);
- // Transform framework specific forms
- Object fromFrameworkConnection = ClassUtils.instantiate(
- FrameworkManager.getInstance().getConnectionConfigurationClass());
- Object toFrameworkConnection = ClassUtils.instantiate(
- FrameworkManager.getInstance().getConnectionConfigurationClass());
- FormUtils.fromForms(fromConnection.getFrameworkPart().getForms(),
- fromFrameworkConnection);
- FormUtils.fromForms(toConnection.getFrameworkPart().getForms(),
- toFrameworkConnection);
-
- Object frameworkJob = ClassUtils.instantiate(
- FrameworkManager.getInstance().getJobConfigurationClass());
+ // Transform framework specific configs
+ // Q(VB) : Aren't the following 2 exactly the same?
+ Object fromFrameworkConnection = ClassUtils.instantiate(FrameworkManager.getInstance()
+ .getConnectionConfigurationClass());
+ FormUtils.fromForms(fromConnection.getFrameworkPart().getForms(), fromFrameworkConnection);
+
+ Object toFrameworkConnection = ClassUtils.instantiate(FrameworkManager.getInstance()
+ .getConnectionConfigurationClass());
+ FormUtils.fromForms(toConnection.getFrameworkPart().getForms(), toFrameworkConnection);
+
+ Object frameworkJob = ClassUtils.instantiate(FrameworkManager.getInstance()
+ .getJobConfigurationClass());
FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);
- // Create request object
- MSubmission summary = new MSubmission(jobId);
- SubmissionRequest request = executionEngine.createSubmissionRequest();
-
- summary.setCreationUser(username);
- summary.setLastUpdateUser(username);
-
- // Save important variables to the submission request
- request.setSummary(summary);
- request.setConnector(Direction.FROM, fromConnector);
- request.setConnector(Direction.TO, toConnector);
- request.setConnectorConnectionConfig(Direction.FROM, fromConnectorConnection);
- request.setConnectorConnectionConfig(Direction.TO, toConnectorConnection);
- request.setConnectorJobConfig(Direction.FROM, fromJob);
- request.setConnectorJobConfig(Direction.TO, toJob);
- // @TODO(Abe): Should we actually have 2 different Framework Connection config objects?
- request.setFrameworkConnectionConfig(Direction.FROM, fromFrameworkConnection);
- request.setFrameworkConnectionConfig(Direction.TO, toFrameworkConnection);
- request.setConfigFrameworkJob(frameworkJob);
- request.setJobName(job.getName());
- request.setJobId(job.getPersistenceId());
- request.setNotificationUrl(notificationBaseUrl + jobId);
+ // Create a job request for submit/execution
+ JobRequest jobRequest = executionEngine.createJobRequest();
+ // Save important variables to the job request
+ jobRequest.setSummary(submission);
+ jobRequest.setConnector(Direction.FROM, fromConnector);
+ jobRequest.setConnector(Direction.TO, toConnector);
+ jobRequest.setConnectorConnectionConfig(Direction.FROM, fromConnectionConfig);
+ jobRequest.setConnectorConnectionConfig(Direction.TO, toConnectorConfig);
+ jobRequest.setConnectorJobConfig(Direction.FROM, fromJob);
+ jobRequest.setConnectorJobConfig(Direction.TO, toJob);
+ // TODO(Abe): Should we actually have 2 different Framework Connection config objects?
+ jobRequest.setFrameworkConnectionConfig(Direction.FROM, fromFrameworkConnection);
+ jobRequest.setFrameworkConnectionConfig(Direction.TO, toFrameworkConnection);
+ jobRequest.setConfigFrameworkJob(frameworkJob);
+ jobRequest.setJobName(job.getName());
+ jobRequest.setJobId(job.getPersistenceId());
+ jobRequest.setNotificationUrl(notificationBaseUrl + jobId);
Class<? extends IntermediateDataFormat<?>> dataFormatClass =
fromConnector.getIntermediateDataFormat();
- request.setIntermediateDataFormat(fromConnector.getIntermediateDataFormat());
- // Create request object
+ jobRequest.setIntermediateDataFormat(fromConnector.getIntermediateDataFormat());
+
+
+ jobRequest.setFrom(fromConnector.getFrom());
+ jobRequest.setTo(toConnector.getTo());
+
+ addStandardJars(jobRequest);
+ addConnectorJars(jobRequest, fromConnector, toConnector, dataFormatClass);
+ addConnectorInitializerJars(jobRequest, Direction.FROM);
+ addConnectorInitializerJars(jobRequest, Direction.TO);
+
+ Schema fromSchema = getSchemaFromConnector(jobRequest, Direction.FROM);
+ Schema toSchema = getSchemaFromConnector(jobRequest, Direction.TO);
+
+ // TODO(Gwen): Need better logic here once the Schema refactor: SQOOP-1378
+ if (fromSchema != null) {
+ jobRequest.getSummary().setFromSchema(fromSchema);
+ }
+ else {
+ jobRequest.getSummary().setFromSchema(toSchema);
+ }
+ LOG.debug("Using entities: " + jobRequest.getFrom() + ", " + jobRequest.getTo());
+ return jobRequest;
+ }
+
+ private void addConnectorJars(JobRequest jobRequest, SqoopConnector fromConnector,
+ SqoopConnector toConnector, Class<? extends IntermediateDataFormat<?>> dataFormatClass) {
+ jobRequest.addJarForClass(fromConnector.getClass());
+ jobRequest.addJarForClass(toConnector.getClass());
+ jobRequest.addJarForClass(dataFormatClass);
+ }
+ private void addStandardJars(JobRequest jobRequest) {
// Let's register all important jars
// sqoop-common
- request.addJarForClass(MapContext.class);
+ jobRequest.addJarForClass(MapContext.class);
// sqoop-core
- request.addJarForClass(FrameworkManager.class);
+ jobRequest.addJarForClass(FrameworkManager.class);
// sqoop-spi
- request.addJarForClass(SqoopConnector.class);
+ jobRequest.addJarForClass(SqoopConnector.class);
// Execution engine jar
- request.addJarForClass(executionEngine.getClass());
- // Connectors in use
- request.addJarForClass(fromConnector.getClass());
- request.addJarForClass(toConnector.getClass());
-
+ jobRequest.addJarForClass(executionEngine.getClass());
// Extra libraries that Sqoop code requires
- request.addJarForClass(JSONValue.class);
-
- // The IDF is used in the ETL process.
- request.addJarForClass(dataFormatClass);
-
-
- // Get callbacks
- request.setFromCallback(fromConnector.getFrom());
- request.setToCallback(toConnector.getTo());
- LOG.debug("Using callbacks: " + request.getFromCallback() + ", " + request.getToCallback());
-
- // Initialize submission from fromConnector perspective
- CallbackBase[] baseCallbacks = {
- request.getFromCallback(),
- request.getToCallback()
- };
+ jobRequest.addJarForClass(JSONValue.class);
+ }
- CallbackBase baseCallback;
- Class<? extends Initializer> initializerClass;
- Initializer initializer;
- InitializerContext initializerContext;
+ MSubmission createJobSubmission(HttpEventContext ctx, long jobId) {
+ MSubmission summary = new MSubmission(jobId);
+ summary.setCreationUser(ctx.getUsername());
+ summary.setLastUpdateUser(ctx.getUsername());
+ return summary;
+ }
- // Initialize From Connector callback.
- baseCallback = request.getFromCallback();
+ SqoopConnector getConnector(long connnectorId) {
+ return ConnectorManager.getInstance().getConnector(connnectorId);
+ }
- initializerClass = baseCallback
- .getInitializer();
- initializer = (Initializer) ClassUtils
- .instantiate(initializerClass);
+ void validateSupportedDirection(SqoopConnector connector, Direction direction) {
+ // Make sure that connector supports the given direction
+ if (!connector.getSupportedDirections().contains(direction)) {
+ throw new SqoopException(FrameworkError.FRAMEWORK_0011, "Connector: "
+ + connector.getClass().getCanonicalName());
+ }
+ }
- if (initializer == null) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0006,
- "Can't create initializer instance: " + initializerClass.getName());
+ MConnection getConnection(long connectionId) {
+ MConnection connection = RepositoryManager.getInstance().getRepository()
+ .findConnection(connectionId);
+ if (!connection.getEnabled()) {
+ throw new SqoopException(FrameworkError.FRAMEWORK_0010, "Connection id: "
+ + connection.getPersistenceId());
}
+ return connection;
+ }
- // Initializer context
- initializerContext = new InitializerContext(request.getConnectorContext(Direction.FROM));
+ MJob getJob(long jobId) {
+ MJob job = RepositoryManager.getInstance().getRepository().findJob(jobId);
+ if (job == null) {
+ throw new SqoopException(FrameworkError.FRAMEWORK_0004, "Unknown job id: " + jobId);
+ }
- // Initialize submission from fromConnector perspective
- initializer.initialize(initializerContext,
- request.getConnectorConnectionConfig(Direction.FROM),
- request.getConnectorJobConfig(Direction.FROM));
+ if (!job.getEnabled()) {
+ throw new SqoopException(FrameworkError.FRAMEWORK_0009, "Job id: " + job.getPersistenceId());
+ }
+ return job;
+ }
+
+ private Schema getSchemaFromConnector(JobRequest jobRequest, Direction direction) {
- // Add job specific jars to
- request.addJars(initializer.getJars(initializerContext,
- request.getConnectorConnectionConfig(Direction.FROM),
- request.getConnectorJobConfig(Direction.FROM)));
+ Initializer initializer = getConnectorInitializer(jobRequest, direction);
- // @TODO(Abe): Alter behavior of Schema here. Need from Schema.
+ // Initializer context
+ InitializerContext initializerContext = getInitializerContext(jobRequest, direction);
+ // Initialize submission from the connector perspective
+ initializer.initialize(initializerContext, jobRequest.getConnectorConnectionConfig(direction),
+ jobRequest.getConnectorJobConfig(direction));
- Schema fromSchema = initializer.getSchema(initializerContext,
- request.getConnectorConnectionConfig(Direction.FROM),
- request.getConnectorJobConfig(Direction.FROM));
+ // TODO(Abe): Alter behavior of Schema here.
+ return initializer.getSchema(initializerContext,
+ jobRequest.getConnectorConnectionConfig(direction),
+ jobRequest.getConnectorJobConfig(direction));
+ }
- // request.getSummary().setConnectorSchema(initializer.getSchema(
- // initializerContext,
- // request.getConnectorConnectionConfig(ConnectorType.FROM),
- // request.getConnectorJobConfig(ConnectorType.FROM)
- // ));
+ private void addConnectorInitializerJars(JobRequest jobRequest, Direction direction) {
- // Initialize To Connector callback.
- baseCallback = request.getToCallback();
+ Initializer initializer = getConnectorInitializer(jobRequest, direction);
+ InitializerContext initializerContext = getInitializerContext(jobRequest, direction);
+ // Add job specific jars to
+ jobRequest.addJars(initializer.getJars(initializerContext,
+ jobRequest.getConnectorConnectionConfig(direction),
+ jobRequest.getConnectorJobConfig(direction)));
+ }
- initializerClass = baseCallback
- .getInitializer();
- initializer = (Initializer) ClassUtils
- .instantiate(initializerClass);
+ private Initializer getConnectorInitializer(JobRequest jobRequest, Direction direction) {
+ Transferable transferable = direction.equals(Direction.FROM) ? jobRequest.getFrom() : jobRequest.getTo();
+ Class<? extends Initializer> initializerClass = transferable.getInitializer();
+ Initializer initializer = (Initializer) ClassUtils.instantiate(initializerClass);
if (initializer == null) {
throw new SqoopException(FrameworkError.FRAMEWORK_0006,
- "Can't create initializer instance: " + initializerClass.getName());
- }
-
- // Initializer context
- initializerContext = new InitializerContext(request.getConnectorContext(Direction.TO));
-
- // Initialize submission from fromConnector perspective
- initializer.initialize(initializerContext,
- request.getConnectorConnectionConfig(Direction.TO),
- request.getConnectorJobConfig(Direction.TO));
-
- // Add job specific jars to
- request.addJars(initializer.getJars(initializerContext,
- request.getConnectorConnectionConfig(Direction.TO),
- request.getConnectorJobConfig(Direction.TO)));
-
- // @TODO(Abe): Alter behavior of Schema here. Need To Schema.
-
- Schema toSchema = initializer.getSchema(initializerContext,
- request.getConnectorConnectionConfig(Direction.TO),
- request.getConnectorJobConfig(Direction.TO));
-
- // Retrieve and persist the schema
-// request.getSummary().setConnectorSchema(initializer.getSchema(
-// initializerContext,
-// request.getConnectorConnectionConfig(ConnectorType.TO),
-// request.getConnectorJobConfig(ConnectorType.TO)
-// ));
-
- //TODO: Need better logic here
- if (fromSchema != null)
- request.getSummary().setConnectorSchema(fromSchema);
- else
- request.getSummary().setConnectorSchema(toSchema);
-
- // Bootstrap job from framework perspective
- prepareSubmission(request);
-
- // Make sure that this job id is not currently running and submit the job
- // only if it's not.
- synchronized (getClass()) {
- MSubmission lastSubmission = repository.findSubmissionLastForJob(jobId);
- if (lastSubmission != null && lastSubmission.getStatus().isRunning()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0002,
- "Job with id " + jobId);
- }
-
- // @TODO(Abe): Call multiple destroyers.
- // TODO(jarcec): We might need to catch all exceptions here to ensure
- // that Destroyer will be executed in all cases.
- boolean submitted = submissionEngine.submit(request);
- if (!submitted) {
- destroySubmission(request);
- summary.setStatus(SubmissionStatus.FAILURE_ON_SUBMIT);
- }
-
- repository.createSubmission(summary);
+ "Can't create connector initializer instance: " + initializerClass.getName());
}
-
- // Return job status most recent
- return summary;
+ return initializer;
}
- private void prepareSubmission(SubmissionRequest request) {
- JobConfiguration jobConfiguration = (JobConfiguration) request
- .getConfigFrameworkJob();
+ private InitializerContext getInitializerContext(JobRequest jobRequest, Direction direction) {
+ return new InitializerContext(jobRequest.getConnectorContext(direction));
+ }
+ void prepareJob(JobRequest request) {
+ JobConfiguration jobConfiguration = (JobConfiguration) request.getConfigFrameworkJob();
// We're directly moving configured number of extractors and loaders to
// underlying request object. In the future we might need to throttle this
// count based on other running jobs to meet our SLAs.
@@ -531,19 +497,19 @@ public class JobManager implements Reconfigurable {
request.setLoaders(jobConfiguration.throttling.loaders);
// Delegate rest of the job to execution engine
- executionEngine.prepareSubmission(request);
+ executionEngine.prepareJob(request);
}
/**
* Callback that will be called only if we failed to submit the job to the
* remote cluster.
*/
- private void destroySubmission(SubmissionRequest request) {
- CallbackBase fromCallback = request.getFromCallback();
- CallbackBase toCallback = request.getToCallback();
+ void destroySubmission(JobRequest request) {
+ Transferable from = request.getFrom();
+ Transferable to = request.getTo();
- Class<? extends Destroyer> fromDestroyerClass = fromCallback.getDestroyer();
- Class<? extends Destroyer> toDestroyerClass = toCallback.getDestroyer();
+ Class<? extends Destroyer> fromDestroyerClass = from.getDestroyer();
+ Class<? extends Destroyer> toDestroyerClass = to.getDestroyer();
Destroyer fromDestroyer = (Destroyer) ClassUtils.instantiate(fromDestroyerClass);
Destroyer toDestroyer = (Destroyer) ClassUtils.instantiate(toDestroyerClass);
@@ -557,15 +523,15 @@ public class JobManager implements Reconfigurable {
"Can't create toDestroyer instance: " + toDestroyerClass.getName());
}
- // @TODO(Abe): Update context to manage multiple connectors. As well as summary.
+ // TODO(Abe): Update context to manage multiple connectors. As well as summary.
DestroyerContext fromDestroyerContext = new DestroyerContext(
request.getConnectorContext(Direction.FROM), false, request.getSummary()
- .getConnectorSchema());
+ .getFromSchema());
DestroyerContext toDestroyerContext = new DestroyerContext(
request.getConnectorContext(Direction.TO), false, request.getSummary()
- .getConnectorSchema());
+ .getToSchema());
- // Initialize submission from connector perspective
+ // destroy submission from connector perspective
fromDestroyer.destroy(fromDestroyerContext, request.getConnectorConnectionConfig(Direction.FROM),
request.getConnectorJobConfig(Direction.FROM));
toDestroyer.destroy(toDestroyerContext, request.getConnectorConnectionConfig(Direction.TO),
@@ -573,42 +539,39 @@ public class JobManager implements Reconfigurable {
}
public MSubmission stop(long jobId, HttpEventContext ctx) {
- String username = ctx.getUsername();
Repository repository = RepositoryManager.getInstance().getRepository();
- MSubmission submission = repository.findSubmissionLastForJob(jobId);
+ MSubmission mSubmission = repository.findSubmissionLastForJob(jobId);
- if (submission == null || !submission.getStatus().isRunning()) {
- throw new SqoopException(FrameworkError.FRAMEWORK_0003,
- "Job with id " + jobId + " is not running");
+ if (mSubmission == null || !mSubmission.getStatus().isRunning()) {
+ throw new SqoopException(FrameworkError.FRAMEWORK_0003, "Job with id " + jobId
+ + " is not running");
}
+ submissionEngine.stop(mSubmission.getExternalId());
- String externalId = submission.getExternalId();
- submissionEngine.stop(externalId);
-
- submission.setLastUpdateUser(username);
+ mSubmission.setLastUpdateUser(ctx.getUsername());
// Fetch new information to verify that the stop command has actually worked
- update(submission);
+ update(mSubmission);
// Return updated structure
- return submission;
+ return mSubmission;
}
public MSubmission status(long jobId) {
Repository repository = RepositoryManager.getInstance().getRepository();
- MSubmission submission = repository.findSubmissionLastForJob(jobId);
+ MSubmission mSubmission = repository.findSubmissionLastForJob(jobId);
- if (submission == null) {
+ if (mSubmission == null) {
return new MSubmission(jobId, new Date(), SubmissionStatus.NEVER_EXECUTED);
}
// If the submission is in running state, let's update it
- if (submission.getStatus().isRunning()) {
- update(submission);
+ if (mSubmission.getStatus().isRunning()) {
+ update(mSubmission);
}
- return submission;
+ return mSubmission;
}
private void update(MSubmission submission) {
@@ -744,4 +707,4 @@ public class JobManager implements Reconfigurable {
LOG.info("Ending submission manager update thread");
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/main/java/org/apache/sqoop/framework/JobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/JobRequest.java b/core/src/main/java/org/apache/sqoop/framework/JobRequest.java
new file mode 100644
index 0000000..1f77693
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/framework/JobRequest.java
@@ -0,0 +1,356 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.framework;
+
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.DirectionError;
+import org.apache.sqoop.common.MutableMapContext;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.idf.IntermediateDataFormat;
+import org.apache.sqoop.connector.spi.SqoopConnector;
+import org.apache.sqoop.job.etl.Transferable;
+import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.utils.ClassUtils;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Submission details class is used when creating new submission and contains
+ * all information that we need to create a new submission (including mappers,
+ * reducers, ...).
+ */
+public class JobRequest {
+
+ /**
+ * Submission summary
+ */
+ MSubmission summary;
+
+ /**
+ * Original job name
+ */
+ String jobName;
+
+ /**
+ * Associated job (from metadata perspective) id
+ */
+ long jobId;
+
+ /**
+ * Connector instances associated with this submission request
+ */
+ SqoopConnector fromConnector;
+ SqoopConnector toConnector;
+
+ /**
+ * List of required local jars for the job
+ */
+ List<String> jars;
+
+ /**
+ * From entity
+ */
+ Transferable from;
+
+ /**
+ * To entity
+ */
+ Transferable to;
+
+ /**
+ * All configuration objects
+ */
+ Object fromConnectorConnectionConfig;
+ Object toConnectorConnectionConfig;
+ Object fromConnectorJobConfig;
+ Object toConnectorJobConfig;
+ Object fromFrameworkConnectionConfig;
+ Object toFrameworkConnectionConfig;
+ Object configFrameworkJob;
+
+ /**
+ * Connector context (submission specific configuration)
+ */
+ MutableMapContext fromConnectorContext;
+ MutableMapContext toConnectorContext;
+
+ /**
+ * Framework context (submission specific configuration)
+ */
+ MutableMapContext frameworkContext;
+
+ /**
+ * Optional notification URL for job progress
+ */
+ String notificationUrl;
+
+ /**
+ * Number of extractors
+ */
+ Integer extractors;
+
+ /**
+ * Number of loaders
+ */
+ Integer loaders;
+
+ /**
+ * The intermediate data format this submission should use.
+ */
+ Class<? extends IntermediateDataFormat> intermediateDataFormat;
+
+ public JobRequest() {
+ this.jars = new LinkedList<String>();
+ this.fromConnectorContext = new MutableMapContext();
+ this.toConnectorContext = new MutableMapContext();
+ this.frameworkContext = new MutableMapContext();
+ this.fromConnector = null;
+ this.toConnector = null;
+ this.fromConnectorConnectionConfig = null;
+ this.toConnectorConnectionConfig = null;
+ this.fromConnectorJobConfig = null;
+ this.toConnectorJobConfig = null;
+ this.fromFrameworkConnectionConfig = null;
+ this.toFrameworkConnectionConfig = null;
+ }
+
+ public MSubmission getSummary() {
+ return summary;
+ }
+
+ public void setSummary(MSubmission summary) {
+ this.summary = summary;
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ public void setJobName(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public long getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(long jobId) {
+ this.jobId = jobId;
+ }
+
+ public SqoopConnector getConnector(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromConnector;
+
+ case TO:
+ return toConnector;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public void setConnector(Direction type, SqoopConnector connector) {
+ switch(type) {
+ case FROM:
+ fromConnector = connector;
+ break;
+
+ case TO:
+ toConnector = connector;
+ break;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public List<String> getJars() {
+ return jars;
+ }
+
+ public void addJar(String jar) {
+ if(!jars.contains(jar)) {
+ jars.add(jar);
+ }
+ }
+
+ public void addJarForClass(Class klass) {
+ addJar(ClassUtils.jarForClass(klass));
+ }
+
+ public void addJars(List<String> jars) {
+ for(String j : jars) {
+ addJar(j);
+ }
+ }
+
+ public Transferable getFrom() {
+ return from;
+ }
+
+ public void setFrom(Transferable from) {
+ this.from = from;
+ }
+
+ public Transferable getTo() {
+ return to;
+ }
+
+ public void setTo(Transferable to) {
+ this.to = to;
+ }
+
+ public Object getConnectorConnectionConfig(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromConnectorConnectionConfig;
+
+ case TO:
+ return toConnectorConnectionConfig;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public void setConnectorConnectionConfig(Direction type, Object config) {
+ switch(type) {
+ case FROM:
+ fromConnectorConnectionConfig = config;
+ break;
+ case TO:
+ toConnectorConnectionConfig = config;
+ break;
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public Object getConnectorJobConfig(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromConnectorJobConfig;
+
+ case TO:
+ return toConnectorJobConfig;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public void setConnectorJobConfig(Direction type, Object config) {
+ switch(type) {
+ case FROM:
+ fromConnectorJobConfig = config;
+ break;
+ case TO:
+ toConnectorJobConfig = config;
+ break;
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public Object getFrameworkConnectionConfig(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromFrameworkConnectionConfig;
+
+ case TO:
+ return toFrameworkConnectionConfig;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public void setFrameworkConnectionConfig(Direction type, Object config) {
+ switch(type) {
+ case FROM:
+ fromFrameworkConnectionConfig = config;
+ break;
+ case TO:
+ toFrameworkConnectionConfig = config;
+ break;
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public Object getConfigFrameworkJob() {
+ return configFrameworkJob;
+ }
+
+ public void setConfigFrameworkJob(Object config) {
+ configFrameworkJob = config;
+ }
+
+ public MutableMapContext getConnectorContext(Direction type) {
+ switch(type) {
+ case FROM:
+ return fromConnectorContext;
+
+ case TO:
+ return toConnectorContext;
+
+ default:
+ throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
+ }
+ }
+
+ public MutableMapContext getFrameworkContext() {
+ return frameworkContext;
+ }
+
+ public String getNotificationUrl() {
+ return notificationUrl;
+ }
+
+ public void setNotificationUrl(String url) {
+ this.notificationUrl = url;
+ }
+
+ public Integer getExtractors() {
+ return extractors;
+ }
+
+ public void setExtractors(Integer extractors) {
+ this.extractors = extractors;
+ }
+
+ public Integer getLoaders() {
+ return loaders;
+ }
+
+ public void setLoaders(Integer loaders) {
+ this.loaders = loaders;
+ }
+
+ public Class<? extends IntermediateDataFormat> getIntermediateDataFormat() {
+ return intermediateDataFormat;
+ }
+
+ public void setIntermediateDataFormat(Class<? extends IntermediateDataFormat> intermediateDataFormat) {
+ this.intermediateDataFormat = intermediateDataFormat;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java b/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java
index 3c0f6eb..732be3b 100644
--- a/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java
+++ b/core/src/main/java/org/apache/sqoop/framework/SubmissionEngine.java
@@ -22,8 +22,8 @@ import org.apache.sqoop.submission.counter.Counters;
import org.apache.sqoop.submission.SubmissionStatus;
/**
- * Submission engine is capable of executing and getting information about
- * submissions to remote (hadoop) cluster.
+ * Submission engine is responsible in conveying the information about the
+ * job instances (submissions) to remote (hadoop) cluster.
*/
public abstract class SubmissionEngine {
@@ -31,6 +31,7 @@ public abstract class SubmissionEngine {
* Initialize submission engine
*
* @param context Configuration context
+ * @param prefix Submission engine prefix
*/
public void initialize(MapContext context, String prefix) {
}
@@ -57,7 +58,7 @@ public abstract class SubmissionEngine {
*
* @return Return true if we were able to submit job to remote cluster.
*/
- public abstract boolean submit(SubmissionRequest submission);
+ public abstract boolean submit(JobRequest submission);
/**
* Hard stop for given submission.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java b/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java
deleted file mode 100644
index bf3f785..0000000
--- a/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.common.DirectionError;
-import org.apache.sqoop.common.MutableMapContext;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.idf.IntermediateDataFormat;
-import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.job.etl.CallbackBase;
-import org.apache.sqoop.model.MSubmission;
-import org.apache.sqoop.utils.ClassUtils;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Submission details class is used when creating new submission and contains
- * all information that we need to create a new submission (including mappers,
- * reducers, ...).
- */
-public class SubmissionRequest {
-
- /**
- * Submission summary
- */
- MSubmission summary;
-
- /**
- * Original job name
- */
- String jobName;
-
- /**
- * Associated job (from metadata perspective) id
- */
- long jobId;
-
- /**
- * Connector instances associated with this submission request
- */
- SqoopConnector fromConnector;
- SqoopConnector toConnector;
-
- /**
- * List of required local jars for the job
- */
- List<String> jars;
-
- /**
- * From connector callback
- */
- CallbackBase fromCallback;
-
- /**
- * To connector callback
- */
- CallbackBase toCallback;
-
- /**
- * All configuration objects
- */
- Object fromConnectorConnectionConfig;
- Object toConnectorConnectionConfig;
- Object fromConnectorJobConfig;
- Object toConnectorJobConfig;
- Object fromFrameworkConnectionConfig;
- Object toFrameworkConnectionConfig;
- Object configFrameworkJob;
-
- /**
- * Connector context (submission specific configuration)
- */
- MutableMapContext fromConnectorContext;
- MutableMapContext toConnectorContext;
-
- /**
- * Framework context (submission specific configuration)
- */
- MutableMapContext frameworkContext;
-
- /**
- * HDFS output directory
- */
- String outputDirectory;
-
- /**
- * Optional notification URL for job progress
- */
- String notificationUrl;
-
- /**
- * Number of extractors
- */
- Integer extractors;
-
- /**
- * Number of loaders
- */
- Integer loaders;
-
- /**
- * The intermediate data format this submission should use.
- */
- Class<? extends IntermediateDataFormat> intermediateDataFormat;
-
- public SubmissionRequest() {
- this.jars = new LinkedList<String>();
- this.fromConnectorContext = new MutableMapContext();
- this.toConnectorContext = new MutableMapContext();
- this.frameworkContext = new MutableMapContext();
- this.fromConnector = null;
- this.toConnector = null;
- this.fromConnectorConnectionConfig = null;
- this.toConnectorConnectionConfig = null;
- this.fromConnectorJobConfig = null;
- this.toConnectorJobConfig = null;
- this.fromFrameworkConnectionConfig = null;
- this.toFrameworkConnectionConfig = null;
- }
-
- public MSubmission getSummary() {
- return summary;
- }
-
- public void setSummary(MSubmission summary) {
- this.summary = summary;
- }
-
- public String getJobName() {
- return jobName;
- }
-
- public void setJobName(String jobName) {
- this.jobName = jobName;
- }
-
- public long getJobId() {
- return jobId;
- }
-
- public void setJobId(long jobId) {
- this.jobId = jobId;
- }
-
- public SqoopConnector getConnector(Direction type) {
- switch(type) {
- case FROM:
- return fromConnector;
-
- case TO:
- return toConnector;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setConnector(Direction type, SqoopConnector connector) {
- switch(type) {
- case FROM:
- fromConnector = connector;
- break;
-
- case TO:
- toConnector = connector;
- break;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public List<String> getJars() {
- return jars;
- }
-
- public void addJar(String jar) {
- if(!jars.contains(jar)) {
- jars.add(jar);
- }
- }
-
- public void addJarForClass(Class klass) {
- addJar(ClassUtils.jarForClass(klass));
- }
-
- public void addJars(List<String> jars) {
- for(String j : jars) {
- addJar(j);
- }
- }
-
- public CallbackBase getFromCallback() {
- return fromCallback;
- }
-
- public void setFromCallback(CallbackBase fromCallback) {
- this.fromCallback = fromCallback;
- }
-
- public CallbackBase getToCallback() {
- return toCallback;
- }
-
- public void setToCallback(CallbackBase toCallback) {
- this.toCallback = toCallback;
- }
-
- public Object getConnectorConnectionConfig(Direction type) {
- switch(type) {
- case FROM:
- return fromConnectorConnectionConfig;
-
- case TO:
- return toConnectorConnectionConfig;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setConnectorConnectionConfig(Direction type, Object config) {
- switch(type) {
- case FROM:
- fromConnectorConnectionConfig = config;
- break;
- case TO:
- toConnectorConnectionConfig = config;
- break;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public Object getConnectorJobConfig(Direction type) {
- switch(type) {
- case FROM:
- return fromConnectorJobConfig;
-
- case TO:
- return toConnectorJobConfig;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setConnectorJobConfig(Direction type, Object config) {
- switch(type) {
- case FROM:
- fromConnectorJobConfig = config;
- break;
- case TO:
- toConnectorJobConfig = config;
- break;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public Object getFrameworkConnectionConfig(Direction type) {
- switch(type) {
- case FROM:
- return fromFrameworkConnectionConfig;
-
- case TO:
- return toFrameworkConnectionConfig;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setFrameworkConnectionConfig(Direction type, Object config) {
- switch(type) {
- case FROM:
- fromFrameworkConnectionConfig = config;
- break;
- case TO:
- toFrameworkConnectionConfig = config;
- break;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public Object getConfigFrameworkJob() {
- return configFrameworkJob;
- }
-
- public void setConfigFrameworkJob(Object config) {
- configFrameworkJob = config;
- }
-
- public MutableMapContext getConnectorContext(Direction type) {
- switch(type) {
- case FROM:
- return fromConnectorContext;
-
- case TO:
- return toConnectorContext;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public MutableMapContext getFrameworkContext() {
- return frameworkContext;
- }
-
- public String getNotificationUrl() {
- return notificationUrl;
- }
-
- public void setNotificationUrl(String url) {
- this.notificationUrl = url;
- }
-
- public Integer getExtractors() {
- return extractors;
- }
-
- public void setExtractors(Integer extractors) {
- this.extractors = extractors;
- }
-
- public Integer getLoaders() {
- return loaders;
- }
-
- public void setLoaders(Integer loaders) {
- this.loaders = loaders;
- }
-
- public Class<? extends IntermediateDataFormat> getIntermediateDataFormat() {
- return intermediateDataFormat;
- }
-
- public void setIntermediateDataFormat(Class<? extends IntermediateDataFormat> intermediateDataFormat) {
- this.intermediateDataFormat = intermediateDataFormat;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java
index 69c1b56..69dd028 100644
--- a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java
+++ b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java
@@ -17,64 +17,140 @@
*/
package org.apache.sqoop.framework;
-import org.apache.sqoop.framework.configuration.ConnectionConfiguration;
-import org.apache.sqoop.framework.configuration.JobConfiguration;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.ValidationResult;
-import org.apache.sqoop.validation.ValidationRunner;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
/**
- *
+ * NOTE(VB): This test class will soon be removed with the Validator refactoring
*/
public class TestFrameworkValidator {
- FrameworkValidator validator;
-
- @Before
- public void setUp() {
- validator = new FrameworkValidator();
- }
-
- @Test
- public void testConnectionValidation() {
- ConnectionConfiguration connectionConfiguration = new ConnectionConfiguration();
- ValidationRunner runner = new ValidationRunner();
- ValidationResult result = runner.validate(connectionConfiguration);
- assertEquals(Status.FINE, result.getStatus());
- assertEquals(0, result.getMessages().size());
- }
-
- @Test
- public void testJobValidation() {
- ValidationRunner runner = new ValidationRunner();
- ValidationResult result;
- JobConfiguration configuration;
-
- // Empty form is allowed
- configuration = new JobConfiguration();
- result = runner.validate(configuration);
- assertEquals(Status.FINE, result.getStatus());
-
- // Explicitly setting extractors and loaders
- configuration = new JobConfiguration();
- configuration.throttling.extractors = 3;
- configuration.throttling.loaders = 3;
- result = runner.validate(configuration);
- assertEquals(Status.FINE, result.getStatus());
- assertEquals(0, result.getMessages().size());
-
- // Negative and zero values for extractors and loaders
-// configuration = new JobConfiguration();
+// FrameworkValidator validator;
+//
+// @Before
+// public void setUp() {
+// validator = new FrameworkValidator();
+// }
+//
+// @Test
+// public void testConnectionValidation() {
+// ConnectionConfiguration connectionConfiguration = new ConnectionConfiguration();
+//
+// Validation validation = validator.validateConnection(connectionConfiguration);
+// assertEquals(Status.FINE, validation.getStatus());
+// assertEquals(0, validation.getMessages().size());
+// }
+//
+// @Test
+// public void testExportJobValidation() {
+// ExportJobConfiguration configuration;
+// Validation validation;
+//
+// // Empty form is not allowed
+// configuration = new ExportJobConfiguration();
+// validation = validator.validateJob(MJob.Type.EXPORT, configuration);
+// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("input.inputDirectory")));
+//
+// // Explicitly setting extractors and loaders
+// configuration = new ExportJobConfiguration();
+// configuration.input.inputDirectory = "/czech/republic";
+// configuration.throttling.extractors = 3;
+// configuration.throttling.loaders = 3;
+//
+// validation = validator.validateJob(MJob.Type.EXPORT, configuration);
+// assertEquals(Status.FINE, validation.getStatus());
+// assertEquals(0, validation.getMessages().size());
+//
+// // Negative and zero values for extractors and loaders
+// configuration = new ExportJobConfiguration();
+// configuration.input.inputDirectory = "/czech/republic";
+// configuration.throttling.extractors = 0;
+// configuration.throttling.loaders = -1;
+//
+// validation = validator.validateJob(MJob.Type.EXPORT, configuration);
+// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.extractors")));
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.loaders")));
+// }
+//
+//
+// @Test
+// public void testImportJobValidation() {
+// ImportJobConfiguration configuration;
+// Validation validation;
+//
+// // Empty form is not allowed
+// configuration = new ImportJobConfiguration();
+// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
+// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.outputDirectory")));
+//
+// // Explicitly setting extractors and loaders
+// configuration = new ImportJobConfiguration();
+// configuration.output.outputDirectory = "/czech/republic";
+// configuration.throttling.extractors = 3;
+// configuration.throttling.loaders = 3;
+//
+// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
+// assertEquals(Status.FINE, validation.getStatus());
+// assertEquals(0, validation.getMessages().size());
+//
+// // Negative and zero values for extractors and loaders
+// configuration = new ImportJobConfiguration();
+// configuration.output.outputDirectory = "/czech/republic";
// configuration.throttling.extractors = 0;
// configuration.throttling.loaders = -1;
-// result = runner.validate(configuration);
-// assertEquals(Status.FINE, result.getStatus());
-// assertTrue(result.getMessages().containsKey("throttling.extractors"));
-// assertTrue(result.getMessages().containsKey("throttling.loaders"));
- }
+//
+// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
+// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.extractors")));
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.loaders")));
+//
+// // specifying both compression as well as customCompression is
+// // unacceptable
+// configuration = new ImportJobConfiguration();
+// configuration.output.outputDirectory = "/czech/republic";
+// configuration.throttling.extractors = 2;
+// configuration.throttling.loaders = 2;
+// configuration.output.compression = OutputCompression.BZIP2;
+// configuration.output.customCompression = "some.compression.codec";
+//
+// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
+// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression")));
+//
+// // specifying a customCompression is fine
+// configuration = new ImportJobConfiguration();
+// configuration.output.outputDirectory = "/czech/republic";
+// configuration.throttling.extractors = 2;
+// configuration.throttling.loaders = 2;
+// configuration.output.compression = OutputCompression.CUSTOM;
+// configuration.output.customCompression = "some.compression.codec";
+//
+// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
+// assertEquals(Status.FINE, validation.getStatus());
+//
+// // specifying a customCompression without codec name is unacceptable
+// configuration = new ImportJobConfiguration();
+// configuration.output.outputDirectory = "/czech/republic";
+// configuration.throttling.extractors = 2;
+// configuration.throttling.loaders = 2;
+// configuration.output.compression = OutputCompression.CUSTOM;
+// configuration.output.customCompression = "";
+//
+// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
+// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression")));
+//
+// configuration = new ImportJobConfiguration();
+// configuration.output.outputDirectory = "/czech/republic";
+// configuration.throttling.extractors = 2;
+// configuration.throttling.loaders = 2;
+// configuration.output.compression = OutputCompression.CUSTOM;
+// configuration.output.customCompression = null;
+//
+// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
+// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
+// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression")));
+//
+// }
}
[24/52] [abbrv] git commit: SQOOP-1454: Sqoop2: From/To: Add client
support for supported directions
Posted by ab...@apache.org.
SQOOP-1454: Sqoop2: From/To: Add client support for supported directions
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/27fb31d4
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/27fb31d4
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/27fb31d4
Branch: refs/heads/SQOOP-1367
Commit: 27fb31d42e7e76ca14def568e454121437f4e9e5
Parents: 3d539dd
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Sun Sep 21 12:42:10 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:18 2014 -0700
----------------------------------------------------------------------
.../sqoop/common/SupportedDirections.java | 41 ++++++
.../java/org/apache/sqoop/model/MConnector.java | 6 +
.../sqoop/common/TestSupportedDirections.java | 55 ++++++++
.../org/apache/sqoop/model/TestMConnector.java | 127 ++++++++++++-------
.../sqoop/connector/ConnectorHandler.java | 4 -
.../sqoop/shell/ShowConnectorFunction.java | 37 +++++-
.../org/apache/sqoop/shell/core/Constants.java | 2 +
.../main/resources/shell-resource.properties | 5 +-
8 files changed, 226 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/27fb31d4/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java b/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java
new file mode 100644
index 0000000..25ba276
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.common;
+
+/**
+ * Represents which Directions are supported.
+ */
+public class SupportedDirections {
+ private boolean from;
+ private boolean to;
+
+ public SupportedDirections(boolean from, boolean to) {
+ this.from = from;
+ this.to = to;
+ }
+
+ /**
+ * Check if direction is supported.
+ * @param direction
+ * @return boolean
+ */
+ public boolean isDirectionSupported(Direction direction) {
+ return direction == Direction.FROM && from
+ || direction == Direction.TO && to;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/27fb31d4/common/src/main/java/org/apache/sqoop/model/MConnector.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConnector.java b/common/src/main/java/org/apache/sqoop/model/MConnector.java
index 335a0cc..3dc1014 100644
--- a/common/src/main/java/org/apache/sqoop/model/MConnector.java
+++ b/common/src/main/java/org/apache/sqoop/model/MConnector.java
@@ -20,6 +20,7 @@ package org.apache.sqoop.model;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.DirectionError;
import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.common.SupportedDirections;
/**
* Connector metadata.
@@ -139,4 +140,9 @@ public final class MConnector extends MPersistableEntity implements MClonable {
public void setVersion(String version) {
this.version = version;
}
+
+ public SupportedDirections getSupportedDirections() {
+ return new SupportedDirections(this.getJobForms(Direction.FROM) != null,
+ this.getJobForms(Direction.TO) != null);
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/27fb31d4/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java b/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java
new file mode 100644
index 0000000..4fbaf82
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.common;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestSupportedDirections {
+
+ @Test
+ public void testIsDirectionSupported() {
+ // Both
+ SupportedDirections supportedDirections = new SupportedDirections(true, true);
+ Assert.assertTrue(
+ supportedDirections.isDirectionSupported(Direction.FROM));
+ Assert.assertTrue(
+ supportedDirections.isDirectionSupported(Direction.TO));
+
+ // FROM
+ supportedDirections = new SupportedDirections(true, false);
+ Assert.assertTrue(
+ supportedDirections.isDirectionSupported(Direction.FROM));
+ Assert.assertFalse(
+ supportedDirections.isDirectionSupported(Direction.TO));
+
+ // TO
+ supportedDirections = new SupportedDirections(false, true);
+ Assert.assertFalse(
+ supportedDirections.isDirectionSupported(Direction.FROM));
+ Assert.assertTrue(
+ supportedDirections.isDirectionSupported(Direction.TO));
+
+ // NONE
+ supportedDirections = new SupportedDirections(false, false);
+ Assert.assertFalse(
+ supportedDirections.isDirectionSupported(Direction.FROM));
+ Assert.assertFalse(
+ supportedDirections.isDirectionSupported(Direction.TO));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/27fb31d4/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConnector.java b/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
index 9672d9d..3fde47b 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConnector.java
@@ -20,8 +20,10 @@ package org.apache.sqoop.model;
import static org.junit.Assert.*;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import org.apache.sqoop.common.Direction;
import org.junit.Test;
/**
@@ -29,6 +31,34 @@ import org.junit.Test;
*/
public class TestMConnector {
+ private MConnector createConnector(List<Direction> supportedDirections) {
+ List<MForm> forms = new ArrayList<MForm>();
+ MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
+ input.setValue(100);
+ MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
+ strInput.setValue("TEST-VALUE");
+ List<MInput<?>> list = new ArrayList<MInput<?>>();
+ list.add(input);
+ list.add(strInput);
+ MForm form = new MForm("FORMNAME", list);
+ forms.add(form);
+
+ MConnectionForms connectionForms1 = new MConnectionForms(forms);
+ MJobForms fromForm = null;
+ MJobForms toForm = null;
+
+ if (supportedDirections.contains(Direction.FROM)) {
+ fromForm = new MJobForms(forms);
+ }
+
+ if (supportedDirections.contains(Direction.TO)) {
+ toForm = new MJobForms(forms);
+ }
+
+ return new MConnector("NAME", "CLASSNAME", "1.0",
+ connectionForms1, fromForm, toForm);
+ }
+
/**
* Test for initialization
*/
@@ -65,47 +95,58 @@ public class TestMConnector {
}
}
-// @Test
-// public void testClone() {
-// List<MForm> forms = new ArrayList<MForm>();
-// MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
-// input.setValue(100);
-// MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
-// strInput.setValue("TEST-VALUE");
-// List<MInput<?>> list = new ArrayList<MInput<?>>();
-// list.add(input);
-// list.add(strInput);
-// MForm form = new MForm("FORMNAME", list);
-// forms.add(form);
-// MConnectionForms connectionForms1 = new MConnectionForms(forms);
-// MJobForms jobform1 = new MJobForms(MJob.Type.EXPORT, forms);
-// List<MJobForms> jobFormList = new ArrayList<MJobForms>();
-// jobFormList.add(jobform1);
-// MConnector connector1 = new MConnector("NAME", "CLASSNAME", "1.0",
-// connectionForms1, jobFormList);
-// assertEquals("NAME", connector1.getUniqueName());
-// assertEquals("CLASSNAME", connector1.getClassName());
-// assertEquals("1.0", connector1.getVersion());
-// //Clone with values. Checking values copying after the cloning. But form values will be null
-// MConnector clone1 = connector1.clone(true);
-// assertEquals("NAME", clone1.getUniqueName());
-// assertEquals("CLASSNAME", clone1.getClassName());
-// assertEquals("1.0", clone1.getVersion());
-// MForm clonedForm1 = clone1.getConnectionForms().getForms().get(0);
-// assertNull(clonedForm1.getInputs().get(0).getValue());
-// assertNull(clonedForm1.getInputs().get(1).getValue());
-//
-// MForm clonedForm2 = clone1.getJobForms(MJob.Type.EXPORT).getForms().get(0);
-// assertNull(clonedForm2.getInputs().get(0).getValue());
-// assertNull(clonedForm2.getInputs().get(1).getValue());
-//
-// //Clone without values. Inputs value will be null after cloning.
-// MConnector clone2 = connector1.clone(false);
-// clonedForm1 = clone2.getConnectionForms().getForms().get(0);
-// assertNull(clonedForm1.getInputs().get(0).getValue());
-// assertNull(clonedForm1.getInputs().get(1).getValue());
-// clonedForm2 = clone2.getJobForms(MJob.Type.EXPORT).getForms().get(0);
-// assertNull(clonedForm2.getInputs().get(0).getValue());
-// assertNull(clonedForm2.getInputs().get(1).getValue());
-// }
+ @Test
+ public void testClone() {
+ MConnector connector1 = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
+ assertEquals("NAME", connector1.getUniqueName());
+ assertEquals("CLASSNAME", connector1.getClassName());
+ assertEquals("1.0", connector1.getVersion());
+ //Clone with values. Checking values copying after the cloning. But form values will be null
+ MConnector clone1 = connector1.clone(true);
+ assertEquals("NAME", clone1.getUniqueName());
+ assertEquals("CLASSNAME", clone1.getClassName());
+ assertEquals("1.0", clone1.getVersion());
+ MForm clonedForm1 = clone1.getConnectionForms().getForms().get(0);
+ assertNull(clonedForm1.getInputs().get(0).getValue());
+ assertNull(clonedForm1.getInputs().get(1).getValue());
+
+ MForm clonedForm2 = clone1.getJobForms(Direction.FROM).getForms().get(0);
+ assertNull(clonedForm2.getInputs().get(0).getValue());
+ assertNull(clonedForm2.getInputs().get(1).getValue());
+
+ MForm clonedForm3 = clone1.getJobForms(Direction.TO).getForms().get(0);
+ assertNull(clonedForm3.getInputs().get(0).getValue());
+ assertNull(clonedForm3.getInputs().get(1).getValue());
+
+ //Clone without values. Inputs value will be null after cloning.
+ MConnector clone2 = connector1.clone(false);
+ clonedForm1 = clone2.getConnectionForms().getForms().get(0);
+ assertNull(clonedForm1.getInputs().get(0).getValue());
+ assertNull(clonedForm1.getInputs().get(1).getValue());
+ clonedForm2 = clone2.getJobForms(Direction.FROM).getForms().get(0);
+ assertNull(clonedForm2.getInputs().get(0).getValue());
+ assertNull(clonedForm2.getInputs().get(1).getValue());
+ clonedForm3 = clone2.getJobForms(Direction.TO).getForms().get(0);
+ assertNull(clonedForm3.getInputs().get(0).getValue());
+ assertNull(clonedForm3.getInputs().get(1).getValue());
+ }
+
+ @Test
+ public void testGetSupportedDirections() {
+ MConnector connector = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
+ assertTrue(connector.getSupportedDirections().isDirectionSupported(Direction.FROM));
+ assertTrue(connector.getSupportedDirections().isDirectionSupported(Direction.TO));
+
+ connector = createConnector(Arrays.asList(Direction.FROM));
+ assertTrue(connector.getSupportedDirections().isDirectionSupported(Direction.FROM));
+ assertFalse(connector.getSupportedDirections().isDirectionSupported(Direction.TO));
+
+ connector = createConnector(Arrays.asList(Direction.TO));
+ assertFalse(connector.getSupportedDirections().isDirectionSupported(Direction.FROM));
+ assertTrue(connector.getSupportedDirections().isDirectionSupported(Direction.TO));
+
+ connector = createConnector(Arrays.asList(new Direction[]{}));
+ assertFalse(connector.getSupportedDirections().isDirectionSupported(Direction.FROM));
+ assertFalse(connector.getSupportedDirections().isDirectionSupported(Direction.TO));
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/27fb31d4/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
index 8782209..6c10b54 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
@@ -98,15 +98,11 @@ public final class ConnectorHandler {
if (connector.getSupportedDirections().contains(Direction.FROM)) {
fromJobForms = new MJobForms(FormUtils.toForms(
connector.getJobConfigurationClass(Direction.FROM)));
- } else {
- fromJobForms = new MJobForms(new ArrayList<MForm>());
}
if (connector.getSupportedDirections().contains(Direction.TO)) {
toJobForms = new MJobForms(FormUtils.toForms(
connector.getJobConfigurationClass(Direction.TO)));
- } else {
- toJobForms = new MJobForms(new ArrayList<MForm>());
}
MConnectionForms connectionForms = new MConnectionForms(
http://git-wip-us.apache.org/repos/asf/sqoop/blob/27fb31d4/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
index bbfbb3f..50978ff 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
@@ -23,6 +23,8 @@ import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.SupportedDirections;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.shell.utils.TableDisplayer;
@@ -33,6 +35,8 @@ import static org.apache.sqoop.shell.utils.FormDisplayer.*;
@SuppressWarnings("serial")
public class ShowConnectorFunction extends SqoopFunction {
+ private static final char SUPPORTED_DIRECTIONS_SEPARATOR = '/';
+
@SuppressWarnings("static-access")
public ShowConnectorFunction() {
this.addOption(OptionBuilder
@@ -66,20 +70,23 @@ public class ShowConnectorFunction extends SqoopFunction {
header.add(resourceString(Constants.RES_TABLE_HEADER_NAME));
header.add(resourceString(Constants.RES_TABLE_HEADER_VERSION));
header.add(resourceString(Constants.RES_TABLE_HEADER_CLASS));
+ header.add(resourceString(Constants.RES_TABLE_HEADER_SUPPORTED_DIRECTIONS));
List<String> ids = new LinkedList<String>();
List<String> uniqueNames = new LinkedList<String>();
List<String> versions = new LinkedList<String>();
List<String> classes = new LinkedList<String>();
+ List<String> supportedDirections = new LinkedList<String>();
for(MConnector connector : connectors) {
ids.add(String.valueOf(connector.getPersistenceId()));
uniqueNames.add(connector.getUniqueName());
versions.add(connector.getVersion());
classes.add(connector.getClassName());
+ supportedDirections.add(getSupportedDirections(connector));
}
- TableDisplayer.display(header, ids, uniqueNames, versions, classes);
+ TableDisplayer.display(header, ids, uniqueNames, versions, classes, supportedDirections);
}
private void showConnectors() {
@@ -105,8 +112,34 @@ public class ShowConnectorFunction extends SqoopFunction {
connector.getPersistenceId(),
connector.getUniqueName(),
connector.getClassName(),
- connector.getVersion()
+ connector.getVersion(),
+ getSupportedDirections(connector)
);
displayFormMetadataDetails(connector, client.getResourceBundle(connector.getPersistenceId()));
}
+
+ /**
+ * Creates a nicely formatted string for which directions are supported.
+ * Example: FROM/TO.
+ * @param connector
+ * @return String
+ */
+ private String getSupportedDirections(MConnector connector) {
+ StringBuffer supportedDirectionsBuffer = new StringBuffer();
+ SupportedDirections supportedDirections
+ = connector.getSupportedDirections();
+
+ if (supportedDirections.isDirectionSupported(Direction.FROM)) {
+ supportedDirectionsBuffer.append(Direction.FROM);
+
+ if (supportedDirections.isDirectionSupported(Direction.TO)) {
+ supportedDirectionsBuffer.append(SUPPORTED_DIRECTIONS_SEPARATOR);
+ supportedDirectionsBuffer.append(Direction.TO);
+ }
+ } else if (supportedDirections.isDirectionSupported(Direction.TO)) {
+ supportedDirectionsBuffer.append(Direction.TO);
+ }
+
+ return supportedDirectionsBuffer.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/27fb31d4/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
index efabc46..f0dc3a6 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
@@ -333,6 +333,8 @@ public class Constants {
"table.header.version";
public static final String RES_TABLE_HEADER_CLASS =
"table.header.class";
+ public static final String RES_TABLE_HEADER_SUPPORTED_DIRECTIONS =
+ "table.header.supported_directions";
public static final String RES_TABLE_HEADER_CONNECTOR =
"table.header.connector";
public static final String RES_TABLE_HEADER_FROM_CONNECTOR =
http://git-wip-us.apache.org/repos/asf/sqoop/blob/27fb31d4/shell/src/main/resources/shell-resource.properties
----------------------------------------------------------------------
diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties
index 73a19e8..247ceae 100644
--- a/shell/src/main/resources/shell-resource.properties
+++ b/shell/src/main/resources/shell-resource.properties
@@ -137,8 +137,8 @@ show.prompt_display_all_connectors = Display all connectors
show.prompt_display_connector_cid = Display the connector with cid
show.connector_usage = Usage: show connector
show.prompt_connectors_to_show = @|bold {0} connector(s) to show: |@
-show.prompt_connector_info = Connector with id {0}:\n Name: {1} \n \
-Class: {2}\n Version: {3}
+show.prompt_connector_info = Connector with id {0}:\n Name: {1} \n \
+Class: {2}\n Version: {3}\n Supported Directions {4}
show.framework_usage = Usage: show framework
show.prompt_framework_opts = @|bold Framework specific options: |@\nPersistent id: {0}
@@ -190,6 +190,7 @@ table.header.id = Id
table.header.name = Name
table.header.version = Version
table.header.class = Class
+table.header.supported_directions = Supported Directions
table.header.connector = Connector
table.header.connector.from = From Connector
table.header.connector.to = To Connector
[34/52] [abbrv] git commit: SQOOP-1545: Sqoop2: From/To: Add
supported directions to Repository
Posted by ab...@apache.org.
SQOOP-1545: Sqoop2: From/To: Add supported directions to Repository
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/fc74316f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/fc74316f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/fc74316f
Branch: refs/heads/SQOOP-1367
Commit: fc74316fbdcee80272106064ac4e1e747c300b97
Parents: b600036
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Oct 9 08:36:35 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 18:22:51 2014 -0700
----------------------------------------------------------------------
.../sqoop/common/SupportedDirections.java | 79 +++-
.../sqoop/common/TestSupportedDirections.java | 19 +
.../sqoop/repository/derby/DerbyRepoError.java | 8 +
.../derby/DerbyRepositoryHandler.java | 398 ++++++++++++++++---
.../repository/derby/DerbySchemaConstants.java | 53 +++
.../repository/derby/DerbySchemaQuery.java | 164 +++++++-
.../sqoop/repository/derby/DerbyTestCase.java | 131 ++++--
.../repository/derby/TestConnectorHandling.java | 62 ++-
.../sqoop/repository/derby/TestJobHandling.java | 4 -
.../sqoop/shell/ShowConnectorFunction.java | 30 +-
.../sqoop/connector/spi/SqoopConnector.java | 2 +-
11 files changed, 808 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java b/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java
index 25ba276..c527117 100644
--- a/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java
+++ b/common/src/main/java/org/apache/sqoop/common/SupportedDirections.java
@@ -20,7 +20,9 @@ package org.apache.sqoop.common;
/**
* Represents which Directions are supported.
*/
-public class SupportedDirections {
+public class SupportedDirections implements Comparable<SupportedDirections> {
+ private static final char SUPPORTED_DIRECTIONS_SEPARATOR = '/';
+
private boolean from;
private boolean to;
@@ -38,4 +40,79 @@ public class SupportedDirections {
return direction == Direction.FROM && from
|| direction == Direction.TO && to;
}
+
+ /**
+ * @return String "FROM", "TO", "FROM/TO", "".
+ */
+ public String toString() {
+ StringBuffer buffer = new StringBuffer();
+
+ if (isDirectionSupported(Direction.FROM)) {
+ buffer.append(Direction.FROM);
+
+ if (isDirectionSupported(Direction.TO)) {
+ buffer.append(SUPPORTED_DIRECTIONS_SEPARATOR);
+ buffer.append(Direction.TO);
+ }
+ } else if (isDirectionSupported(Direction.TO)) {
+ buffer.append(Direction.TO);
+ }
+
+ return buffer.toString();
+ }
+
+ public static SupportedDirections fromString(String supportedDirections) {
+ boolean from = false, to = false;
+
+ if (supportedDirections != null && !supportedDirections.equals("")) {
+ for (String direction : supportedDirections.split("/")) {
+ switch (Direction.valueOf(direction)) {
+ case FROM:
+ from = true;
+ break;
+
+ case TO:
+ to = true;
+ break;
+ }
+ }
+ }
+
+ return new SupportedDirections(from, to);
+ }
+
+ public static SupportedDirections fromDirection(Direction direction) {
+ boolean from = false, to = false;
+ switch (direction) {
+ case FROM:
+ from = true;
+ break;
+
+ case TO:
+ to = true;
+ break;
+ }
+ return new SupportedDirections(from, to);
+ }
+
+ @Override
+ public int compareTo(SupportedDirections o) {
+ int hash = 0;
+ if (this.isDirectionSupported(Direction.FROM)) {
+ hash |= 1;
+ }
+ if (this.isDirectionSupported(Direction.TO)) {
+ hash |= 2;
+ }
+
+ int oHash = 0;
+ if (this.isDirectionSupported(Direction.FROM)) {
+ oHash |= 1;
+ }
+ if (this.isDirectionSupported(Direction.TO)) {
+ oHash |= 2;
+ }
+
+ return hash - oHash;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java b/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java
index 4fbaf82..4f0cdd6 100644
--- a/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java
+++ b/common/src/test/java/org/apache/sqoop/common/TestSupportedDirections.java
@@ -52,4 +52,23 @@ public class TestSupportedDirections {
Assert.assertFalse(
supportedDirections.isDirectionSupported(Direction.TO));
}
+
+ @Test
+ public void testToString() {
+ // Both
+ SupportedDirections supportedDirections = new SupportedDirections(true, true);
+ Assert.assertEquals("FROM/TO", supportedDirections.toString());
+
+ // FROM
+ supportedDirections = new SupportedDirections(true, false);
+ Assert.assertEquals("FROM", supportedDirections.toString());
+
+ // TO
+ supportedDirections = new SupportedDirections(false, true);
+ Assert.assertEquals("TO", supportedDirections.toString());
+
+ // NONE
+ supportedDirections = new SupportedDirections(false, false);
+ Assert.assertEquals("", supportedDirections.toString());
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
index cc31d06..0f0f7c4 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java
@@ -181,6 +181,14 @@ public enum DerbyRepoError implements ErrorCode {
DERBYREPO_0044("Update of driver config failed"),
DERBYREPO_0045("Can't retrieve all connectors"),
+
+ DERBYREPO_0046("Could not add directions"),
+
+ DERBYREPO_0047("Could not get ID of recently added direction"),
+
+ DERBYREPO_0048("Could not register config direction"),
+
+ DERBYREPO_0049("Could not set connector direction")
;
private final String message;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index 73d8387..10a7b1a 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -33,13 +33,16 @@ import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.DirectionError;
import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.common.SupportedDirections;
import org.apache.sqoop.connector.ConnectorHandler;
import org.apache.sqoop.connector.ConnectorManagerUtils;
import org.apache.sqoop.model.MBooleanInput;
@@ -88,6 +91,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
*/
private static final String CONNECTOR_HDFS = "hdfs-connector";
+ private static final String LINK_HDFS = "hdfs-link";
+
private JdbcRepositoryContext repoContext;
/**
@@ -121,7 +126,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
// Register the job config type, since driver config is per job
registerConfigs(null, null, mDriver.getDriverConfig().getConfigs(),
- MConfigType.JOB.name(), baseConfigStmt, baseInputStmt);
+ MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn);
} catch (SQLException ex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0014, mDriver.toString(), ex);
@@ -150,14 +155,17 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
// Register link type config
registerConfigs(connectorId, null, mc.getLinkConfig().getConfigs(),
- MConfigType.LINK.name(), baseConfigStmt, baseInputStmt);
+ MConfigType.LINK.name(), baseConfigStmt, baseInputStmt, conn);
// Register both from/to job type config
- registerConfigs(connectorId, Direction.FROM, mc.getConfig(Direction.FROM).getConfigs(),
- MConfigType.JOB.name(), baseConfigStmt, baseInputStmt);
- registerConfigs(connectorId, Direction.TO, mc.getConfig(Direction.TO).getConfigs(),
- MConfigType.JOB.name(), baseConfigStmt, baseInputStmt);
-
+ if (mc.getSupportedDirections().isDirectionSupported(Direction.FROM)) {
+ registerConfigs(connectorId, Direction.FROM, mc.getConfig(Direction.FROM).getConfigs(),
+ MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn);
+ }
+ if (mc.getSupportedDirections().isDirectionSupported(Direction.TO)) {
+ registerConfigs(connectorId, Direction.TO, mc.getConfig(Direction.TO).getConfigs(),
+ MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn);
+ }
} catch (SQLException ex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0014,
mc.toString(), ex);
@@ -167,6 +175,34 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
+ private void insertConnectorDirection(Long connectorId, Direction direction, Connection conn)
+ throws SQLException {
+ PreparedStatement stmt = null;
+
+ try {
+ stmt = conn.prepareStatement(STMT_INSERT_SQ_CONNECTOR_DIRECTIONS);
+ stmt.setLong(1, connectorId);
+ stmt.setLong(2, getDirection(direction, conn));
+
+ if (stmt.executeUpdate() != 1) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0049);
+ }
+ } finally {
+ closeStatements(stmt);
+ }
+ }
+
+ private void insertConnectorDirections(Long connectorId, SupportedDirections directions, Connection conn)
+ throws SQLException {
+ if (directions.isDirectionSupported(Direction.FROM)) {
+ insertConnectorDirection(connectorId, Direction.FROM, conn);
+ }
+
+ if (directions.isDirectionSupported(Direction.TO)) {
+ insertConnectorDirection(connectorId, Direction.TO, conn);
+ }
+ }
+
private long getConnectorId(MConnector mc, Connection conn) {
PreparedStatement baseConnectorStmt = null;
try {
@@ -187,6 +223,10 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
if (!rsetConnectorId.next()) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0013);
}
+
+ insertConnectorDirections(rsetConnectorId.getLong(1),
+ mc.getSupportedDirections(), conn);
+
return rsetConnectorId.getLong(1);
} catch (SQLException ex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0014,
@@ -399,6 +439,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
if(version <= 3) {
// Schema modifications
+ runQuery(QUERY_CREATE_TABLE_SQ_DIRECTION, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_RENAME_COLUMN_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_LINK_TO_SQB_FROM_LINK, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_LINK, conn);
@@ -412,6 +453,9 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
updteJobInternals(conn, registerHdfsConnector(conn));
}
+ // Change direction from VARCHAR to BIGINT + foreign key.
+ updateDirections(conn, insertDirections(conn));
+
// Wait to remove SQB_TYPE (IMPORT/EXPORT) until we update data.
// Data updates depend on knowledge of the type of job.
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE, conn);
@@ -443,6 +487,110 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
/**
+ * Insert directions: FROM and TO.
+ * @param conn
+ * @return Map<Direction, Long> direction ID => Direction
+ */
+ protected Map<Direction, Long> insertDirections(Connection conn) {
+ // Add directions
+ Map<Direction, Long> directionMap = new TreeMap<Direction, Long>();
+ PreparedStatement insertDirectionStmt = null;
+ try {
+ // Insert directions and get IDs.
+ for (Direction direction : Direction.values()) {
+ insertDirectionStmt = conn.prepareStatement(STMT_INSERT_DIRECTION, Statement.RETURN_GENERATED_KEYS);
+ insertDirectionStmt.setString(1, direction.toString());
+ if (insertDirectionStmt.executeUpdate() != 1) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0046, "Could not add directions FROM and TO.");
+ }
+
+ ResultSet directionId = insertDirectionStmt.getGeneratedKeys();
+ if (directionId.next()) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Loaded direction: " + directionId.getLong(1));
+ }
+
+ directionMap.put(direction, directionId.getLong(1));
+ } else {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0047, "Could not get ID of direction " + direction);
+ }
+ }
+ } catch (SQLException e) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0000, e);
+ } finally {
+ closeStatements(insertDirectionStmt);
+ }
+
+ return directionMap;
+ }
+
+ /**
+ * Add normalized M2M for SQ_CONNECTOR and SQ_CONFIG for Direction.
+ * 1. Remember all ID => direction for configs.
+ * 2. Drop SQF_DIRECTION (varhchar).
+ * 3. Add new M2M tables for SQ_CONNECTOR and SQ_CONFIG.
+ * 4. Add directions via updating SQ_CONFIG with proper Direction IDs.
+ * 5. Make sure all connectors have all supported directions.
+ * @param conn
+ */
+ protected void updateDirections(Connection conn, Map<Direction, Long> directionMap) {
+ // Remember directions
+ Statement fetchFormsStmt = null,
+ fetchConnectorsStmt = null;
+ List<Long> connectorIds = new LinkedList<Long>();
+ List<Long> configIds = new LinkedList<Long>();
+ List<String> directions = new LinkedList<String>();
+ try {
+ fetchFormsStmt = conn.createStatement();
+ ResultSet rs = fetchFormsStmt.executeQuery(STMT_FETCH_CONFIG_DIRECTIONS);
+ while (rs.next()) {
+ configIds.add(rs.getLong(1));
+ directions.add(rs.getString(2));
+ }
+ rs.close();
+ } catch (SQLException e) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0000, e);
+ } finally {
+ closeStatements(fetchFormsStmt);
+ }
+
+ // Change Schema
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_DROP_COLUMN_SQ_CFG_DIRECTION_VARCHAR, conn);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONNECTOR_DIRECTIONS, conn);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONFIG_DIRECTIONS, conn);
+
+ // Add directions back
+ while (!configIds.isEmpty() && !directions.isEmpty()) {
+ Long configId = configIds.remove(0);
+ String directionString = directions.remove(0);
+ if (directionString != null && !directionString.isEmpty()) {
+ Direction direction = Direction.valueOf(directionString);
+ runQuery(STMT_INSERT_SQ_CONFIG_DIRECTIONS, conn, configId, directionMap.get(direction));
+ }
+ }
+
+ // Add connector directions
+ try {
+ fetchConnectorsStmt = conn.createStatement();
+ ResultSet rs = fetchConnectorsStmt.executeQuery(STMT_SELECT_CONNECTOR_ALL);
+ while (rs.next()) {
+ connectorIds.add(rs.getLong(1));
+ }
+ rs.close();
+ } catch (SQLException e) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0000, e);
+ } finally {
+ closeStatements(fetchConnectorsStmt);
+ }
+
+ for (Long connectorId : connectorIds) {
+ for (Long directionId : directionMap.values()) {
+ runQuery(STMT_INSERT_SQ_CONNECTOR_DIRECTIONS, conn, connectorId, directionId);
+ }
+ }
+ }
+
+ /**
* Upgrade job data from IMPORT/EXPORT to FROM/TO.
* Since the framework is no longer responsible for HDFS,
* the HDFS connector/link must be added.
@@ -509,13 +657,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_DRIVER_INDEX, conn,
new Long(0), "throttling");
- MLink hdfsLink = createHdfsLink(conn);
+ Long linkId = createHdfsLink(conn, connectorId);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_LINK_COPY_SQB_FROM_LINK, conn,
"EXPORT");
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_FROM_LINK, conn,
- new Long(hdfsLink.getPersistenceId()), "EXPORT");
+ new Long(linkId), "EXPORT");
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_LINK, conn,
- new Long(hdfsLink.getPersistenceId()), "IMPORT");
+ new Long(linkId), "IMPORT");
runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_SQ_CFG_NAME, conn,
"fromJobConfig", "table", Direction.FROM.toString());
@@ -556,7 +704,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
if (handler.getUniqueName().equals(CONNECTOR_HDFS)) {
try {
PreparedStatement baseConnectorStmt = conn.prepareStatement(
- STMT_INSERT_CONNECTOR_BASE,
+ STMT_INSERT_CONNECTOR_WITHOUT_SUPPORTED_DIRECTIONS,
Statement.RETURN_GENERATED_KEYS);
baseConnectorStmt.setString(1, handler.getMetadata().getUniqueName());
baseConnectorStmt.setString(2, handler.getMetadata().getClassName());
@@ -588,22 +736,46 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
*
* NOTE: Upgrade path only!
*/
- private MLink createHdfsLink(Connection conn) {
+ private Long createHdfsLink(Connection conn, Long connectorId) {
if (LOG.isTraceEnabled()) {
LOG.trace("Creating HDFS link.");
}
- MConnector hdfsConnector = this.findConnector(CONNECTOR_HDFS, conn);
- MLink hdfsLink = new MLink(
- hdfsConnector.getPersistenceId(),
- hdfsConnector.getLinkConfig());
- this.createLink(hdfsLink, conn);
+ PreparedStatement stmt = null;
+ int result;
+ try {
+ stmt = conn.prepareStatement(STMT_INSERT_LINK,
+ Statement.RETURN_GENERATED_KEYS);
+ stmt.setString(1, LINK_HDFS);
+ stmt.setLong(2, connectorId);
+ stmt.setBoolean(3, true);
+ stmt.setNull(4, Types.VARCHAR);
+ stmt.setTimestamp(5, new Timestamp(System.currentTimeMillis()));
+ stmt.setNull(6, Types.VARCHAR);
+ stmt.setTimestamp(7, new Timestamp(System.currentTimeMillis()));
+
+ result = stmt.executeUpdate();
+ if (result != 1) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0012,
+ Integer.toString(result));
+ }
- if (LOG.isTraceEnabled()) {
- LOG.trace("Created HDFS link.");
- }
+ ResultSet rsetConnectionId = stmt.getGeneratedKeys();
- return hdfsLink;
+ if (!rsetConnectionId.next()) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0013);
+ }
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Created HDFS link.");
+ }
+
+ return rsetConnectionId.getLong(1);
+ } catch (SQLException ex) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0019, ex);
+ } finally {
+ closeStatements(stmt);
+ }
}
/**
@@ -695,7 +867,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
// Register a driver config as a job type with no owner/connector and direction
registerConfigs(null/* owner*/, null /*direction*/, mDriver.getDriverConfig().getConfigs(),
- MConfigType.JOB.name(), baseConfigStmt, baseInputStmt);
+ MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn);
// We're using hardcoded value for driver config as they are
// represented as NULL in the database.
@@ -1744,13 +1916,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
Counters counters = new Counters();
- while(rs.next()) {
+ while (rs.next()) {
String groupName = rs.getString(1);
String counterName = rs.getString(2);
long value = rs.getLong(3);
CounterGroup group = counters.getCounterGroup(groupName);
- if(group == null) {
+ if (group == null) {
group = new CounterGroup(groupName);
counters.addCounterGroup(group);
}
@@ -1758,7 +1930,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
group.addCounter(new Counter(counterName, value));
}
- if(counters.isEmpty()) {
+ if (counters.isEmpty()) {
return null;
} else {
return counters;
@@ -1769,7 +1941,83 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
}
- private List<MConnector> loadConnectors(PreparedStatement stmt,Connection conn) throws SQLException {
+ private Long getDirection(Direction direction, Connection conn) throws SQLException {
+ PreparedStatement directionStmt = null;
+ ResultSet rs = null;
+
+ try {
+ directionStmt = conn.prepareStatement(STMT_SELECT_SQD_ID_BY_SQD_NAME);
+ directionStmt.setString(1, direction.toString());
+ rs = directionStmt.executeQuery();
+
+ rs.next();
+ return rs.getLong(1);
+ } finally {
+ if (rs != null) {
+ closeResultSets(rs);
+ }
+ if (directionStmt != null) {
+ closeStatements(directionStmt);
+ }
+ }
+ }
+
+ private Direction getDirection(long directionId, Connection conn) throws SQLException {
+ PreparedStatement directionStmt = null;
+ ResultSet rs = null;
+
+ try {
+ directionStmt = conn.prepareStatement(STMT_SELECT_SQD_NAME_BY_SQD_ID);
+ directionStmt.setLong(1, directionId);
+ rs = directionStmt.executeQuery();
+
+ rs.next();
+ return Direction.valueOf(rs.getString(1));
+ } finally {
+ if (rs != null) {
+ closeResultSets(rs);
+ }
+ if (directionStmt != null) {
+ closeStatements(directionStmt);
+ }
+ }
+ }
+
+ private SupportedDirections findConnectorSupportedDirections(long connectorId, Connection conn) throws SQLException {
+ PreparedStatement connectorDirectionsStmt = null;
+ ResultSet rs = null;
+
+ boolean from = false, to = false;
+
+ try {
+ connectorDirectionsStmt = conn.prepareStatement(STMT_SELECT_SQ_CONNECTOR_DIRECTIONS);
+ connectorDirectionsStmt.setLong(1, connectorId);
+ rs = connectorDirectionsStmt.executeQuery();
+
+ while(rs.next()) {
+ switch(getDirection(rs.getLong(2), conn)) {
+ case FROM:
+ from = true;
+ break;
+
+ case TO:
+ to = true;
+ break;
+ }
+ }
+ } finally {
+ if (rs != null) {
+ closeResultSets(rs);
+ }
+ if (connectorDirectionsStmt != null) {
+ closeStatements(connectorDirectionsStmt);
+ }
+ }
+
+ return new SupportedDirections(from, to);
+ }
+
+ private List<MConnector> loadConnectors(PreparedStatement stmt, Connection conn) throws SQLException {
List<MConnector> connectors = new ArrayList<MConnector>();
ResultSet rsConnectors = null;
PreparedStatement connectorConfigFetchStmt = null;
@@ -1792,13 +2040,21 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
List<MConfig> fromConfig = new ArrayList<MConfig>();
List<MConfig> toConfig = new ArrayList<MConfig>();
- loadConfigTypes(linkConfig, fromConfig, toConfig,
- connectorConfigFetchStmt, connectorConfigInputFetchStmt, 1);
+ loadConfigTypes(linkConfig, fromConfig, toConfig, connectorConfigFetchStmt,
+ connectorConfigInputFetchStmt, 1, conn);
+ SupportedDirections supportedDirections
+ = findConnectorSupportedDirections(connectorId, conn);
+ MFromConfig fromJobConfig = null;
+ MToConfig toJobConfig = null;
+ if (supportedDirections.isDirectionSupported(Direction.FROM)) {
+ fromJobConfig = new MFromConfig(fromConfig);
+ }
+ if (supportedDirections.isDirectionSupported(Direction.TO)) {
+ toJobConfig = new MToConfig(toConfig);
+ }
MConnector mc = new MConnector(connectorName, connectorClassName, connectorVersion,
- new MLinkConfig(linkConfig),
- new MFromConfig(fromConfig),
- new MToConfig(toConfig));
+ new MLinkConfig(linkConfig), fromJobConfig, toJobConfig);
mc.setPersistenceId(connectorId);
connectors.add(mc);
@@ -1845,7 +2101,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
List<MConfig> toConfig = new ArrayList<MConfig>();
loadConfigTypes(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement,
- connectorConfigInputStatement, 2);
+ connectorConfigInputStatement, 2, conn);
MLink link = new MLink(connectorId, new MLinkConfig(connectorLinkConfig));
link.setPersistenceId(id);
@@ -1911,7 +2167,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
List<MConfig> fromConnectorToJobConfig = new ArrayList<MConfig>();
loadConfigTypes(fromConnectorLinkConfig, fromConnectorFromJobConfig, fromConnectorToJobConfig,
- fromConfigFetchStmt, jobInputFetchStmt, 2);
+ fromConfigFetchStmt, jobInputFetchStmt, 2, conn);
// TO entity configs
List<MConfig> toConnectorLinkConfig = new ArrayList<MConfig>();
@@ -1922,7 +2178,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
List<MConfig> driverConfig = new ArrayList<MConfig>();
loadConfigTypes(toConnectorLinkConfig, toConnectorFromJobConfig, toConnectorToJobConfig,
- toConfigFetchStmt, jobInputFetchStmt, 2);
+ toConfigFetchStmt, jobInputFetchStmt, 2, conn);
loadDriverConfigs(driverConfig, driverConfigfetchStmt, jobInputFetchStmt, 2);
@@ -1951,6 +2207,21 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
return jobs;
}
+ private void registerConfigDirection(Long configId, Direction direction, Connection conn)
+ throws SQLException {
+ PreparedStatement stmt = null;
+ try {
+ stmt = conn.prepareStatement(STMT_INSERT_SQ_CONFIG_DIRECTIONS);
+ stmt.setLong(1, configId);
+ stmt.setLong(2, getDirection(direction, conn));
+ if (stmt.executeUpdate() != 1) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0048);
+ }
+ } finally {
+ closeStatements(stmt);
+ }
+ }
+
/**
* Register configs in derby database. This method will insert the ids
* generated by the repository into the configs passed in itself.
@@ -1962,12 +2233,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* @param type
* @param baseConfigStmt
* @param baseInputStmt
+ * @param conn
* @return short number of configs registered.
* @throws SQLException
*/
private short registerConfigs(Long connectorId, Direction direction,
List<MConfig> configs, String type, PreparedStatement baseConfigStmt,
- PreparedStatement baseInputStmt)
+ PreparedStatement baseInputStmt, Connection conn)
throws SQLException {
short configIndex = 0;
@@ -1977,14 +2249,10 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
} else {
baseConfigStmt.setLong(1, connectorId);
}
- if(direction == null) {
- baseConfigStmt.setNull(2, Types.VARCHAR);
- } else {
- baseConfigStmt.setString(2, direction.name());
- }
- baseConfigStmt.setString(3, config.getName());
- baseConfigStmt.setString(4, type);
- baseConfigStmt.setShort(5, configIndex++);
+
+ baseConfigStmt.setString(2, config.getName());
+ baseConfigStmt.setString(3, type);
+ baseConfigStmt.setShort(4, configIndex++);
int baseConfigCount = baseConfigStmt.executeUpdate();
if (baseConfigCount != 1) {
@@ -1999,6 +2267,10 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
long configId = rsetConfigId.getLong(1);
config.setPersistenceId(configId);
+ if (direction != null) {
+ registerConfigDirection(configId, direction, conn);
+ }
+
// Insert all the inputs
List<MInput<?>> inputs = config.getInputs();
registerConfigInputs(configId, inputs, baseInputStmt);
@@ -2071,7 +2343,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
} else if (args[i] instanceof Long) {
stmt.setLong(i + 1, (Long) args[i]);
} else {
- stmt.setObject(i, args[i]);
+ stmt.setObject(i + 1, args[i]);
}
}
@@ -2115,9 +2387,9 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
while (rsetConfig.next()) {
long configId = rsetConfig.getLong(1);
Long fromConnectorId = rsetConfig.getLong(2);
- String configName = rsetConfig.getString(4);
- String configTYpe = rsetConfig.getString(5);
- int configIndex = rsetConfig.getInt(6);
+ String configName = rsetConfig.getString(3);
+ String configTYpe = rsetConfig.getString(4);
+ int configIndex = rsetConfig.getInt(5);
List<MInput<?>> configInputs = new ArrayList<MInput<?>>();
MConfig mDriverConfig = new MConfig(configName, configInputs);
@@ -2211,6 +2483,26 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
}
+ private Direction findConfigDirection(long configId, Connection conn) throws SQLException {
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+
+ try {
+ stmt = conn.prepareStatement(STMT_SELECT_SQ_CONFIG_DIRECTIONS);
+ stmt.setLong(1, configId);
+ rs = stmt.executeQuery();
+ rs.next();
+ return getDirection(rs.getLong(2), conn);
+ } finally {
+ if (rs != null) {
+ closeResultSets(rs);
+ }
+ if (stmt != null) {
+ closeStatements(stmt);
+ }
+ }
+ }
+
/**
* Load configs and corresponding inputs from Derby database.
*
@@ -2222,21 +2514,21 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* @param toConfig TO job configs that will be filled up
* @param configFetchStmt Prepared statement for fetching configs
* @param inputFetchStmt Prepare statement for fetching inputs
+ * @param conn Connection object that is used to find config direction.
* @throws SQLException In case of any failure on Derby side
*/
public void loadConfigTypes(List<MConfig> linkConfig, List<MConfig> fromConfig,
List<MConfig> toConfig, PreparedStatement configFetchStmt, PreparedStatement inputFetchStmt,
- int configPosition) throws SQLException {
+ int configPosition, Connection conn) throws SQLException {
// Get list of structures from database
ResultSet rsetConfig = configFetchStmt.executeQuery();
while (rsetConfig.next()) {
long configId = rsetConfig.getLong(1);
Long configConnectorId = rsetConfig.getLong(2);
- String operation = rsetConfig.getString(3);
- String configName = rsetConfig.getString(4);
- String configType = rsetConfig.getString(5);
- int configIndex = rsetConfig.getInt(6);
+ String configName = rsetConfig.getString(3);
+ String configType = rsetConfig.getString(4);
+ int configIndex = rsetConfig.getInt(5);
List<MInput<?>> configInputs = new ArrayList<MInput<?>>();
MConfig config = new MConfig(configName, configInputs);
@@ -2324,7 +2616,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
linkConfig.add(config);
break;
case JOB:
- Direction type = Direction.valueOf(operation);
+ Direction type = findConfigDirection(configId, conn);
List<MConfig> jobConfigs;
switch(type) {
case FROM:
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
index ad749ed..cf6e657 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
@@ -41,6 +41,17 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQM_VALUE = "SQM_VALUE";
+ // SQ_DIRECTION
+
+ public static final String TABLE_SQ_DIRECTION_NAME = "SQ_DIRECTION";
+
+ public static final String TABLE_SQ_DIRECTION = SCHEMA_PREFIX
+ + TABLE_SQ_DIRECTION_NAME;
+
+ public static final String COLUMN_SQD_ID = "SQD_ID";
+
+ public static final String COLUMN_SQD_NAME = "SQD_NAME";
+
// SQ_CONNECTOR
public static final String TABLE_SQ_CONNECTOR_NAME = "SQ_CONNECTOR";
@@ -56,6 +67,27 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQC_VERSION = "SQC_VERSION";
+ // SQ_CONNECTOR_DIRECTIONS
+
+ public static final String TABLE_SQ_CONNECTOR_DIRECTIONS_NAME = "SQ_CONNECTOR_DIRECTIONS";
+
+ public static final String TABLE_SQ_CONNECTOR_DIRECTIONS = SCHEMA_PREFIX
+ + TABLE_SQ_CONNECTOR_DIRECTIONS_NAME;
+
+ public static final String COLUMN_SQCD_ID = "SQCD_ID";
+
+ public static final String COLUMN_SQCD_CONNECTOR = "SQCD_CONNECTOR";
+
+ public static final String COLUMN_SQCD_DIRECTION = "SQCD_DIRECTION";
+
+ public static final String CONSTRAINT_SQCD_SQC_NAME = CONSTRAINT_PREFIX + "SQCD_SQC";
+
+ public static final String CONSTRAINT_SQCD_SQC = SCHEMA_PREFIX + CONSTRAINT_SQCD_SQC_NAME;
+
+ public static final String CONSTRAINT_SQCD_SQD_NAME = CONSTRAINT_PREFIX + "SQCD_SQD";
+
+ public static final String CONSTRAINT_SQCD_SQD = SCHEMA_PREFIX + CONSTRAINT_SQCD_SQD_NAME;
+
// SQ_CONFIG
public static final String TABLE_SQ_CONFIG_NAME = "SQ_CONFIG";
@@ -81,6 +113,27 @@ public final class DerbySchemaConstants {
public static final String CONSTRAINT_SQ_CFG_SQC = SCHEMA_PREFIX + CONSTRAINT_SQ_CFG_SQC_NAME;
+ // SQ_CONFIG_DIRECTIONS
+
+ public static final String TABLE_SQ_CONFIG_DIRECTIONS_NAME = "SQ_CONFIG_DIRECTIONS";
+
+ public static final String TABLE_SQ_CONFIG_DIRECTIONS = SCHEMA_PREFIX
+ + TABLE_SQ_CONFIG_DIRECTIONS_NAME;
+
+ public static final String COLUMN_SQ_CFG_DIR_ID = "SQ_CFG_DIR_ID";
+
+ public static final String COLUMN_SQ_CFG_DIR_CONFIG = "SQ_CFG_DIR_CONFIG";
+
+ public static final String COLUMN_SQ_CFG_DIR_DIRECTION = "SQ_CFG_DIR_DIRECTION";
+
+ public static final String CONSTRAINT_SQ_CFG_DIR_CONFIG_NAME = CONSTRAINT_PREFIX + "SQ_CFG_DIR_CONFIG";
+
+ public static final String CONSTRAINT_SQ_CFG_DIR_CONFIG = SCHEMA_PREFIX + CONSTRAINT_SQ_CFG_DIR_CONFIG_NAME;
+
+ public static final String CONSTRAINT_SQ_CFG_DIR_DIRECTION_NAME = CONSTRAINT_PREFIX + "SQ_CFG_DIR_DIRECTION";
+
+ public static final String CONSTRAINT_SQ_CFG_DIR_DIRECTION = SCHEMA_PREFIX + CONSTRAINT_SQ_CFG_DIR_DIRECTION_NAME;
+
// SQ_INPUT
public static final String TABLE_SQ_INPUT_NAME = "SQ_INPUT";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
index 478afe2..56ea147 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
@@ -35,31 +35,65 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* </pre>
* </p>
* <p>
+ * <strong>SQ_DIRECTION</strong>: Directions.
+ * <pre>
+ * +---------------------------------------+
+ * | SQ_DIRECTION |
+ * +---------------------------------------+
+ * | SQD_ID: BIGINT PK AUTO-GEN |
+ * | SQD_NAME: VARCHAR(64) | "FROM"|"TO"
+ * +---------------------------------------+
+ * </pre>
+ * </p>
+ * <p>
* <strong>SQ_CONNECTOR</strong>: Connector registration.
* <pre>
- * +----------------------------+
- * | SQ_CONNECTOR |
- * +----------------------------+
- * | SQC_ID: BIGINT PK AUTO-GEN |
- * | SQC_NAME: VARCHAR(64) |
- * | SQC_CLASS: VARCHAR(255) |
- * | SQC_VERSION: VARCHAR(64) |
- * +----------------------------+
+ * +-----------------------------+
+ * | SQ_CONNECTOR |
+ * +-----------------------------+
+ * | SQC_ID: BIGINT PK AUTO-GEN |
+ * | SQC_NAME: VARCHAR(64) |
+ * | SQC_CLASS: VARCHAR(255) |
+ * | SQC_VERSION: VARCHAR(64) |
+ * +-----------------------------+
+ * </pre>
+ * </p>
+ * <p>
+ * <strong>SQ_CONNECTOR_DIRECTIONS</strong>: Connector directions.
+ * <pre>
+ * +------------------------------+
+ * | SQ_CONNECTOR_DIRECTIONS |
+ * +------------------------------+
+ * | SQCD_ID: BIGINT PK AUTO-GEN |
+ * | SQCD_CONNECTOR: BIGINT | FK SQCD_CONNECTOR(SQC_ID)
+ * | SQCD_DIRECTION: BIGINT | FK SQCD_DIRECTION(SQD_ID)
+ * +------------------------------+
* </pre>
* </p>
* <p>
* <strong>SQ_CONFIG</strong>: Config details.
* <pre>
- * +----------------------------------+
- * | SQ_CONFIG |
- * +----------------------------------+
+ * +-------------------------------------+
+ * | SQ_CONFIG |
+ * +-------------------------------------+
* | SQ_CFG_ID: BIGINT PK AUTO-GEN |
* | SQ_CFG_OWNER: BIGINT | FK SQ_CFG_OWNER(SQC_ID),NULL for driver
- * | SQ_CFG_DIRECTION: VARCHAR(32) | "FROM"|"TO"|NULL
* | SQ_CFG_NAME: VARCHAR(64) |
* | SQ_CFG_TYPE: VARCHAR(32) | "LINK"|"JOB"
* | SQ_CFG_INDEX: SMALLINT |
- * +----------------------------------+
+ * +-------------------------------------+
+ * </pre>
+ * </p>
+ * <p>
+ * <strong>SQ_CONFIG_DIRECTIONS</strong>: Connector directions.
+ * <pre>
+ * +------------------------------+
+ * | SQ_CONNECTOR_DIRECTIONS |
+ * +------------------------------+
+ * | SQCD_ID: BIGINT PK AUTO-GEN |
+ * | SQCD_CONFIG: BIGINT | FK SQCD_CONFIG(SQ_CFG_ID)
+ * | SQCD_DIRECTION: BIGINT | FK SQCD_DIRECTION(SQD_ID)
+ * +------------------------------+
* </pre>
* </p>
* <p>
@@ -118,11 +152,11 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* <strong>SQ_LINK_INPUT</strong>: N:M relationship link and input
* <pre>
* +----------------------------+
- * | SQ_LINK_INPUT |
+ * | SQ_LINK_INPUT |
* +----------------------------+
- * | SQ_LNKI_LINK: BIGINT PK | FK SQ_LINK(SQ_LNK_ID)
- * | SQ_LNKI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID)
- * | SQ_LNKI_VALUE: LONG VARCHAR |
+ * | SQ_LNKI_LINK: BIGINT PK | FK SQ_LINK(SQ_LNK_ID)
+ * | SQ_LNKI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID)
+ * | SQ_LNKI_VALUE: LONG VARCHAR|
* +----------------------------+
* </pre>
* </p>
@@ -212,6 +246,13 @@ public final class DerbySchemaQuery {
+ COLUMN_SQM_VALUE + " VARCHAR(64) "
+ ")";
+ // DDL: Create table SQ_DIRECTION
+ public static final String QUERY_CREATE_TABLE_SQ_DIRECTION =
+ "CREATE TABLE " + TABLE_SQ_DIRECTION + " ("
+ + COLUMN_SQD_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
+ + COLUMN_SQD_NAME + " VARCHAR(64)"
+ + ")";
+
// DDL: Create table SQ_CONNECTOR
public static final String QUERY_CREATE_TABLE_SQ_CONNECTOR =
"CREATE TABLE " + TABLE_SQ_CONNECTOR + " ("
@@ -221,6 +262,20 @@ public final class DerbySchemaQuery {
+ COLUMN_SQC_VERSION + " VARCHAR(64) "
+ ")";
+ // DDL: Create table SQ_CONNECTOR_DIRECTIONS
+ public static final String QUERY_CREATE_TABLE_SQ_CONNECTOR_DIRECTIONS =
+ "CREATE TABLE " + TABLE_SQ_CONNECTOR_DIRECTIONS + " ("
+ + COLUMN_SQCD_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
+ + COLUMN_SQCD_CONNECTOR + " BIGINT, "
+ + COLUMN_SQCD_DIRECTION + " BIGINT, "
+ + "CONSTRAINT " + CONSTRAINT_SQCD_SQC + " "
+ + "FOREIGN KEY (" + COLUMN_SQCD_CONNECTOR + ") "
+ + "REFERENCES " + TABLE_SQ_CONNECTOR + " (" + COLUMN_SQC_ID + "), "
+ + "CONSTRAINT " + CONSTRAINT_SQCD_SQD + " "
+ + "FOREIGN KEY (" + COLUMN_SQCD_DIRECTION + ") "
+ + "REFERENCES " + TABLE_SQ_DIRECTION + " (" + COLUMN_SQD_ID + ")"
+ + ")";
+
// DDL: Create table SQ_CONFIG ( It stores the configs defined by every connector), if connector is null then it is driver config
public static final String QUERY_CREATE_TABLE_SQ_CONFIG =
"CREATE TABLE " + TABLE_SQ_CONFIG + " ("
@@ -235,6 +290,20 @@ public final class DerbySchemaQuery {
+ "REFERENCES " + TABLE_SQ_CONNECTOR + " (" + COLUMN_SQC_ID + ")"
+ ")";
+ // DDL: Create table SQ_CONFIG_DIRECTIONS
+ public static final String QUERY_CREATE_TABLE_SQ_CONFIG_DIRECTIONS =
+ "CREATE TABLE " + TABLE_SQ_CONFIG_DIRECTIONS + " ("
+ + COLUMN_SQ_CFG_DIR_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
+ + COLUMN_SQ_CFG_DIR_CONFIG + " BIGINT, "
+ + COLUMN_SQ_CFG_DIR_DIRECTION + " BIGINT, "
+ + "CONSTRAINT " + CONSTRAINT_SQ_CFG_DIR_CONFIG + " "
+ + "FOREIGN KEY (" + COLUMN_SQ_CFG_DIR_CONFIG + ") "
+ + "REFERENCES " + TABLE_SQ_CONFIG + " (" + COLUMN_SQ_CFG_ID + "), "
+ + "CONSTRAINT " + CONSTRAINT_SQ_CFG_DIR_DIRECTION + " "
+ + "FOREIGN KEY (" + COLUMN_SQ_CFG_DIR_DIRECTION + ") "
+ + "REFERENCES " + TABLE_SQ_DIRECTION + " (" + COLUMN_SQD_ID + ")"
+ + ")";
+
// DDL: Create table SQ_INPUT
public static final String QUERY_CREATE_TABLE_SQ_INPUT =
"CREATE TABLE " + TABLE_SQ_INPUT + " ("
@@ -435,6 +504,14 @@ public final class DerbySchemaQuery {
+ COLUMN_SQM_VALUE + ") "
+ "VALUES(?, ?)";
+ public static final String STMT_SELECT_SQD_ID_BY_SQD_NAME =
+ "SELECT " + COLUMN_SQD_ID + " FROM " + TABLE_SQ_DIRECTION
+ + " WHERE " + COLUMN_SQD_NAME + "=?";
+
+ public static final String STMT_SELECT_SQD_NAME_BY_SQD_ID =
+ "SELECT " + COLUMN_SQD_NAME + " FROM " + TABLE_SQ_DIRECTION
+ + " WHERE " + COLUMN_SQD_ID + "=?";
+
// DML: Fetch connector Given Name
public static final String STMT_FETCH_BASE_CONNECTOR =
"SELECT "
@@ -459,7 +536,6 @@ public final class DerbySchemaQuery {
"SELECT "
+ COLUMN_SQ_CFG_ID + ", "
+ COLUMN_SQ_CFG_OWNER + ", "
- + COLUMN_SQ_CFG_DIRECTION + ", "
+ COLUMN_SQ_CFG_NAME + ", "
+ COLUMN_SQ_CFG_TYPE + ", "
+ COLUMN_SQ_CFG_INDEX
@@ -472,13 +548,12 @@ public final class DerbySchemaQuery {
"SELECT "
+ COLUMN_SQ_CFG_ID + ", "
+ COLUMN_SQ_CFG_OWNER + ", "
- + COLUMN_SQ_CFG_DIRECTION + ", "
+ COLUMN_SQ_CFG_NAME + ", "
+ COLUMN_SQ_CFG_TYPE + ", "
+ COLUMN_SQ_CFG_INDEX
+ " FROM " + TABLE_SQ_CONFIG
+ " WHERE " + COLUMN_SQ_CFG_OWNER + " IS NULL "
- + " ORDER BY " + COLUMN_SQ_CFG_TYPE + ", " + COLUMN_SQ_CFG_DIRECTION + ", " + COLUMN_SQ_CFG_INDEX;
+ + " ORDER BY " + COLUMN_SQ_CFG_TYPE + ", " + COLUMN_SQ_CFG_INDEX;
// DML: Fetch inputs for a given config
public static final String STMT_FETCH_INPUT =
@@ -544,15 +619,21 @@ public final class DerbySchemaQuery {
+ COLUMN_SQC_VERSION
+ ") VALUES (?, ?, ?)";
+ public static final String STMT_INSERT_CONNECTOR_WITHOUT_SUPPORTED_DIRECTIONS =
+ "INSERT INTO " + TABLE_SQ_CONNECTOR + " ("
+ + COLUMN_SQC_NAME + ", "
+ + COLUMN_SQC_CLASS + ", "
+ + COLUMN_SQC_VERSION
+ + ") VALUES (?, ?, ?)";
+
// DML: Insert config base
public static final String STMT_INSERT_CONFIG_BASE =
"INSERT INTO " + TABLE_SQ_CONFIG + " ("
+ COLUMN_SQ_CFG_OWNER + ", "
- + COLUMN_SQ_CFG_DIRECTION + ", "
+ COLUMN_SQ_CFG_NAME + ", "
+ COLUMN_SQ_CFG_TYPE + ", "
+ COLUMN_SQ_CFG_INDEX
- + ") VALUES ( ?, ?, ?, ?, ?)";
+ + ") VALUES ( ?, ?, ?, ?)";
// DML: Insert config input
public static final String STMT_INSERT_INPUT_BASE =
@@ -1058,6 +1139,45 @@ public final class DerbySchemaQuery {
"ALTER TABLE " + TABLE_SQ_LINK + " ADD CONSTRAINT "
+ CONSTRAINT_SQ_LNK_NAME_UNIQUE + " UNIQUE (" + COLUMN_SQ_LNK_NAME + ")";
+ public static final String STMT_INSERT_DIRECTION = "INSERT INTO " + TABLE_SQ_DIRECTION + " "
+ + "(" + COLUMN_SQD_NAME + ") VALUES (?)";
+
+ // DML: Fetch all configs
+ public static final String STMT_FETCH_CONFIG_DIRECTIONS =
+ "SELECT "
+ + COLUMN_SQ_CFG_ID + ", "
+ + COLUMN_SQ_CFG_DIRECTION
+ + " FROM " + TABLE_SQ_CONFIG;
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_CONFIG_DROP_COLUMN_SQ_CFG_DIRECTION_VARCHAR =
+ "ALTER TABLE " + TABLE_SQ_CONFIG + " DROP COLUMN " + COLUMN_SQ_CFG_DIRECTION;
+
+ public static final String STMT_INSERT_SQ_CONNECTOR_DIRECTIONS =
+ "INSERT INTO " + TABLE_SQ_CONNECTOR_DIRECTIONS + " "
+ + "(" + COLUMN_SQCD_CONNECTOR + ", " + COLUMN_SQCD_DIRECTION + ")"
+ + " VALUES (?, ?)";
+
+ public static final String STMT_INSERT_SQ_CONFIG_DIRECTIONS =
+ "INSERT INTO " + TABLE_SQ_CONFIG_DIRECTIONS + " "
+ + "(" + COLUMN_SQ_CFG_DIR_CONFIG + ", " + COLUMN_SQ_CFG_DIR_DIRECTION + ")"
+ + " VALUES (?, ?)";
+
+ public static final String STMT_SELECT_SQ_CONNECTOR_DIRECTIONS_ALL =
+ "SELECT " + COLUMN_SQCD_CONNECTOR + ", " + COLUMN_SQCD_DIRECTION
+ + " FROM " + TABLE_SQ_CONNECTOR_DIRECTIONS;
+
+ public static final String STMT_SELECT_SQ_CONNECTOR_DIRECTIONS =
+ STMT_SELECT_SQ_CONNECTOR_DIRECTIONS_ALL + " WHERE "
+ + COLUMN_SQCD_CONNECTOR + " = ?";
+
+ public static final String STMT_SELECT_SQ_CONFIG_DIRECTIONS_ALL =
+ "SELECT " + COLUMN_SQ_CFG_DIR_CONFIG + ", " + COLUMN_SQ_CFG_DIR_DIRECTION
+ + " FROM " + TABLE_SQ_CONFIG_DIRECTIONS;
+
+ public static final String STMT_SELECT_SQ_CONFIG_DIRECTIONS =
+ STMT_SELECT_SQ_CONFIG_DIRECTIONS_ALL + " WHERE "
+ + COLUMN_SQ_CFG_DIR_CONFIG + " = ?";
+
private DerbySchemaQuery() {
// Disable explicit object creation
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index 2da084f..9316687 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -19,10 +19,13 @@ package org.apache.sqoop.repository.derby;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_SCHEMA_SQOOP;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_CONFIG;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_CONFIG_DIRECTIONS;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_CONNECTOR;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_CONNECTOR_DIRECTIONS;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_COUNTER;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_COUNTER_GROUP;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_COUNTER_SUBMISSION;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_DIRECTION;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_INPUT;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_JOB;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_JOB_INPUT;
@@ -31,6 +34,7 @@ import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TA
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_SUBMISSION;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_CREATE_TABLE_SQ_SYSTEM;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_CONFIG_RENAME_COLUMN_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_CONFIG_DROP_COLUMN_SQ_CFG_DIRECTION_VARCHAR;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_CREATION_USER;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_ENABLED;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_LINK;
@@ -47,6 +51,8 @@ import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_T
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_LINK_ADD_UNIQUE_CONSTRAINT_NAME;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_CREATION_USER;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_UPDATE_USER;
+import static org.apache.sqoop.repository.derby.DerbySchemaQuery.STMT_INSERT_DIRECTION;
+
import static org.junit.Assert.assertEquals;
import java.sql.Connection;
@@ -147,6 +153,7 @@ abstract public class DerbyTestCase {
}
if (version > 3) {
+ runQuery(QUERY_CREATE_TABLE_SQ_DIRECTION);
runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_RENAME_COLUMN_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_LINK_TO_SQB_FROM_LINK);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_LINK);
@@ -156,6 +163,13 @@ abstract public class DerbyTestCase {
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_UNIQUE_CONSTRAINT_NAME);
runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_UNIQUE_CONSTRAINT_NAME);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_DROP_COLUMN_SQ_CFG_DIRECTION_VARCHAR);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONNECTOR_DIRECTIONS);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONFIG_DIRECTIONS);
+
+ for (Direction direction : Direction.values()) {
+ runQuery(STMT_INSERT_DIRECTION, direction.toString());
+ }
}
runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) VALUES('version', '" + version + "')");
@@ -196,6 +210,42 @@ abstract public class DerbyTestCase {
}
}
+ /**
+ * Run single, arbitrary insert query on derby memory repository.
+ *
+ * @param query Query to execute
+ * @return Long id of newly inserted row (-1 if none).
+ * @throws Exception
+ */
+ protected Long runInsertQuery(String query, Object... args) throws Exception {
+ PreparedStatement stmt = null;
+ try {
+ stmt = getDerbyDatabaseConnection().prepareStatement(query, PreparedStatement.RETURN_GENERATED_KEYS);
+
+ for (int i = 0; i < args.length; ++i) {
+ if (args[i] instanceof String) {
+ stmt.setString(i + 1, (String)args[i]);
+ } else if (args[i] instanceof Long) {
+ stmt.setLong(i + 1, (Long)args[i]);
+ } else {
+ stmt.setString(i + 1, args[i].toString());
+ }
+ }
+
+ if (!stmt.execute()) {
+ ResultSet rs = stmt.getGeneratedKeys();
+ rs.next();
+ return rs.getLong(1);
+ }
+ } finally {
+ if (stmt != null) {
+ stmt.close();
+ }
+ }
+
+ return -1L;
+ }
+
protected Connection getDerbyDatabaseConnection() {
return connection;
}
@@ -291,54 +341,59 @@ abstract public class DerbyTestCase {
}
protected void loadConnectorAndDriverConfigVersion4() throws Exception {
+ Long configId;
+
// Connector entry
runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ "VALUES('A', 'org.apache.sqoop.test.A', '1.0-test')");
for (String connector : new String[]{"1"}) {
+ // Directions
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTOR_DIRECTIONS(SQCD_CONNECTOR, SQCD_DIRECTION)"
+ + "VALUES(" + connector + ", 1)");
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTOR_DIRECTIONS(SQCD_CONNECTOR, SQCD_DIRECTION)"
+ + "VALUES(" + connector + ", 2)");
+
// connector configs
- for (String direction : new String[]{"null", "'FROM'", "'TO'"}) {
+ for (String direction : new String[]{null, "1", "2"}) {
String type;
- if (direction.equals("null")) {
+ if (direction == null) {
type = "LINK";
} else {
type = "JOB";
}
- runQuery("INSERT INTO SQOOP.SQ_CONFIG"
- + "(SQ_CFG_OWNER, SQ_CFG_DIRECTION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
- + "VALUES("
- + connector + ", "
- + direction
- + ", 'C1', '"
- + type
- + "', 0)");
- runQuery("INSERT INTO SQOOP.SQ_CONFIG"
- + "(SQ_CFG_OWNER, SQ_CFG_DIRECTION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
- + "VALUES("
- + connector + ", "
- + direction
- + ", 'C2', '"
- + type
- + "', 1)");
+ configId = runInsertQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ + "VALUES(" + connector + ", 'C1', '" + type + "', 0)");
+
+ if (direction != null) {
+ runInsertQuery("INSERT INTO SQOOP.SQ_CONFIG_DIRECTIONS"
+ + "(SQ_CFG_DIR_CONFIG, SQ_CFG_DIR_DIRECTION) "
+ + "VALUES(" + configId + ", " + direction + ")");
+ }
+
+ configId = runInsertQuery("INSERT INTO SQOOP.SQ_CONFIG"
+ + "(SQ_CFG_OWNER, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ + "VALUES(" + connector + ", 'C2', '" + type + "', 1)");
+
+ if (direction != null) {
+ runInsertQuery("INSERT INTO SQOOP.SQ_CONFIG_DIRECTIONS"
+ + "(SQ_CFG_DIR_CONFIG, SQ_CFG_DIR_DIRECTION) "
+ + "VALUES(" + configId + ", " + direction + ")");
+ }
}
}
// driver config
for (String type : new String[]{"JOB"}) {
runQuery("INSERT INTO SQOOP.SQ_CONFIG"
- + "(SQ_CFG_OWNER, SQ_CFG_DIRECTION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
- + "VALUES(NULL, NULL"
- + ", 'C1', '"
- + type
- + "', 0)");
+ + "(SQ_CFG_OWNER, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ + "VALUES(NULL" + ", 'C1', '" + type + "', 0)");
runQuery("INSERT INTO SQOOP.SQ_CONFIG"
- + "(SQ_CFG_OWNER, SQ_CFG_DIRECTION, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
- + "VALUES(NULL, NULL"
- + ", 'C2', '"
- + type
- + "', 1)");
+ + "(SQ_CFG_OWNER, SQ_CFG_NAME, SQ_CFG_TYPE, SQ_CFG_INDEX) "
+ + "VALUES(NULL" + ", 'C2', '" + type + "', 1)");
}
// Input entries
@@ -512,8 +567,10 @@ abstract public class DerbyTestCase {
*/
public void addConnector() throws Exception {
// Connector entry
- runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
- + "VALUES('B', 'org.apache.sqoop.test.B', '1.0-test')");
+ Long connectorId = runInsertQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ + "VALUES('B', 'org.apache.sqoop.test.B', '1.0-test')");
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTOR_DIRECTIONS (SQCD_CONNECTOR, SQCD_DIRECTION) VALUES (" + connectorId + ", 1)");
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTOR_DIRECTIONS (SQCD_CONNECTOR, SQCD_DIRECTION) VALUES (" + connectorId + ", 2)");
}
/**
@@ -560,8 +617,20 @@ abstract public class DerbyTestCase {
}
protected MConnector getConnector() {
+ return getConnector(true, true);
+ }
+
+ protected MConnector getConnector(boolean from, boolean to) {
+ MFromConfig fromJobForms = null;
+ MToConfig toJobForms = null;
+ if (from) {
+ fromJobForms = getFromConfig();
+ }
+ if (to) {
+ toJobForms = getToConfig();
+ }
return new MConnector("A", "org.apache.sqoop.test.A", "1.0-test",
- getLinkConfig(), getFromConfig(), getToConfig());
+ getLinkConfig(), fromJobForms, toJobForms);
}
protected MDriver getDriver() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
index a0e8b91..fc95222 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
@@ -78,8 +78,6 @@ public class TestConnectorHandling extends DerbyTestCase {
assertEquals(connectors.size(),2);
assertEquals(connectors.get(0).getUniqueName(),"A");
assertEquals(connectors.get(1).getUniqueName(),"B");
-
-
}
@Test
@@ -101,4 +99,64 @@ public class TestConnectorHandling extends DerbyTestCase {
assertNotNull(retrieved);
assertEquals(connector, retrieved);
}
+
+ @Test
+ public void testFromDirection() throws Exception {
+ MConnector connector = getConnector(true, false);
+
+ handler.registerConnector(connector, getDerbyDatabaseConnection());
+
+ // Connector should get persistence ID
+ assertEquals(1, connector.getPersistenceId());
+
+ // Now check content in corresponding tables
+ assertCountForTable("SQOOP.SQ_CONNECTOR", 1);
+ assertCountForTable("SQOOP.SQ_CONFIG", 4);
+ assertCountForTable("SQOOP.SQ_INPUT", 8);
+
+ // Registered connector should be easily recovered back
+ MConnector retrieved = handler.findConnector("A", getDerbyDatabaseConnection());
+ assertNotNull(retrieved);
+ assertEquals(connector, retrieved);
+ }
+
+ @Test
+ public void testToDirection() throws Exception {
+ MConnector connector = getConnector(false, true);
+
+ handler.registerConnector(connector, getDerbyDatabaseConnection());
+
+ // Connector should get persistence ID
+ assertEquals(1, connector.getPersistenceId());
+
+ // Now check content in corresponding tables
+ assertCountForTable("SQOOP.SQ_CONNECTOR", 1);
+ assertCountForTable("SQOOP.SQ_CONFIG", 4);
+ assertCountForTable("SQOOP.SQ_INPUT", 8);
+
+ // Registered connector should be easily recovered back
+ MConnector retrieved = handler.findConnector("A", getDerbyDatabaseConnection());
+ assertNotNull(retrieved);
+ assertEquals(connector, retrieved);
+ }
+
+ @Test
+ public void testNeitherDirection() throws Exception {
+ MConnector connector = getConnector(false, false);
+
+ handler.registerConnector(connector, getDerbyDatabaseConnection());
+
+ // Connector should get persistence ID
+ assertEquals(1, connector.getPersistenceId());
+
+ // Now check content in corresponding tables
+ assertCountForTable("SQOOP.SQ_CONNECTOR", 1);
+ assertCountForTable("SQOOP.SQ_CONFIG", 2);
+ assertCountForTable("SQOOP.SQ_INPUT", 4);
+
+ // Registered connector should be easily recovered back
+ MConnector retrieved = handler.findConnector("A", getDerbyDatabaseConnection());
+ assertNotNull(retrieved);
+ assertEquals(connector, retrieved);
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
index 0752923..01a05b2 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
@@ -31,10 +31,6 @@ import org.apache.sqoop.model.MStringInput;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import static org.junit.Assert.*;
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
index 09fb195..d605457 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
@@ -35,7 +35,6 @@ import static org.apache.sqoop.shell.utils.ConfigDisplayer.*;
@SuppressWarnings("serial")
public class ShowConnectorFunction extends SqoopFunction {
- private static final char SUPPORTED_DIRECTIONS_SEPARATOR = '/';
@SuppressWarnings("static-access")
public ShowConnectorFunction() {
@@ -83,7 +82,7 @@ public class ShowConnectorFunction extends SqoopFunction {
uniqueNames.add(connector.getUniqueName());
versions.add(connector.getVersion());
classes.add(connector.getClassName());
- supportedDirections.add(getSupportedDirections(connector));
+ supportedDirections.add(connector.getSupportedDirections().toString());
}
TableDisplayer.display(header, ids, uniqueNames, versions, classes, supportedDirections);
@@ -113,33 +112,8 @@ public class ShowConnectorFunction extends SqoopFunction {
connector.getUniqueName(),
connector.getClassName(),
connector.getVersion(),
- getSupportedDirections(connector)
+ connector.getSupportedDirections().toString()
);
displayConnectorConfigDetails(connector, client.getConnectorConfigBundle(connector.getPersistenceId()));
}
-
- /**
- * Creates a nicely formatted string for which directions are supported.
- * Example: FROM/TO.
- * @param connector
- * @return String
- */
- private String getSupportedDirections(MConnector connector) {
- StringBuffer supportedDirectionsBuffer = new StringBuffer();
- SupportedDirections supportedDirections
- = connector.getSupportedDirections();
-
- if (supportedDirections.isDirectionSupported(Direction.FROM)) {
- supportedDirectionsBuffer.append(Direction.FROM);
-
- if (supportedDirections.isDirectionSupported(Direction.TO)) {
- supportedDirectionsBuffer.append(SUPPORTED_DIRECTIONS_SEPARATOR);
- supportedDirectionsBuffer.append(Direction.TO);
- }
- } else if (supportedDirections.isDirectionSupported(Direction.TO)) {
- supportedDirectionsBuffer.append(Direction.TO);
- }
-
- return supportedDirectionsBuffer.toString();
- }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fc74316f/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
index 4272470..5315e1f 100644
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
@@ -73,7 +73,7 @@ public abstract class SqoopConnector {
public abstract From getFrom();
/**
- * @return an <tt>To</tt> that provides classes for performing export.
+ * @return an <tt>To</tt> that provides classes for performing export.n
*/
public abstract To getTo();
[28/52] [abbrv] git commit: SQOOP-1544: Sqoop2: From/To:
ConnectorBean should work support different directions
Posted by ab...@apache.org.
SQOOP-1544: Sqoop2: From/To: ConnectorBean should work support different directions
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/35a060e0
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/35a060e0
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/35a060e0
Branch: refs/heads/SQOOP-1367
Commit: 35a060e04852c65603223fbb4dc78a18e5c54793
Parents: 2c20d92
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Fri Sep 26 08:03:58 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:59:24 2014 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/json/ConnectorBean.java | 31 +++++++----
.../apache/sqoop/json/TestConnectorBean.java | 58 ++++++++++++++++++++
.../java/org/apache/sqoop/json/TestUtil.java | 14 ++++-
3 files changed, 92 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/35a060e0/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
index 465dd69..d726a14 100644
--- a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
@@ -86,10 +86,15 @@ public class ConnectorBean implements JsonBean {
object.put(CON_FORMS, extractForms(connector.getConnectionForms().getForms(), skipSensitive));
object.put(JOB_FORMS, new JSONObject());
- ((JSONObject)object.get(JOB_FORMS)).put(
- Direction.FROM, extractForms(connector.getJobForms(Direction.FROM).getForms(), skipSensitive));
- ((JSONObject)object.get(JOB_FORMS)).put(
- Direction.TO, extractForms(connector.getJobForms(Direction.TO).getForms(), skipSensitive));
+ if (connector.getJobForms(Direction.FROM) != null) {
+ ((JSONObject)object.get(JOB_FORMS)).put(
+ Direction.FROM, extractForms(connector.getJobForms(Direction.FROM).getForms(), skipSensitive));
+ }
+
+ if (connector.getJobForms(Direction.TO) != null) {
+ ((JSONObject)object.get(JOB_FORMS)).put(
+ Direction.TO, extractForms(connector.getJobForms(Direction.TO).getForms(), skipSensitive));
+ }
array.add(object);
}
@@ -124,17 +129,23 @@ public class ConnectorBean implements JsonBean {
String className = (String) object.get(CLASS);
String version = (String) object.get(VERSION);
+ MJobForms fromJob = null;
+ MJobForms toJob = null;
List<MForm> connForms = restoreForms((JSONArray) object.get(CON_FORMS));
JSONObject jobJson = (JSONObject) object.get(JOB_FORMS);
JSONArray fromJobJson = (JSONArray)jobJson.get(Direction.FROM.name());
JSONArray toJobJson = (JSONArray)jobJson.get(Direction.TO.name());
- List<MForm> fromJobForms = restoreForms(fromJobJson);
- List<MForm> toJobForms = restoreForms(toJobJson);
- MJobForms fromJob = new MJobForms(fromJobForms);
- MJobForms toJob = new MJobForms(toJobForms);
+ if (fromJobJson != null) {
+ List<MForm> fromJobForms = restoreForms(fromJobJson);
+ fromJob = new MJobForms(fromJobForms);
+ }
+ if (toJobJson != null) {
+ List<MForm> toJobForms = restoreForms(toJobJson);
+ toJob = new MJobForms(toJobForms);
+ }
MConnectionForms connection = new MConnectionForms(connForms);
- MConnector connector = new MConnector(uniqueName, className, version, connection, fromJob,
- toJob);
+ MConnector connector = new MConnector(uniqueName, className, version,
+ connection, fromJob, toJob);
connector.setPersistenceId(connectorId);
connectors.add(connector);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/35a060e0/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
index e078474..c3aa84f 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
@@ -73,4 +73,62 @@ public class TestConnectorBean {
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));
}
+
+ @Test
+ public void testSingleDirection() {
+ // Create testing connector
+ List<MConnector> connectors = new LinkedList<MConnector>();
+ connectors.add(getConnector("jdbc", true, false));
+ connectors.add(getConnector("mysql", false, true));
+
+ // Create testing bundles
+ Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
+ bundles.put(1L, getResourceBundle());
+ bundles.put(2L, getResourceBundle());
+
+ // Serialize it to JSON object
+ ConnectorBean bean = new ConnectorBean(connectors, bundles);
+ JSONObject json = bean.extract(false);
+
+ // "Move" it across network in text form
+ String string = json.toJSONString();
+
+ // Retrieved transferred object
+ JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
+ ConnectorBean retrievedBean = new ConnectorBean();
+ retrievedBean.restore(retrievedJson);
+
+ assertEquals(connectors.size(), retrievedBean.getConnectors().size());
+ assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
+ assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
+ }
+
+ @Test
+ public void testNoDirection() {
+ // Create testing connector
+ List<MConnector> connectors = new LinkedList<MConnector>();
+ connectors.add(getConnector("jdbc", false, false));
+ connectors.add(getConnector("mysql", false, false));
+
+ // Create testing bundles
+ Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
+ bundles.put(1L, getResourceBundle());
+ bundles.put(2L, getResourceBundle());
+
+ // Serialize it to JSON object
+ ConnectorBean bean = new ConnectorBean(connectors, bundles);
+ JSONObject json = bean.extract(false);
+
+ // "Move" it across network in text form
+ String string = json.toJSONString();
+
+ // Retrieved transferred object
+ JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
+ ConnectorBean retrievedBean = new ConnectorBean();
+ retrievedBean.restore(retrievedJson);
+
+ assertEquals(connectors.size(), retrievedBean.getConnectors().size());
+ assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
+ assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/35a060e0/common/src/test/java/org/apache/sqoop/json/TestUtil.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestUtil.java b/common/src/test/java/org/apache/sqoop/json/TestUtil.java
index eaf9122..9875219 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestUtil.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestUtil.java
@@ -40,8 +40,20 @@ import java.util.ResourceBundle;
*/
public class TestUtil {
public static MConnector getConnector(String name) {
+ return getConnector(name, true, true);
+ }
+
+ public static MConnector getConnector(String name, boolean from, boolean to) {
+ MJobForms fromJobForms = null;
+ MJobForms toJobForms = null;
+ if (from) {
+ fromJobForms = getJobForms();
+ }
+ if (to) {
+ toJobForms = getJobForms();
+ }
return new MConnector(name, name + ".class", "1.0-test",
- getConnectionForms(), getJobForms(), getJobForms());
+ getConnectionForms(), fromJobForms, toJobForms);
}
public static MDriverConfig getDriverConfig() {
[13/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/repository/Repository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java
index 8e8dd80..ea1608a 100644
--- a/core/src/main/java/org/apache/sqoop/repository/Repository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java
@@ -17,36 +17,36 @@
*/
package org.apache.sqoop.repository;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.framework.FrameworkManager;
+import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnection;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFramework;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.utils.ClassUtils;
import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.Validator;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-
/**
- * Defines the contract of a Repository used by Sqoop. A Repository allows
- * Sqoop to store metadata, statistics and other state relevant to Sqoop
- * Jobs in the system.
+ * Defines the contract for repository used by Sqoop. A Repository allows
+ * Sqoop to store entities such as connectors, links, jobs, submissions and its related configs,
+ * statistics and other state relevant the entities in the store
*/
public abstract class Repository {
@@ -76,11 +76,11 @@ public abstract class Repository {
/**
* Registers given connector in the repository and return registered
- * variant. This method might return an exception in case that metadata for
+ * variant. This method might return an exception in case that
* given connector are already registered with different structure.
*
- * @param mConnector the connector metadata to be registered
- * autoupgrade whether to upgrade framework automatically
+ * @param mConnector the connector to be registered
+ * autoupgrade whether to upgrade driver config automatically
* @return Registered connector structure
*/
public abstract MConnector registerConnector(MConnector mConnector, boolean autoUpgrade);
@@ -88,7 +88,7 @@ public abstract class Repository {
/**
* Search for connector with given name in repository.
*
- * And return corresponding metadata structure.
+ * And return corresponding entity structure.
*
* @param shortName Connector unique name
* @return null if connector is not yet registered in repository or
@@ -105,76 +105,75 @@ public abstract class Repository {
/**
- * Registers given framework in the repository and return registered
- * variant. This method might return an exception in case that metadata for
- * given framework are already registered with different structure.
+ * Registers given driverConfig in the repository and return registered
+ * variant. This method might return an exception in case that the
+ * given driverConfig are already registered with different structure.
*
- * @param mFramework framework metadata to be registered
- * autoupgrade whether to upgrade framework automatically
+ * @param mDriverConfig driverConfig to be registered
+ * autoupgrade whether to upgrade driverConfig automatically
* @return Registered connector structure
*/
- public abstract MFramework registerFramework(MFramework mFramework, boolean autoUpgrade);
+ public abstract MDriverConfig registerDriverConfig(MDriverConfig mDriverConfig, boolean autoUpgrade);
/**
- * Save given connection to repository. This connection must not be already
+ * Save given link to repository. This link must not be already
* present in the repository otherwise exception will be thrown.
*
- * @param connection Connection object to serialize into repository.
+ * @param link link object to serialize into repository.
*/
- public abstract void createConnection(MConnection connection);
+ public abstract void createLink(MLink link);
/**
- * Update given connection representation in repository. This connection
+ * Update given link representation in repository. This link
* object must already exists in the repository otherwise exception will be
* thrown.
*
- * @param connection Connection object that should be updated in repository.
+ * @param link link object that should be updated in repository.
*/
- public abstract void updateConnection(MConnection connection);
+ public abstract void updateLink(MLink link);
/**
- * Update given connection representation in repository. This connection
+ * Update given link representation in repository. This link
* object must already exists in the repository otherwise exception will be
* thrown.
*
- * @param connection Connection object that should be updated in repository.
+ * @param link Link object that should be updated in repository.
* @param tx The repository transaction to use to push the data to the
* repository. If this is null, a new transaction will be created.
* method will not call begin, commit,
* rollback or close on this transaction.
*/
- public abstract void updateConnection(final MConnection connection,
- RepositoryTransaction tx);
+ public abstract void updateLink(final MLink link, RepositoryTransaction tx);
/**
- * Enable or disable connection with given id from metadata repository
+ * Enable or disable Link with given id from the repository
*
- * @param id Connection object that is going to be enabled or disabled
+ * @param id Link object that is going to be enabled or disabled
* @param enabled enable or disable
*/
- public abstract void enableConnection(long id, boolean enabled);
+ public abstract void enableLink(long id, boolean enabled);
/**
- * Delete connection with given id from metadata repository.
+ * Delete Link with given id from the repository.
*
- * @param id Connection object that should be removed from repository
+ * @param id Link object that should be removed from repository
*/
- public abstract void deleteConnection(long id);
+ public abstract void deleteLink(long id);
/**
- * Find connection with given id in repository.
+ * Find link with given id in repository.
*
- * @param id Connection id
- * @return Deserialized form of the connection that is saved in repository
+ * @param id Link id
+ * @return Deserialized form of the link that is saved in repository
*/
- public abstract MConnection findConnection(long id);
+ public abstract MLink findLink(long id);
/**
- * Get all connection objects.
+ * Get all Link objects.
*
- * @return List will all saved connection objects
+ * @return List will all saved link objects
*/
- public abstract List<MConnection> findConnections();
+ public abstract List<MLink> findLinks();
/**
* Save given job to repository. This job object must not be already present
@@ -285,18 +284,18 @@ public abstract class Repository {
public abstract MSubmission findSubmissionLastForJob(long jobId);
/**
- * Retrieve connections which use the given connector.
- * @param connectorID Connector ID whose connections should be fetched
- * @return List of MConnections that use <code>connectorID</code>.
+ * Retrieve links which use the given connector.
+ * @param connectorID Connector ID whose links should be fetched
+ * @return List of MLink that use <code>connectorID</code>.
*/
- public abstract List<MConnection> findConnectionsForConnector(long
+ public abstract List<MLink> findLinksForConnector(long
connectorID);
/**
- * Retrieve jobs which use the given connection.
+ * Retrieve jobs which use the given link.
*
* @param connectorID Connector ID whose jobs should be fetched
- * @return List of MJobs that use <code>connectionID</code>.
+ * @return List of MJobs that use <code>linkID</code>.
*/
public abstract List<MJob> findJobsForConnector(long
connectorID);
@@ -317,29 +316,25 @@ public abstract class Repository {
* method will not call begin, commit,
* rollback or close on this transaction.
*/
- protected abstract void updateConnector(MConnector newConnector,
- RepositoryTransaction tx);
-
+ protected abstract void updateConnector(MConnector newConnector, RepositoryTransaction tx);
/**
- * Update the framework with the new data supplied in the
- * <tt>mFramework</tt>. Also Update all forms associated with the framework
+ * Update the driverConfig with the new data supplied in the
+ * <tt>mDriverConfig</tt>. Also Update all forms associated with the driverConfig
* in the repository with the forms specified in
- * <tt>mFramework</tt>. <tt>mFramework </tt> must
+ * <tt>mDriverConfig</tt>. <tt>mDriverConfig </tt> must
* minimally have the connectorID and all required forms (including ones
* which may not have changed). After this operation the repository is
* guaranteed to only have the new forms specified in this object.
*
- * @param mFramework The new data to be inserted into the repository for
- * the framework.
+ * @param mDriverConfig The new data to be inserted into the repository for
+ * the driverConfig.
* @param tx The repository transaction to use to push the data to the
* repository. If this is null, a new transaction will be created.
* method will not call begin, commit,
* rollback or close on this transaction.
*/
- protected abstract void updateFramework(MFramework mFramework,
- RepositoryTransaction tx);
-
+ protected abstract void updateDriverConfig(MDriverConfig mDriverConfig, RepositoryTransaction tx);
/**
* Delete all inputs for a job
@@ -351,24 +346,22 @@ public abstract class Repository {
protected abstract void deleteJobInputs(long jobId, RepositoryTransaction tx);
/**
- * Delete all inputs for a connection
- * @param connectionID The id of the connection whose inputs are to be
+ * Delete all inputs for a link
+ * @param linkId The id of the link whose inputs are to be
* deleted.
* @param tx The repository transaction to use to push the data to the
* repository. If this is null, a new transaction will be created.
* method will not call begin, commit,
* rollback or close on this transaction.
*/
- protected abstract void deleteConnectionInputs(long connectionID,
- RepositoryTransaction tx);
+ protected abstract void deleteLinkInputs(long linkId, RepositoryTransaction tx);
- private void deleteConnectionsAndJobs(List<MConnection> connections,
- List<MJob> jobs, RepositoryTransaction tx) {
+ private void deletelinksAndJobs(List<MLink> links, List<MJob> jobs, RepositoryTransaction tx) {
for (MJob job : jobs) {
deleteJobInputs(job.getPersistenceId(), tx);
}
- for (MConnection connection : connections) {
- deleteConnectionInputs(connection.getPersistenceId(), tx);
+ for (MLink link : links) {
+ deleteLinkInputs(link.getPersistenceId(), tx);
}
}
@@ -376,7 +369,7 @@ public abstract class Repository {
* Upgrade the connector with the same {@linkplain MConnector#uniqueName}
* in the repository with values from <code>newConnector</code>.
* <p/>
- * All connections and jobs associated with this connector will be upgraded
+ * All links and jobs associated with this connector will be upgraded
* automatically.
*
* @param oldConnector The old connector that should be upgraded.
@@ -389,18 +382,18 @@ public abstract class Repository {
newConnector.setPersistenceId(connectorID);
/* Algorithms:
* 1. Get an upgrader for the connector.
- * 2. Get all connections associated with the connector.
+ * 2. Get all links associated with the connector.
* 3. Get all jobs associated with the connector.
- * 4. Delete the inputs for all of the jobs and connections (in that order)
+ * 4. Delete the inputs for all of the jobs and links (in that order)
* 5. Remove all inputs and forms associated with the connector, and
* register the new forms and inputs.
- * 6. Create new connections and jobs with connector part being the ones
+ * 6. Create new links and jobs with connector part being the ones
* returned by the upgrader.
- * 7. Validate new connections and jobs with connector's validator
- * 8. If any invalid connections or jobs detected, throw an exception
+ * 7. Validate new links and jobs with connector's validator
+ * 8. If any invalid links or jobs detected, throw an exception
* and stop the bootup of Sqoop server
- * 9. Otherwise, Insert the connection inputs followed by job inputs (using
- * updateJob and updateConnection)
+ * 9. Otherwise, Insert the link inputs followed by job inputs (using
+ * updateJob and updatelink)
*/
RepositoryTransaction tx = null;
try {
@@ -412,33 +405,33 @@ public abstract class Repository {
boolean upgradeSuccessful = true;
- MetadataUpgrader upgrader = connector.getMetadataUpgrader();
- List<MConnection> connections = findConnectionsForConnector(
+ RepositoryUpgrader upgrader = connector.getRepositoryUpgrader();
+ List<MLink> links = findLinksForConnector(
connectorID);
List<MJob> jobs = findJobsForConnector(connectorID);
// -- BEGIN TXN --
tx = getTransaction();
tx.begin();
- deleteConnectionsAndJobs(connections, jobs, tx);
+ deletelinksAndJobs(links, jobs, tx);
updateConnector(newConnector, tx);
- for (MConnection connection : connections) {
+ for (MLink link : links) {
// Make a new copy of the forms from the connector,
// else the values will get set in the forms in the connector for
- // each connection.
+ // each link.
List<MForm> forms = newConnector.getConnectionForms().clone(false).getForms();
- MConnectionForms newConnectionForms = new MConnectionForms(forms);
- upgrader.upgrade(connection.getConnectorPart(), newConnectionForms);
- MConnection newConnection = new MConnection(connection, newConnectionForms, connection.getFrameworkPart());
+ MConnectionForms newlinkForms = new MConnectionForms(forms);
+ upgrader.upgrade(link.getConnectorPart(), newlinkForms);
+ MLink newlink = new MLink(link, newlinkForms, link.getFrameworkPart());
// Transform form structures to objects for validations
- Object newConfigurationObject = ClassUtils.instantiate(connector.getConnectionConfigurationClass());
- FormUtils.fromForms(newConnection.getConnectorPart().getForms(), newConfigurationObject);
+ Object newConfigurationObject = ClassUtils.instantiate(connector.getLinkConfigurationClass());
+ FormUtils.fromForms(newlink.getConnectorPart().getForms(), newConfigurationObject);
- Validation validation = validator.validateConnection(newConfigurationObject);
+ Validation validation = validator.validateLink(newConfigurationObject);
if (validation.getStatus().canProceed()) {
- updateConnection(newConnection, tx);
+ updateLink(newlink, tx);
} else {
- logInvalidModelObject("connection", newConnection, validation);
+ logInvalidModelObject("link", newlink, validation);
upgradeSuccessful = false;
}
}
@@ -513,57 +506,57 @@ public abstract class Repository {
}
}
- public final void upgradeFramework(MFramework framework) {
- LOG.info("Upgrading framework metadata");
+ public final void upgradeDriverConfig(MDriverConfig driverConfig) {
+ LOG.info("Upgrading driver config");
RepositoryTransaction tx = null;
try {
- MetadataUpgrader upgrader = FrameworkManager.getInstance()
- .getMetadataUpgrader();
- List<MConnection> connections = findConnections();
+ RepositoryUpgrader upgrader = Driver.getInstance()
+ .getDriverConfigRepositoryUpgrader();
+ List<MLink> links = findLinks();
List<MJob> jobs = findJobs();
- Validator validator = FrameworkManager.getInstance().getValidator();
+ Validator validator = Driver.getInstance().getValidator();
boolean upgradeSuccessful = true;
// -- BEGIN TXN --
tx = getTransaction();
tx.begin();
- deleteConnectionsAndJobs(connections, jobs, tx);
- updateFramework(framework, tx);
- for (MConnection connection : connections) {
+ deletelinksAndJobs(links, jobs, tx);
+ updateDriverConfig(driverConfig, tx);
+ for (MLink link : links) {
// Make a new copy of the forms from the connector,
// else the values will get set in the forms in the connector for
- // each connection.
- // @TODO(Abe): From/To connection forms.
- List<MForm> forms = framework.getConnectionForms().clone(false).getForms();
- MConnectionForms newConnectionForms = new MConnectionForms(forms);
- upgrader.upgrade(connection.getFrameworkPart(), newConnectionForms);
- MConnection newConnection = new MConnection(connection, connection.getConnectorPart(), newConnectionForms);
+ // each link.
+ // @TODO(Abe): From/To link forms.
+ List<MForm> forms = driverConfig.getConnectionForms().clone(false).getForms();
+ MConnectionForms newlinkForms = new MConnectionForms(forms);
+ upgrader.upgrade(link.getFrameworkPart(), newlinkForms);
+ MLink newlink = new MLink(link, link.getConnectorPart(), newlinkForms);
// Transform form structures to objects for validations
- Object newConfigurationObject = ClassUtils.instantiate(FrameworkManager.getInstance().getConnectionConfigurationClass());
- FormUtils.fromForms(newConnection.getFrameworkPart().getForms(), newConfigurationObject);
+ Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getLinkConfigurationClass());
+ FormUtils.fromForms(newlink.getFrameworkPart().getForms(), newConfigurationObject);
- Validation validation = validator.validateConnection(newConfigurationObject);
+ Validation validation = validator.validateLink(newConfigurationObject);
if (validation.getStatus().canProceed()) {
- updateConnection(newConnection, tx);
+ updateLink(newlink, tx);
} else {
- logInvalidModelObject("connection", newConnection, validation);
+ logInvalidModelObject("link", newlink, validation);
upgradeSuccessful = false;
}
}
for (MJob job : jobs) {
// Make a new copy of the forms from the framework,
// else the values will get set in the forms in the connector for
- // each connection.
- List<MForm> forms = framework.getJobForms().clone(false).getForms();
+ // each link.
+ List<MForm> forms = driverConfig.getJobForms().clone(false).getForms();
MJobForms newJobForms = new MJobForms(forms);
upgrader.upgrade(job.getFrameworkPart(), newJobForms);
MJob newJob = new MJob(job, job.getConnectorPart(Direction.FROM), job.getConnectorPart(Direction.TO), newJobForms);
// Transform form structures to objects for validations
- Object newConfigurationObject = ClassUtils.instantiate(FrameworkManager.getInstance().getJobConfigurationClass());
+ Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getJobConfigurationClass());
FormUtils.fromForms(newJob.getFrameworkPart().getForms(), newConfigurationObject);
Validation validation = validator.validateJob(newConfigurationObject);
@@ -594,7 +587,7 @@ public abstract class Repository {
if(tx != null) {
tx.close();
}
- LOG.info("Framework metadata upgrade finished");
+ LOG.info("Driver config upgrade finished");
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
index 3f3a9e6..54e37d9 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
@@ -40,8 +40,8 @@ public enum RepositoryError implements ErrorCode {
/** The system was unable to find or load the JDBC repository handler. */
JDBCREPO_0001("Invalid JDBC Repository Handler specified"),
- /** An invalid JDBC connection URL was specified. */
- JDBCREPO_0002("Invalid JDBC connection URL specified"),
+ /** An invalid JDBC link URL was specified. */
+ JDBCREPO_0002("Invalid JDBC link URL specified"),
/** An invalid JDBC driver class name was specified. */
JDBCREPO_0003("Invalid JDBC driver class specified"),
@@ -49,17 +49,17 @@ public enum RepositoryError implements ErrorCode {
/** An invalid JDBC transaction isolation level was specified. */
JDBCREPO_0004("Invalid JDBC transaction isolation level specified"),
- /** The value specified for maximum connection pool connections is invalid.*/
- JDBCREPO_0005("Invalid maximum connections specified for connection pool"),
+ /** The value specified for maximum link pool links is invalid.*/
+ JDBCREPO_0005("Invalid maximum links specified for link pool"),
/** The system attempted to use an inactive transaction. */
JDBCREPO_0006("Transaction is not active"),
/**
- * The system was unable to obtain a connection lease for the
+ * The system was unable to obtain a link lease for the
* requested transaction.
*/
- JDBCREPO_0007("Unable to lease connection"),
+ JDBCREPO_0007("Unable to lease link"),
/** The system attempted to commit a transaction marked for rollback.*/
JDBCREPO_0008("Attempt to commit a transaction marked for rollback"),
@@ -76,23 +76,23 @@ public enum RepositoryError implements ErrorCode {
*/
JDBCREPO_0011("Attempt to reinitialize JDBC repository context"),
- /** Failure in metadata repository operation. */
- JDBCREPO_0012("Failure in metadata repository operation."),
+ /** Failure in config repository operation. */
+ JDBCREPO_0012("Failure in config repository operation."),
- /** The system found a change in connector metadata that requires upgrade. */
- JDBCREPO_0013("Connector metadata changed - upgrade may be required"),
+ /** The system found a change in connector config that requires upgrade. */
+ JDBCREPO_0013("Connector config changed - upgrade may be required"),
- /** The system found a change in framework metadata that requires upgrade. */
- JDBCREPO_0014("Framework metadata changed - upgrade may be required"),
+ /** The system found a change in driver config that requires upgrade. */
+ JDBCREPO_0014("Driver config changed - upgrade may be required"),
- /** Connection that we're trying to create is already saved in repository **/
- JDBCREPO_0015("Cannot create connection that was already created"),
+ /** link that we're trying to create is already saved in repository **/
+ JDBCREPO_0015("Cannot create link that was already created"),
- /** Connection that we're trying to update is not yet saved **/
- JDBCREPO_0016("Cannot update connection that was not yet created"),
+ /** link that we're trying to update is not yet saved **/
+ JDBCREPO_0016("Cannot update link that was not yet created"),
- /** Invalid connection id **/
- JDBCREPO_0017("Given connection id is invalid"),
+ /** Invalid link id **/
+ JDBCREPO_0017("Given link id is invalid"),
/** Job that we're trying to create is already saved in repository **/
JDBCREPO_0018("Cannot create job that was already created"),
@@ -103,8 +103,8 @@ public enum RepositoryError implements ErrorCode {
/** Invalid job id **/
JDBCREPO_0020("Given job id is invalid"),
- /** Connection ID is in use **/
- JDBCREPO_0021("Given connection id is in use"),
+ /** link ID is in use **/
+ JDBCREPO_0021("Given link id is in use"),
/** Job ID is in use **/
JDBCREPO_0022("Given job id is in use"),
@@ -121,8 +121,8 @@ public enum RepositoryError implements ErrorCode {
/** Upgrade required but not allowed **/
JDBCREPO_0026("Upgrade required but not allowed"),
- /** Invalid connections or jobs when upgrading connector **/
- JDBCREPO_0027("Invalid connections or jobs when upgrading connector");
+ /** Invalid links or jobs when upgrading connector **/
+ JDBCREPO_0027("Invalid links or jobs when upgrading connector");
;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/resources/driver-config.properties
----------------------------------------------------------------------
diff --git a/core/src/main/resources/driver-config.properties b/core/src/main/resources/driver-config.properties
new file mode 100644
index 0000000..9ec5d9e
--- /dev/null
+++ b/core/src/main/resources/driver-config.properties
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Driver Configs
+
+############################
+
+# Throttling From
+#
+throttling.label = Throttling resources
+throttling.help = Set throttling boundaries to not overload your systems
+
+throttling.extractors.label = Extractors
+throttling.extractors.help = Number of extractors that Sqoop will use
+
+throttling.loaders.label = Loaders
+throttling.loaders.help = Number of loaders that Sqoop will use
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/main/resources/framework-resources.properties
----------------------------------------------------------------------
diff --git a/core/src/main/resources/framework-resources.properties b/core/src/main/resources/framework-resources.properties
deleted file mode 100644
index 7ecb9ae..0000000
--- a/core/src/main/resources/framework-resources.properties
+++ /dev/null
@@ -1,71 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Framework Resources
-
-############################
-# Security Form
-#
-security.label = Security related configuration options
-security.help = You must supply the information requested in order to \
- create a job object.
-
-security.maxConnections.label = Max connections
-security.maxConnections.help = Maximal number of connections that this \
- connection object can use at one point in time
-
-# Output From
-#
-output.label = Output configuration
-output.help = You must supply the information requested in order to \
- get information where you want to store your data.
-
-output.storageType.label = Storage type
-output.storageType.help = Target on Hadoop ecosystem where to store data
-
-output.outputFormat.label = Output format
-output.outputFormat.help = Format in which data should be serialized
-
-output.compression.label = Compression format
-output.compression.help = Compression that should be used for the data
-
-output.customCompression.label = Custom compression format
-output.customCompression.help = Full class name of the custom compression
-
-output.outputDirectory.label = Output directory
-output.outputDirectory.help = Output directory for final data
-
-output.ignored.label = Ignored
-output.ignored.help = This value is ignored
-
-# Input Form
-#
-input.label = Input configuration
-input.help = Specifies information required to get data from Hadoop ecosystem
-
-input.inputDirectory.label = Input directory
-input.inputDirectory.help = Directory that should be exported
-
-# Throttling From
-#
-throttling.label = Throttling resources
-throttling.help = Set throttling boundaries to not overload your systems
-
-throttling.extractors.label = Extractors
-throttling.extractors.help = Number of extractors that Sqoop will use
-
-throttling.loaders.label = Loaders
-throttling.loaders.help = Number of loaders that Sqoop will use
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/core/MockInvalidConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/core/MockInvalidConfigurationProvider.java b/core/src/test/java/org/apache/sqoop/core/MockInvalidConfigurationProvider.java
index 4043e3e..550aaee 100644
--- a/core/src/test/java/org/apache/sqoop/core/MockInvalidConfigurationProvider.java
+++ b/core/src/test/java/org/apache/sqoop/core/MockInvalidConfigurationProvider.java
@@ -29,26 +29,19 @@ public class MockInvalidConfigurationProvider implements ConfigurationProvider {
@Override
public void initialize(File configDir, Properties bootstrapCongiruation) {
- // TODO Auto-generated method stub
-
}
@Override
public void registerListener(ConfigurationListener listener) {
- // TODO Auto-generated method stub
-
}
@Override
public Map<String, String> getConfiguration() {
- // TODO Auto-generated method stub
return null;
}
@Override
public void destroy() {
- // TODO Auto-generated method stub
-
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/core/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/core/TestConfiguration.java b/core/src/test/java/org/apache/sqoop/core/TestConfiguration.java
deleted file mode 100644
index e7fe370..0000000
--- a/core/src/test/java/org/apache/sqoop/core/TestConfiguration.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.core;
-
-import java.util.Properties;
-
-import org.apache.sqoop.common.SqoopException;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestConfiguration {
-
- @Before
- public void setUp() throws Exception {
- // Unset any configuration dir if it is set by another test
- System.getProperties().remove(ConfigurationConstants.SYSPROP_CONFIG_DIR);
- SqoopConfiguration.getInstance().destroy();
- }
-
- @Test
- public void testConfigurationInitFailure() {
- boolean success = false;
- try {
- SqoopConfiguration.getInstance().initialize();
- } catch (Exception ex) {
- Assert.assertTrue(ex instanceof SqoopException);
- Assert.assertSame(((SqoopException) ex).getErrorCode(),
- CoreError.CORE_0001);
- success = true;
- }
-
- Assert.assertTrue(success);
- }
-
- @Test
- public void testBootstrapConfigurationInitFailure() {
- boolean success = false;
- try {
- String configDirPath = TestUtils.createEmptyConfigDirectory();
- System.setProperty(ConfigurationConstants.SYSPROP_CONFIG_DIR,
- configDirPath);
- SqoopConfiguration.getInstance().initialize();
- } catch (Exception ex) {
- Assert.assertTrue(ex instanceof SqoopException);
- Assert.assertSame(((SqoopException) ex).getErrorCode(),
- CoreError.CORE_0002);
- success = true;
- }
-
- Assert.assertTrue(success);
- }
-
- @Test
- public void testConfigurationProviderNotSet() throws Exception {
- boolean success = false;
- Properties bootProps = new Properties();
- bootProps.setProperty("foo", "bar");
- TestUtils.setupTestConfigurationUsingProperties(bootProps, null);
- try {
- SqoopConfiguration.getInstance().initialize();
- } catch (Exception ex) {
- Assert.assertTrue(ex instanceof SqoopException);
- Assert.assertSame(((SqoopException) ex).getErrorCode(),
- CoreError.CORE_0003);
- success = true;
- }
-
- Assert.assertTrue(success);
- }
-
- @Test
- public void testConfigurationProviderInvalid() throws Exception {
- boolean success = false;
- Properties bootProps = new Properties();
- bootProps.setProperty(ConfigurationConstants.BOOTCFG_CONFIG_PROVIDER,
- "foobar");
- TestUtils.setupTestConfigurationUsingProperties(bootProps, null);
- try {
- SqoopConfiguration.getInstance().initialize();
- } catch (Exception ex) {
- Assert.assertTrue(ex instanceof SqoopException);
- Assert.assertSame(((SqoopException) ex).getErrorCode(),
- CoreError.CORE_0004);
-
- success = true;
- }
-
- Assert.assertTrue(success);
- }
-
- @Test
- public void testConfiugrationProviderCannotLoad() throws Exception {
- boolean success = false;
- Properties bootProps = new Properties();
- bootProps.setProperty(ConfigurationConstants.BOOTCFG_CONFIG_PROVIDER,
- MockInvalidConfigurationProvider.class.getCanonicalName());
- TestUtils.setupTestConfigurationUsingProperties(bootProps, null);
- try {
- SqoopConfiguration.getInstance().initialize();
- } catch (Exception ex) {
- Assert.assertTrue(ex instanceof SqoopException);
- Assert.assertSame(((SqoopException) ex).getErrorCode(),
- CoreError.CORE_0005);
- success = true;
- }
-
- Assert.assertTrue(success);
- }
-
- @Test
- public void testPropertiesConfigProviderNoFile() throws Exception {
- boolean success = false;
- Properties bootProps = new Properties();
- bootProps.setProperty(ConfigurationConstants.BOOTCFG_CONFIG_PROVIDER,
- PropertiesConfigurationProvider.class.getCanonicalName());
- TestUtils.setupTestConfigurationUsingProperties(bootProps, null);
- try {
- SqoopConfiguration.getInstance().initialize();
- } catch (Exception ex) {
- Assert.assertTrue(ex instanceof SqoopException);
- Assert.assertSame(((SqoopException) ex).getErrorCode(),
- CoreError.CORE_0006);
- success = true;
- }
-
- Assert.assertTrue(success);
- }
-
- @Test
- public void testSystemNotInitialized() throws Exception {
- boolean success = false;
- try {
- SqoopConfiguration.getInstance().getContext();
- } catch (Exception ex) {
- Assert.assertTrue(ex instanceof SqoopException);
- Assert.assertSame(((SqoopException) ex).getErrorCode(),
- CoreError.CORE_0007);
- success = true;
- }
-
- Assert.assertTrue(success);
- }
-
- @Test
- public void testConfigurationInitSuccess() throws Exception {
- TestUtils.setupTestConfigurationWithExtraConfig(null, null);
- SqoopConfiguration.getInstance().initialize();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java b/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java
new file mode 100644
index 0000000..4d58bd1
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.core;
+
+import java.util.Properties;
+
+import org.apache.sqoop.common.SqoopException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestSqoopConfiguration {
+
+ @Before
+ public void setUp() throws Exception {
+ // Unset any configuration dir if it is set by another test
+ System.getProperties().remove(ConfigurationConstants.SYSPROP_CONFIG_DIR);
+ SqoopConfiguration.getInstance().destroy();
+ }
+
+ @Test
+ public void testConfigurationInitFailure() {
+ boolean success = false;
+ try {
+ SqoopConfiguration.getInstance().initialize();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof SqoopException);
+ Assert.assertSame(((SqoopException) ex).getErrorCode(),
+ CoreError.CORE_0001);
+ success = true;
+ }
+
+ Assert.assertTrue(success);
+ }
+
+ @Test
+ public void testBootstrapConfigurationInitFailure() {
+ boolean success = false;
+ try {
+ String configDirPath = TestUtils.createEmptyConfigDirectory();
+ System.setProperty(ConfigurationConstants.SYSPROP_CONFIG_DIR,
+ configDirPath);
+ SqoopConfiguration.getInstance().initialize();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof SqoopException);
+ Assert.assertSame(((SqoopException) ex).getErrorCode(),
+ CoreError.CORE_0002);
+ success = true;
+ }
+
+ Assert.assertTrue(success);
+ }
+
+ @Test
+ public void testConfigurationProviderNotSet() throws Exception {
+ boolean success = false;
+ Properties bootProps = new Properties();
+ bootProps.setProperty("foo", "bar");
+ TestUtils.setupTestConfigurationUsingProperties(bootProps, null);
+ try {
+ SqoopConfiguration.getInstance().initialize();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof SqoopException);
+ Assert.assertSame(((SqoopException) ex).getErrorCode(),
+ CoreError.CORE_0003);
+ success = true;
+ }
+
+ Assert.assertTrue(success);
+ }
+
+ @Test
+ public void testConfigurationProviderInvalid() throws Exception {
+ boolean success = false;
+ Properties bootProps = new Properties();
+ bootProps.setProperty(ConfigurationConstants.BOOTCFG_CONFIG_PROVIDER,
+ "foobar");
+ TestUtils.setupTestConfigurationUsingProperties(bootProps, null);
+ try {
+ SqoopConfiguration.getInstance().initialize();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof SqoopException);
+ Assert.assertSame(((SqoopException) ex).getErrorCode(),
+ CoreError.CORE_0004);
+
+ success = true;
+ }
+
+ Assert.assertTrue(success);
+ }
+
+ @Test
+ public void testConfiugrationProviderCannotLoad() throws Exception {
+ boolean success = false;
+ Properties bootProps = new Properties();
+ bootProps.setProperty(ConfigurationConstants.BOOTCFG_CONFIG_PROVIDER,
+ MockInvalidConfigurationProvider.class.getCanonicalName());
+ TestUtils.setupTestConfigurationUsingProperties(bootProps, null);
+ try {
+ SqoopConfiguration.getInstance().initialize();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof SqoopException);
+ Assert.assertSame(((SqoopException) ex).getErrorCode(),
+ CoreError.CORE_0005);
+ success = true;
+ }
+
+ Assert.assertTrue(success);
+ }
+
+ @Test
+ public void testPropertiesConfigProviderNoFile() throws Exception {
+ boolean success = false;
+ Properties bootProps = new Properties();
+ bootProps.setProperty(ConfigurationConstants.BOOTCFG_CONFIG_PROVIDER,
+ PropertiesConfigurationProvider.class.getCanonicalName());
+ TestUtils.setupTestConfigurationUsingProperties(bootProps, null);
+ try {
+ SqoopConfiguration.getInstance().initialize();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof SqoopException);
+ Assert.assertSame(((SqoopException) ex).getErrorCode(),
+ CoreError.CORE_0006);
+ success = true;
+ }
+
+ Assert.assertTrue(success);
+ }
+
+ @Test
+ public void testSystemNotInitialized() throws Exception {
+ boolean success = false;
+ try {
+ SqoopConfiguration.getInstance().getContext();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof SqoopException);
+ Assert.assertSame(((SqoopException) ex).getErrorCode(),
+ CoreError.CORE_0007);
+ success = true;
+ }
+
+ Assert.assertTrue(success);
+ }
+
+ @Test
+ public void testConfigurationInitSuccess() throws Exception {
+ TestUtils.setupTestConfigurationWithExtraConfig(null, null);
+ SqoopConfiguration.getInstance().initialize();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/core/TestUtils.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/core/TestUtils.java b/core/src/test/java/org/apache/sqoop/core/TestUtils.java
index 5b230ba..65f07ff 100644
--- a/core/src/test/java/org/apache/sqoop/core/TestUtils.java
+++ b/core/src/test/java/org/apache/sqoop/core/TestUtils.java
@@ -17,13 +17,11 @@
*/
package org.apache.sqoop.core;
-import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.util.Enumeration;
import java.util.Properties;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
new file mode 100644
index 0000000..9c39d23
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.driver.DriverConfigUpgrader;
+import org.apache.sqoop.model.*;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ */
+public class TestDriverConfigUpgrader {
+
+ DriverConfigUpgrader upgrader;
+
+ @Before
+ public void initializeUpgrader() {
+ upgrader = new DriverConfigUpgrader();
+ }
+
+ /**
+ * We take the same forms on input and output and we
+ * expect that all values will be correctly transferred.
+ */
+ @Test
+ public void testConnectionUpgrade() {
+ MConnectionForms original = connection1();
+ MConnectionForms target = connection1();
+
+ original.getStringInput("f1.s1").setValue("A");
+ original.getStringInput("f1.s2").setValue("B");
+ original.getIntegerInput("f1.i").setValue(3);
+
+ upgrader.upgrade(original, target);
+
+ assertEquals("A", target.getStringInput("f1.s1").getValue());
+ assertEquals("B", target.getStringInput("f1.s2").getValue());
+ assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
+ }
+
+ /**
+ * We take the same forms on input and output and we
+ * expect that all values will be correctly transferred.
+ */
+ @Test
+ public void testJobUpgrade() {
+ MJobForms original = job1();
+ MJobForms target = job1();
+
+ original.getStringInput("f1.s1").setValue("A");
+ original.getStringInput("f1.s2").setValue("B");
+ original.getIntegerInput("f1.i").setValue(3);
+
+ upgrader.upgrade(original, target);
+
+ assertEquals("A", target.getStringInput("f1.s1").getValue());
+ assertEquals("B", target.getStringInput("f1.s2").getValue());
+ assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
+ }
+
+ /**
+ * Upgrade scenario when new input has been added to the target forms.
+ */
+ @Test
+ public void testNonExistingInput() {
+ MConnectionForms original = connection1();
+ MConnectionForms target = connection2();
+
+ original.getStringInput("f1.s1").setValue("A");
+ original.getStringInput("f1.s2").setValue("B");
+ original.getIntegerInput("f1.i").setValue(3);
+
+ upgrader.upgrade(original, target);
+
+ assertEquals("A", target.getStringInput("f1.s1").getValue());
+ assertNull(target.getStringInput("f1.s2_").getValue());
+ assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
+ }
+
+ /**
+ * Upgrade scenario when entire has been added in the target and
+ * therefore is missing in the original.
+ */
+ @Test
+ public void testNonExistingForm() {
+ MConnectionForms original = connection1();
+ MConnectionForms target = connection3();
+
+ original.getStringInput("f1.s1").setValue("A");
+ original.getStringInput("f1.s2").setValue("B");
+ original.getIntegerInput("f1.i").setValue(3);
+
+ upgrader.upgrade(original, target);
+
+ assertNull(target.getStringInput("f2.s1").getValue());
+ assertNull(target.getStringInput("f2.s2").getValue());
+ assertNull(target.getIntegerInput("f2.i").getValue());
+ }
+
+ MJobForms job1() {
+ return new MJobForms(forms1());
+ }
+
+ MConnectionForms connection1() {
+ return new MConnectionForms(forms1());
+ }
+
+ MConnectionForms connection2() {
+ return new MConnectionForms(forms2());
+ }
+
+ MConnectionForms connection3() {
+ return new MConnectionForms(forms3());
+ }
+
+ List<MForm> forms1() {
+ List<MForm> list = new LinkedList<MForm>();
+ list.add(new MForm("f1", inputs1("f1")));
+ return list;
+ }
+
+ List<MInput<?>> inputs1(String formName) {
+ List<MInput<?>> list = new LinkedList<MInput<?>>();
+ list.add(new MStringInput(formName + ".s1", false, (short)30));
+ list.add(new MStringInput(formName + ".s2", false, (short)30));
+ list.add(new MIntegerInput(formName + ".i", false));
+ return list;
+ }
+
+ List<MForm> forms2() {
+ List<MForm> list = new LinkedList<MForm>();
+ list.add(new MForm("f1", inputs2("f1")));
+ return list;
+ }
+
+ List<MInput<?>> inputs2(String formName) {
+ List<MInput<?>> list = new LinkedList<MInput<?>>();
+ list.add(new MStringInput(formName + ".s1", false, (short)30));
+ list.add(new MStringInput(formName + ".s2_", false, (short)30));
+ list.add(new MIntegerInput(formName + ".i", false));
+ return list;
+ }
+
+ List<MForm> forms3() {
+ List<MForm> list = new LinkedList<MForm>();
+ list.add(new MForm("f2", inputs1("f2")));
+ return list;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
new file mode 100644
index 0000000..80f45e5
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.ConnectorManager;
+import org.apache.sqoop.connector.spi.SqoopConnector;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.repository.JdbcRepository;
+import org.apache.sqoop.repository.Repository;
+import org.apache.sqoop.repository.RepositoryManager;
+import org.apache.sqoop.request.HttpEventContext;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestJobManager {
+ private JobManager jobManager;
+ private SqoopConnector sqoopConnectorMock;
+ private ConnectorManager connectorMgrMock;
+ private RepositoryManager repositoryManagerMock;
+ private Repository jdbcRepoMock;
+
+ @Before
+ public void setUp() {
+ jobManager = new JobManager();
+ connectorMgrMock = mock(ConnectorManager.class);
+ sqoopConnectorMock = mock(SqoopConnector.class);
+ ConnectorManager.setInstance(connectorMgrMock);
+ repositoryManagerMock = mock(RepositoryManager.class);
+ RepositoryManager.setInstance(repositoryManagerMock);
+ jdbcRepoMock = mock(JdbcRepository.class);
+ }
+
+ @Test
+ public void testCreateJobSubmission() {
+
+ HttpEventContext testCtx = new HttpEventContext();
+ testCtx.setUsername("testUser");
+ MSubmission jobSubmission = jobManager.createJobSubmission(testCtx, 1234L);
+ assertEquals(jobSubmission.getCreationUser(), "testUser");
+ assertEquals(jobSubmission.getLastUpdateUser(), "testUser");
+ }
+
+ @Test
+ public void testGetConnector() {
+ when(connectorMgrMock.getConnector(123l)).thenReturn(sqoopConnectorMock);
+ when(sqoopConnectorMock.getSupportedDirections()).thenReturn(getSupportedDirections());
+ assertEquals(jobManager.getConnector(123l), sqoopConnectorMock);
+ verify(connectorMgrMock, times(1)).getConnector(123l);
+ }
+
+ @Test
+ public void testUnsupportedDirectionForConnector() {
+ // invalid job id/ direction
+ SqoopException exception = new SqoopException(DriverError.DRIVER_0011, "Connector: "
+ + sqoopConnectorMock.getClass().getCanonicalName());
+ List<Direction> supportedDirections = getSupportedDirections();
+ when(sqoopConnectorMock.getSupportedDirections()).thenReturn(supportedDirections);
+
+ try {
+ // invalid direction
+ jobManager.validateSupportedDirection(sqoopConnectorMock, null);
+ } catch (SqoopException ex) {
+ assertEquals(ex.getMessage(), exception.getMessage());
+ verify(sqoopConnectorMock, times(1)).getSupportedDirections();
+ return;
+ }
+
+ fail("Should throw out an exception with message: " + exception.getMessage());
+ }
+
+ @Test
+ public void testGetLink() {
+ MLink testLink = new MLink(123l, null, null);
+ testLink.setEnabled(true);
+ MLink mConnectionSpy = org.mockito.Mockito.spy(testLink);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findLink(123l)).thenReturn(mConnectionSpy);
+ assertEquals(jobManager.getLink(123l), mConnectionSpy);
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findLink(123l);
+ }
+
+ @Test
+ public void testDisabledLink() {
+ MLink testConnection = new MLink(123l, null, null);
+ testConnection.setPersistenceId(1234);
+ testConnection.setEnabled(false);
+ SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection id: "
+ + testConnection.getPersistenceId());
+
+ MLink mConnectionSpy = org.mockito.Mockito.spy(testConnection);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findLink(123l)).thenReturn(mConnectionSpy);
+ try {
+ jobManager.getLink(123l);
+ } catch (SqoopException ex) {
+ assertEquals(ex.getMessage(), exception.getMessage());
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findLink(123l);
+ }
+ }
+
+ @Test
+ public void testGetJob() {
+ MJob testJob = job(123l, 456l);
+ testJob.setEnabled(true);
+ MJob mJobSpy = org.mockito.Mockito.spy(testJob);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findJob(123l)).thenReturn(mJobSpy);
+ assertEquals(jobManager.getJob(123l), mJobSpy);
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findJob(123l);
+ }
+
+ @Test
+ public void testDisabledJob() {
+ MJob testJob = job(123l, 456l);
+ testJob.setEnabled(false);
+ testJob.setPersistenceId(1111);
+ SqoopException exception = new SqoopException(DriverError.DRIVER_0009, "Job id: "
+ + testJob.getPersistenceId());
+
+ MJob mJobSpy = org.mockito.Mockito.spy(testJob);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findJob(123l)).thenReturn(mJobSpy);
+ try {
+ jobManager.getJob(123l);
+ } catch (SqoopException ex) {
+ assertEquals(ex.getMessage(), exception.getMessage());
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findJob(123l);
+ }
+ }
+
+ @Test
+ public void testUnknownJob() {
+ long testJobId = 555l;
+ SqoopException exception = new SqoopException(DriverError.DRIVER_0004, "Unknown job id: "
+ + testJobId);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findJob(testJobId)).thenReturn(null);
+ try {
+ jobManager.getJob(testJobId);
+ } catch (SqoopException ex) {
+ assertEquals(ex.getMessage(), exception.getMessage());
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findJob(testJobId);
+ }
+ }
+
+ private MJob job(long fromId, long toId) {
+ MJob job = new MJob(fromId, toId, 1L, 2L, null, null, null);
+ job.setName("Vampire");
+ job.setCreationUser("Buffy");
+ return job;
+ }
+
+ public List<Direction> getSupportedDirections() {
+ return Arrays.asList(new Direction[] { Direction.FROM, Direction.TO });
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
new file mode 100644
index 0000000..768c978
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+
+import org.apache.sqoop.driver.JobRequest;
+import org.apache.sqoop.utils.ClassUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestJobRequest {
+
+ private JobRequest jobRequest;
+
+ @Before
+ public void initializeSubmissionRequest() {
+ jobRequest = new JobRequest();
+ }
+
+ @Test
+ public void testAddJar() {
+ jobRequest.addJar("A");
+ jobRequest.addJar("B");
+ jobRequest.addJar("A");
+
+ assertEquals(2, jobRequest.getJars().size());
+ assertEquals("A", jobRequest.getJars().get(0));
+ assertEquals("B", jobRequest.getJars().get(1));
+ }
+
+ @Test
+ public void testAddJarForClass() {
+ jobRequest.addJarForClass(TestJobRequest.class);
+ assertEquals(1, jobRequest.getJars().size());
+ assertTrue(jobRequest.getJars().contains(ClassUtils.jarForClass(TestJobRequest.class)));
+ }
+
+ @Test
+ public void testAddJars() {
+ jobRequest.addJars(Arrays.asList("A", "B"));
+ jobRequest.addJars(Arrays.asList("B", "C"));
+
+ assertEquals(3, jobRequest.getJars().size());
+ assertEquals("A", jobRequest.getJars().get(0));
+ assertEquals("B", jobRequest.getJars().get(1));
+ assertEquals("C", jobRequest.getJars().get(2));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java
deleted file mode 100644
index 81d197e..0000000
--- a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.model.*;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-/**
- */
-public class TestFrameworkMetadataUpgrader {
-
- FrameworkMetadataUpgrader upgrader;
-
- @Before
- public void initializeUpgrader() {
- upgrader = new FrameworkMetadataUpgrader();
- }
-
- /**
- * We take the same forms on input and output and we
- * expect that all values will be correctly transferred.
- */
- @Test
- public void testConnectionUpgrade() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection1();
-
- original.getStringInput("f1.s1").setValue("A");
- original.getStringInput("f1.s2").setValue("B");
- original.getIntegerInput("f1.i").setValue(3);
-
- upgrader.upgrade(original, target);
-
- assertEquals("A", target.getStringInput("f1.s1").getValue());
- assertEquals("B", target.getStringInput("f1.s2").getValue());
- assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
- }
-
- /**
- * We take the same forms on input and output and we
- * expect that all values will be correctly transferred.
- */
- @Test
- public void testJobUpgrade() {
- MJobForms original = job1();
- MJobForms target = job1();
-
- original.getStringInput("f1.s1").setValue("A");
- original.getStringInput("f1.s2").setValue("B");
- original.getIntegerInput("f1.i").setValue(3);
-
- upgrader.upgrade(original, target);
-
- assertEquals("A", target.getStringInput("f1.s1").getValue());
- assertEquals("B", target.getStringInput("f1.s2").getValue());
- assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
- }
-
- /**
- * Upgrade scenario when new input has been added to the target forms.
- */
- @Test
- public void testNonExistingInput() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection2();
-
- original.getStringInput("f1.s1").setValue("A");
- original.getStringInput("f1.s2").setValue("B");
- original.getIntegerInput("f1.i").setValue(3);
-
- upgrader.upgrade(original, target);
-
- assertEquals("A", target.getStringInput("f1.s1").getValue());
- assertNull(target.getStringInput("f1.s2_").getValue());
- assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
- }
-
- /**
- * Upgrade scenario when entire has been added in the target and
- * therefore is missing in the original.
- */
- @Test
- public void testNonExistingForm() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection3();
-
- original.getStringInput("f1.s1").setValue("A");
- original.getStringInput("f1.s2").setValue("B");
- original.getIntegerInput("f1.i").setValue(3);
-
- upgrader.upgrade(original, target);
-
- assertNull(target.getStringInput("f2.s1").getValue());
- assertNull(target.getStringInput("f2.s2").getValue());
- assertNull(target.getIntegerInput("f2.i").getValue());
- }
-
- MJobForms job1() {
- return new MJobForms(forms1());
- }
-
- MConnectionForms connection1() {
- return new MConnectionForms(forms1());
- }
-
- MConnectionForms connection2() {
- return new MConnectionForms(forms2());
- }
-
- MConnectionForms connection3() {
- return new MConnectionForms(forms3());
- }
-
- List<MForm> forms1() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f1", inputs1("f1")));
- return list;
- }
-
- List<MInput<?>> inputs1(String formName) {
- List<MInput<?>> list = new LinkedList<MInput<?>>();
- list.add(new MStringInput(formName + ".s1", false, (short)30));
- list.add(new MStringInput(formName + ".s2", false, (short)30));
- list.add(new MIntegerInput(formName + ".i", false));
- return list;
- }
-
- List<MForm> forms2() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f1", inputs2("f1")));
- return list;
- }
-
- List<MInput<?>> inputs2(String formName) {
- List<MInput<?>> list = new LinkedList<MInput<?>>();
- list.add(new MStringInput(formName + ".s1", false, (short)30));
- list.add(new MStringInput(formName + ".s2_", false, (short)30));
- list.add(new MIntegerInput(formName + ".i", false));
- return list;
- }
-
- List<MForm> forms3() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f2", inputs1("f2")));
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java
deleted file mode 100644
index 69dd028..0000000
--- a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-
-/**
- * NOTE(VB): This test class will soon be removed with the Validator refactoring
- */
-public class TestFrameworkValidator {
-
-// FrameworkValidator validator;
-//
-// @Before
-// public void setUp() {
-// validator = new FrameworkValidator();
-// }
-//
-// @Test
-// public void testConnectionValidation() {
-// ConnectionConfiguration connectionConfiguration = new ConnectionConfiguration();
-//
-// Validation validation = validator.validateConnection(connectionConfiguration);
-// assertEquals(Status.FINE, validation.getStatus());
-// assertEquals(0, validation.getMessages().size());
-// }
-//
-// @Test
-// public void testExportJobValidation() {
-// ExportJobConfiguration configuration;
-// Validation validation;
-//
-// // Empty form is not allowed
-// configuration = new ExportJobConfiguration();
-// validation = validator.validateJob(MJob.Type.EXPORT, configuration);
-// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("input.inputDirectory")));
-//
-// // Explicitly setting extractors and loaders
-// configuration = new ExportJobConfiguration();
-// configuration.input.inputDirectory = "/czech/republic";
-// configuration.throttling.extractors = 3;
-// configuration.throttling.loaders = 3;
-//
-// validation = validator.validateJob(MJob.Type.EXPORT, configuration);
-// assertEquals(Status.FINE, validation.getStatus());
-// assertEquals(0, validation.getMessages().size());
-//
-// // Negative and zero values for extractors and loaders
-// configuration = new ExportJobConfiguration();
-// configuration.input.inputDirectory = "/czech/republic";
-// configuration.throttling.extractors = 0;
-// configuration.throttling.loaders = -1;
-//
-// validation = validator.validateJob(MJob.Type.EXPORT, configuration);
-// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.extractors")));
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.loaders")));
-// }
-//
-//
-// @Test
-// public void testImportJobValidation() {
-// ImportJobConfiguration configuration;
-// Validation validation;
-//
-// // Empty form is not allowed
-// configuration = new ImportJobConfiguration();
-// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
-// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.outputDirectory")));
-//
-// // Explicitly setting extractors and loaders
-// configuration = new ImportJobConfiguration();
-// configuration.output.outputDirectory = "/czech/republic";
-// configuration.throttling.extractors = 3;
-// configuration.throttling.loaders = 3;
-//
-// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
-// assertEquals(Status.FINE, validation.getStatus());
-// assertEquals(0, validation.getMessages().size());
-//
-// // Negative and zero values for extractors and loaders
-// configuration = new ImportJobConfiguration();
-// configuration.output.outputDirectory = "/czech/republic";
-// configuration.throttling.extractors = 0;
-// configuration.throttling.loaders = -1;
-//
-// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
-// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.extractors")));
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.loaders")));
-//
-// // specifying both compression as well as customCompression is
-// // unacceptable
-// configuration = new ImportJobConfiguration();
-// configuration.output.outputDirectory = "/czech/republic";
-// configuration.throttling.extractors = 2;
-// configuration.throttling.loaders = 2;
-// configuration.output.compression = OutputCompression.BZIP2;
-// configuration.output.customCompression = "some.compression.codec";
-//
-// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
-// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression")));
-//
-// // specifying a customCompression is fine
-// configuration = new ImportJobConfiguration();
-// configuration.output.outputDirectory = "/czech/republic";
-// configuration.throttling.extractors = 2;
-// configuration.throttling.loaders = 2;
-// configuration.output.compression = OutputCompression.CUSTOM;
-// configuration.output.customCompression = "some.compression.codec";
-//
-// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
-// assertEquals(Status.FINE, validation.getStatus());
-//
-// // specifying a customCompression without codec name is unacceptable
-// configuration = new ImportJobConfiguration();
-// configuration.output.outputDirectory = "/czech/republic";
-// configuration.throttling.extractors = 2;
-// configuration.throttling.loaders = 2;
-// configuration.output.compression = OutputCompression.CUSTOM;
-// configuration.output.customCompression = "";
-//
-// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
-// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression")));
-//
-// configuration = new ImportJobConfiguration();
-// configuration.output.outputDirectory = "/czech/republic";
-// configuration.throttling.extractors = 2;
-// configuration.throttling.loaders = 2;
-// configuration.output.compression = OutputCompression.CUSTOM;
-// configuration.output.customCompression = null;
-//
-// validation = validator.validateJob(MJob.Type.IMPORT, configuration);
-// assertEquals(Status.UNACCEPTABLE, validation.getStatus());
-// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression")));
-//
-// }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/framework/TestJobManager.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestJobManager.java b/core/src/test/java/org/apache/sqoop/framework/TestJobManager.java
deleted file mode 100644
index 2732b1c..0000000
--- a/core/src/test/java/org/apache/sqoop/framework/TestJobManager.java
+++ /dev/null
@@ -1,173 +0,0 @@
-package org.apache.sqoop.framework;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MSubmission;
-import org.apache.sqoop.repository.JdbcRepository;
-import org.apache.sqoop.repository.Repository;
-import org.apache.sqoop.repository.RepositoryManager;
-import org.apache.sqoop.request.HttpEventContext;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestJobManager {
- private JobManager jobManager;
- private SqoopConnector sqoopConnectorMock;
- private ConnectorManager connectorMgrMock;
- private RepositoryManager repositoryManagerMock;
- private Repository jdbcRepoMock;
-
- @Before
- public void setUp() {
- jobManager = new JobManager();
- connectorMgrMock = mock(ConnectorManager.class);
- sqoopConnectorMock = mock(SqoopConnector.class);
- ConnectorManager.setInstance(connectorMgrMock);
- repositoryManagerMock = mock(RepositoryManager.class);
- RepositoryManager.setInstance(repositoryManagerMock);
- jdbcRepoMock = mock(JdbcRepository.class);
- }
-
- @Test
- public void testCreateJobSubmission() {
-
- HttpEventContext testCtx = new HttpEventContext();
- testCtx.setUsername("testUser");
- MSubmission jobSubmission = jobManager.createJobSubmission(testCtx, 1234L);
- assertEquals(jobSubmission.getCreationUser(), "testUser");
- assertEquals(jobSubmission.getLastUpdateUser(), "testUser");
- }
-
- @Test
- public void testGetConnector() {
- when(connectorMgrMock.getConnector(123l)).thenReturn(sqoopConnectorMock);
- when(sqoopConnectorMock.getSupportedDirections()).thenReturn(getSupportedDirections());
- assertEquals(jobManager.getConnector(123l), sqoopConnectorMock);
- verify(connectorMgrMock, times(1)).getConnector(123l);
- }
-
- @Test
- public void testUnsupportedDirectionForConnector() {
- // invalid job id/ direction
- SqoopException exception = new SqoopException(FrameworkError.FRAMEWORK_0011, "Connector: "
- + sqoopConnectorMock.getClass().getCanonicalName());
- List<Direction> supportedDirections = getSupportedDirections();
- when(sqoopConnectorMock.getSupportedDirections()).thenReturn(supportedDirections);
-
- try {
- // invalid direction
- jobManager.validateSupportedDirection(sqoopConnectorMock, null);
- } catch (SqoopException ex) {
- assertEquals(ex.getMessage(), exception.getMessage());
- verify(sqoopConnectorMock, times(1)).getSupportedDirections();
- return;
- }
-
- fail("Should throw out an exception with message: " + exception.getMessage());
- }
-
- @Test
- public void testGetConnection() {
- MConnection testConnection = new MConnection(123l, null, null);
- testConnection.setEnabled(true);
- MConnection mConnectionSpy = org.mockito.Mockito.spy(testConnection);
- when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
- when(jdbcRepoMock.findConnection(123l)).thenReturn(mConnectionSpy);
- assertEquals(jobManager.getConnection(123l), mConnectionSpy);
- verify(repositoryManagerMock, times(1)).getRepository();
- verify(jdbcRepoMock, times(1)).findConnection(123l);
- }
-
- @Test
- public void testDisabledConnection() {
- MConnection testConnection = new MConnection(123l, null, null);
- testConnection.setPersistenceId(1234);
- testConnection.setEnabled(false);
- SqoopException exception = new SqoopException(FrameworkError.FRAMEWORK_0010, "Connection id: "
- + testConnection.getPersistenceId());
-
- MConnection mConnectionSpy = org.mockito.Mockito.spy(testConnection);
- when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
- when(jdbcRepoMock.findConnection(123l)).thenReturn(mConnectionSpy);
- try {
- jobManager.getConnection(123l);
- } catch (SqoopException ex) {
- assertEquals(ex.getMessage(), exception.getMessage());
- verify(repositoryManagerMock, times(1)).getRepository();
- verify(jdbcRepoMock, times(1)).findConnection(123l);
- }
- }
-
- @Test
- public void testGetJob() {
- MJob testJob = job(123l, 456l);
- testJob.setEnabled(true);
- MJob mJobSpy = org.mockito.Mockito.spy(testJob);
- when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
- when(jdbcRepoMock.findJob(123l)).thenReturn(mJobSpy);
- assertEquals(jobManager.getJob(123l), mJobSpy);
- verify(repositoryManagerMock, times(1)).getRepository();
- verify(jdbcRepoMock, times(1)).findJob(123l);
- }
-
- @Test
- public void testDisabledJob() {
- MJob testJob = job(123l, 456l);
- testJob.setEnabled(false);
- testJob.setPersistenceId(1111);
- SqoopException exception = new SqoopException(FrameworkError.FRAMEWORK_0009, "Job id: "
- + testJob.getPersistenceId());
-
- MJob mJobSpy = org.mockito.Mockito.spy(testJob);
- when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
- when(jdbcRepoMock.findJob(123l)).thenReturn(mJobSpy);
- try {
- jobManager.getJob(123l);
- } catch (SqoopException ex) {
- assertEquals(ex.getMessage(), exception.getMessage());
- verify(repositoryManagerMock, times(1)).getRepository();
- verify(jdbcRepoMock, times(1)).findJob(123l);
- }
- }
-
- @Test
- public void testUnknownJob() {
- long testJobId = 555l;
- SqoopException exception = new SqoopException(FrameworkError.FRAMEWORK_0004, "Unknown job id: "
- + testJobId);
- when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
- when(jdbcRepoMock.findJob(testJobId)).thenReturn(null);
- try {
- jobManager.getJob(testJobId);
- } catch (SqoopException ex) {
- assertEquals(ex.getMessage(), exception.getMessage());
- verify(repositoryManagerMock, times(1)).getRepository();
- verify(jdbcRepoMock, times(1)).findJob(testJobId);
- }
- }
-
- private MJob job(long fromId, long toId) {
- MJob job = new MJob(fromId, toId, 1L, 2L, null, null, null);
- job.setName("Vampire");
- job.setCreationUser("Buffy");
- return job;
- }
-
- public List<Direction> getSupportedDirections() {
- return Arrays.asList(new Direction[] { Direction.FROM, Direction.TO });
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/core/src/test/java/org/apache/sqoop/framework/TestJobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestJobRequest.java b/core/src/test/java/org/apache/sqoop/framework/TestJobRequest.java
deleted file mode 100644
index 6ca1c6a..0000000
--- a/core/src/test/java/org/apache/sqoop/framework/TestJobRequest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.utils.ClassUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- *
- */
-public class TestJobRequest {
-
- private JobRequest jobRequest;
-
- @Before
- public void initializeSubmissionRequest() {
- jobRequest = new JobRequest();
- }
-
- @Test
- public void testAddJar() {
- jobRequest.addJar("A");
- jobRequest.addJar("B");
- jobRequest.addJar("A");
-
- assertEquals(2, jobRequest.getJars().size());
- assertEquals("A", jobRequest.getJars().get(0));
- assertEquals("B", jobRequest.getJars().get(1));
- }
-
- @Test
- public void testAddJarForClass() {
- jobRequest.addJarForClass(TestJobRequest.class);
- jobRequest.addJarForClass(TestFrameworkValidator.class);
-
- assertEquals(1, jobRequest.getJars().size());
- assertTrue(jobRequest.getJars().contains(ClassUtils.jarForClass(TestJobRequest.class)));
- }
-
- @Test
- public void testAddJars() {
- jobRequest.addJars(Arrays.asList("A", "B"));
- jobRequest.addJars(Arrays.asList("B", "C"));
-
- assertEquals(3, jobRequest.getJars().size());
- assertEquals("A", jobRequest.getJars().get(0));
- assertEquals("B", jobRequest.getJars().get(1));
- assertEquals("C", jobRequest.getJars().get(2));
- }
-}
[22/52] [abbrv] SQOOP-1496: Sqoop2: Revisit/Refactor the
SubmissionEngine/ExecutionEngine APIs
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/test/java/org/apache/sqoop/framework/TestJobManager.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestJobManager.java b/core/src/test/java/org/apache/sqoop/framework/TestJobManager.java
new file mode 100644
index 0000000..2732b1c
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/framework/TestJobManager.java
@@ -0,0 +1,173 @@
+package org.apache.sqoop.framework;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.ConnectorManager;
+import org.apache.sqoop.connector.spi.SqoopConnector;
+import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.repository.JdbcRepository;
+import org.apache.sqoop.repository.Repository;
+import org.apache.sqoop.repository.RepositoryManager;
+import org.apache.sqoop.request.HttpEventContext;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestJobManager {
+ private JobManager jobManager;
+ private SqoopConnector sqoopConnectorMock;
+ private ConnectorManager connectorMgrMock;
+ private RepositoryManager repositoryManagerMock;
+ private Repository jdbcRepoMock;
+
+ @Before
+ public void setUp() {
+ jobManager = new JobManager();
+ connectorMgrMock = mock(ConnectorManager.class);
+ sqoopConnectorMock = mock(SqoopConnector.class);
+ ConnectorManager.setInstance(connectorMgrMock);
+ repositoryManagerMock = mock(RepositoryManager.class);
+ RepositoryManager.setInstance(repositoryManagerMock);
+ jdbcRepoMock = mock(JdbcRepository.class);
+ }
+
+ @Test
+ public void testCreateJobSubmission() {
+
+ HttpEventContext testCtx = new HttpEventContext();
+ testCtx.setUsername("testUser");
+ MSubmission jobSubmission = jobManager.createJobSubmission(testCtx, 1234L);
+ assertEquals(jobSubmission.getCreationUser(), "testUser");
+ assertEquals(jobSubmission.getLastUpdateUser(), "testUser");
+ }
+
+ @Test
+ public void testGetConnector() {
+ when(connectorMgrMock.getConnector(123l)).thenReturn(sqoopConnectorMock);
+ when(sqoopConnectorMock.getSupportedDirections()).thenReturn(getSupportedDirections());
+ assertEquals(jobManager.getConnector(123l), sqoopConnectorMock);
+ verify(connectorMgrMock, times(1)).getConnector(123l);
+ }
+
+ @Test
+ public void testUnsupportedDirectionForConnector() {
+ // invalid job id/ direction
+ SqoopException exception = new SqoopException(FrameworkError.FRAMEWORK_0011, "Connector: "
+ + sqoopConnectorMock.getClass().getCanonicalName());
+ List<Direction> supportedDirections = getSupportedDirections();
+ when(sqoopConnectorMock.getSupportedDirections()).thenReturn(supportedDirections);
+
+ try {
+ // invalid direction
+ jobManager.validateSupportedDirection(sqoopConnectorMock, null);
+ } catch (SqoopException ex) {
+ assertEquals(ex.getMessage(), exception.getMessage());
+ verify(sqoopConnectorMock, times(1)).getSupportedDirections();
+ return;
+ }
+
+ fail("Should throw out an exception with message: " + exception.getMessage());
+ }
+
+ @Test
+ public void testGetConnection() {
+ MConnection testConnection = new MConnection(123l, null, null);
+ testConnection.setEnabled(true);
+ MConnection mConnectionSpy = org.mockito.Mockito.spy(testConnection);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findConnection(123l)).thenReturn(mConnectionSpy);
+ assertEquals(jobManager.getConnection(123l), mConnectionSpy);
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findConnection(123l);
+ }
+
+ @Test
+ public void testDisabledConnection() {
+ MConnection testConnection = new MConnection(123l, null, null);
+ testConnection.setPersistenceId(1234);
+ testConnection.setEnabled(false);
+ SqoopException exception = new SqoopException(FrameworkError.FRAMEWORK_0010, "Connection id: "
+ + testConnection.getPersistenceId());
+
+ MConnection mConnectionSpy = org.mockito.Mockito.spy(testConnection);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findConnection(123l)).thenReturn(mConnectionSpy);
+ try {
+ jobManager.getConnection(123l);
+ } catch (SqoopException ex) {
+ assertEquals(ex.getMessage(), exception.getMessage());
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findConnection(123l);
+ }
+ }
+
+ @Test
+ public void testGetJob() {
+ MJob testJob = job(123l, 456l);
+ testJob.setEnabled(true);
+ MJob mJobSpy = org.mockito.Mockito.spy(testJob);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findJob(123l)).thenReturn(mJobSpy);
+ assertEquals(jobManager.getJob(123l), mJobSpy);
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findJob(123l);
+ }
+
+ @Test
+ public void testDisabledJob() {
+ MJob testJob = job(123l, 456l);
+ testJob.setEnabled(false);
+ testJob.setPersistenceId(1111);
+ SqoopException exception = new SqoopException(FrameworkError.FRAMEWORK_0009, "Job id: "
+ + testJob.getPersistenceId());
+
+ MJob mJobSpy = org.mockito.Mockito.spy(testJob);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findJob(123l)).thenReturn(mJobSpy);
+ try {
+ jobManager.getJob(123l);
+ } catch (SqoopException ex) {
+ assertEquals(ex.getMessage(), exception.getMessage());
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findJob(123l);
+ }
+ }
+
+ @Test
+ public void testUnknownJob() {
+ long testJobId = 555l;
+ SqoopException exception = new SqoopException(FrameworkError.FRAMEWORK_0004, "Unknown job id: "
+ + testJobId);
+ when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
+ when(jdbcRepoMock.findJob(testJobId)).thenReturn(null);
+ try {
+ jobManager.getJob(testJobId);
+ } catch (SqoopException ex) {
+ assertEquals(ex.getMessage(), exception.getMessage());
+ verify(repositoryManagerMock, times(1)).getRepository();
+ verify(jdbcRepoMock, times(1)).findJob(testJobId);
+ }
+ }
+
+ private MJob job(long fromId, long toId) {
+ MJob job = new MJob(fromId, toId, 1L, 2L, null, null, null);
+ job.setName("Vampire");
+ job.setCreationUser("Buffy");
+ return job;
+ }
+
+ public List<Direction> getSupportedDirections() {
+ return Arrays.asList(new Direction[] { Direction.FROM, Direction.TO });
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/test/java/org/apache/sqoop/framework/TestJobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestJobRequest.java b/core/src/test/java/org/apache/sqoop/framework/TestJobRequest.java
new file mode 100644
index 0000000..6ca1c6a
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/framework/TestJobRequest.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.framework;
+
+import org.apache.sqoop.utils.ClassUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ */
+public class TestJobRequest {
+
+ private JobRequest jobRequest;
+
+ @Before
+ public void initializeSubmissionRequest() {
+ jobRequest = new JobRequest();
+ }
+
+ @Test
+ public void testAddJar() {
+ jobRequest.addJar("A");
+ jobRequest.addJar("B");
+ jobRequest.addJar("A");
+
+ assertEquals(2, jobRequest.getJars().size());
+ assertEquals("A", jobRequest.getJars().get(0));
+ assertEquals("B", jobRequest.getJars().get(1));
+ }
+
+ @Test
+ public void testAddJarForClass() {
+ jobRequest.addJarForClass(TestJobRequest.class);
+ jobRequest.addJarForClass(TestFrameworkValidator.class);
+
+ assertEquals(1, jobRequest.getJars().size());
+ assertTrue(jobRequest.getJars().contains(ClassUtils.jarForClass(TestJobRequest.class)));
+ }
+
+ @Test
+ public void testAddJars() {
+ jobRequest.addJars(Arrays.asList("A", "B"));
+ jobRequest.addJars(Arrays.asList("B", "C"));
+
+ assertEquals(3, jobRequest.getJars().size());
+ assertEquals("A", jobRequest.getJars().get(0));
+ assertEquals("B", jobRequest.getJars().get(1));
+ assertEquals("C", jobRequest.getJars().get(2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java b/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java
deleted file mode 100644
index 3078ed2..0000000
--- a/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.framework;
-
-import org.apache.sqoop.utils.ClassUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- *
- */
-public class TestSubmissionRequest {
-
- private SubmissionRequest submissionRequest;
-
- @Before
- public void initializeSubmissionRequest() {
- submissionRequest = new SubmissionRequest();
- }
-
- @Test
- public void testAddJar() {
- submissionRequest.addJar("A");
- submissionRequest.addJar("B");
- submissionRequest.addJar("A");
-
- assertEquals(2, submissionRequest.getJars().size());
- assertEquals("A", submissionRequest.getJars().get(0));
- assertEquals("B", submissionRequest.getJars().get(1));
- }
-
- @Test
- public void testAddJarForClass() {
- submissionRequest.addJarForClass(TestSubmissionRequest.class);
- submissionRequest.addJarForClass(TestFrameworkValidator.class);
-
- assertEquals(1, submissionRequest.getJars().size());
- assertTrue(submissionRequest.getJars().contains(ClassUtils.jarForClass(TestSubmissionRequest.class)));
- }
-
- @Test
- public void testAddJars() {
- submissionRequest.addJars(Arrays.asList("A", "B"));
- submissionRequest.addJars(Arrays.asList("B", "C"));
-
- assertEquals(3, submissionRequest.getJars().size());
- assertEquals("A", submissionRequest.getJars().get(0));
- assertEquals("B", submissionRequest.getJars().get(1));
- assertEquals("C", submissionRequest.getJars().get(2));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
index 50daa62..0fcae75 100644
--- a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
+++ b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
@@ -52,39 +52,39 @@ import static org.mockito.Mockito.*;
public class TestJdbcRepository {
- private JdbcRepository repo;
- private JdbcRepositoryTransaction tx;
- private ConnectorManager connectorMgr;
- private FrameworkManager frameworkMgr;
- private JdbcRepositoryHandler repoHandler;
- private Validator validator;
- private MetadataUpgrader upgrader;
+ private JdbcRepository repoSpy;
+ private JdbcRepositoryTransaction repoTransactionMock;
+ private ConnectorManager connectorMgrMock;
+ private FrameworkManager frameworkMgrMock;
+ private JdbcRepositoryHandler repoHandlerMock;
+ private Validator validatorMock;
+ private MetadataUpgrader upgraderMock;
- private Validation valid;
- private Validation invalid;
+ private Validation validRepoMock;
+ private Validation invalidRepoMock;
@Before
public void setUp() throws Exception {
- tx = mock(JdbcRepositoryTransaction.class);
- connectorMgr = mock(ConnectorManager.class);
- frameworkMgr = mock(FrameworkManager.class);
- repoHandler = mock(JdbcRepositoryHandler.class);
- validator = mock(Validator.class);
- upgrader = mock(MetadataUpgrader.class);
- repo = spy(new JdbcRepository(repoHandler, null));
+ repoTransactionMock = mock(JdbcRepositoryTransaction.class);
+ connectorMgrMock = mock(ConnectorManager.class);
+ frameworkMgrMock = mock(FrameworkManager.class);
+ repoHandlerMock = mock(JdbcRepositoryHandler.class);
+ validatorMock = mock(Validator.class);
+ upgraderMock = mock(MetadataUpgrader.class);
+ repoSpy = spy(new JdbcRepository(repoHandlerMock, null));
// setup transaction and connector manager
- doReturn(tx).when(repo).getTransaction();
- ConnectorManager.setInstance(connectorMgr);
- FrameworkManager.setInstance(frameworkMgr);
+ doReturn(repoTransactionMock).when(repoSpy).getTransaction();
+ ConnectorManager.setInstance(connectorMgrMock);
+ FrameworkManager.setInstance(frameworkMgrMock);
- valid = mock(Validation.class);
- when(valid.getStatus()).thenReturn(Status.ACCEPTABLE);
- invalid = mock(Validation.class);
- when(invalid.getStatus()).thenReturn(Status.UNACCEPTABLE);
+ validRepoMock = mock(Validation.class);
+ when(validRepoMock.getStatus()).thenReturn(Status.ACCEPTABLE);
+ invalidRepoMock = mock(Validation.class);
+ when(invalidRepoMock.getStatus()).thenReturn(Status.UNACCEPTABLE);
- doNothing().when(upgrader).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
- doNothing().when(upgrader).upgrade(any(MJobForms.class), any(MJobForms.class));
+ doNothing().when(upgraderMock).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
+ doNothing().when(upgraderMock).upgrade(any(MJobForms.class), any(MJobForms.class));
}
/**
@@ -95,20 +95,20 @@ public class TestJdbcRepository {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1, "1.0");
- when(repoHandler.findConnector(anyString(), any(Connection.class))).thenReturn(oldConnector);
+ when(repoHandlerMock.findConnector(anyString(), any(Connection.class))).thenReturn(oldConnector);
// make the upgradeConnector to throw an exception to prove that it has been called
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"upgradeConnector() has been called.");
- doThrow(exception).when(connectorMgr).getConnector(anyString());
+ doThrow(exception).when(connectorMgrMock).getConnector(anyString());
try {
- repo.registerConnector(newConnector, true);
+ repoSpy.registerConnector(newConnector, true);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnector(anyString(), any(Connection.class));
- verify(connectorMgr, times(1)).getConnector(anyString());
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnector(anyString(), any(Connection.class));
+ verify(connectorMgrMock, times(1)).getConnector(anyString());
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -123,13 +123,13 @@ public class TestJdbcRepository {
MConnector newConnector = connector(1, "1.1");
MConnector oldConnector = connector(1);
- when(repoHandler.findConnector(anyString(), any(Connection.class))).thenReturn(oldConnector);
+ when(repoHandlerMock.findConnector(anyString(), any(Connection.class))).thenReturn(oldConnector);
try {
- repo.registerConnector(newConnector, false);
+ repoSpy.registerConnector(newConnector, false);
} catch (SqoopException ex) {
- verify(repoHandler, times(1)).findConnector(anyString(), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnector(anyString(), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
assertEquals(ex.getErrorCode(), RepositoryError.JDBCREPO_0026);
return ;
}
@@ -145,20 +145,20 @@ public class TestJdbcRepository {
MFramework newFramework = framework();
MFramework oldFramework = anotherFramework();
- when(repoHandler.findFramework(any(Connection.class))).thenReturn(oldFramework);
+ when(repoHandlerMock.findFramework(any(Connection.class))).thenReturn(oldFramework);
// make the upgradeFramework to throw an exception to prove that it has been called
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"upgradeFramework() has been called.");
- doThrow(exception).when(repoHandler).findConnections(any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).findConnections(any(Connection.class));
try {
- repo.registerFramework(newFramework, true);
+ repoSpy.registerFramework(newFramework, true);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findFramework(any(Connection.class));
- verify(repoHandler, times(1)).findConnections(any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findFramework(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -173,14 +173,14 @@ public class TestJdbcRepository {
MFramework newFramework = framework();
MFramework oldFramework = anotherFramework();
- when(repoHandler.findFramework(any(Connection.class))).thenReturn(oldFramework);
+ when(repoHandlerMock.findFramework(any(Connection.class))).thenReturn(oldFramework);
try {
- repo.registerFramework(newFramework, false);
+ repoSpy.registerFramework(newFramework, false);
} catch (SqoopException ex) {
assertEquals(ex.getErrorCode(), RepositoryError.JDBCREPO_0026);
- verify(repoHandler, times(1)).findFramework(any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findFramework(any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -198,53 +198,53 @@ public class TestJdbcRepository {
// prepare the sqoop connector
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validator.validateConnection(any(MConnection.class))).thenReturn(valid);
- when(validator.validateJob(any(MJob.class))).thenReturn(valid);
- when(sqconnector.getValidator()).thenReturn(validator);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
+ when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
- when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
+ when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
// prepare the connections and jobs
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
// mock necessary methods for upgradeConnector() procedure
- doReturn(connectionList).when(repo).findConnectionsForConnector(anyLong());
- doReturn(jobList).when(repo).findJobsForConnector(anyLong());
- doNothing().when(repo).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
- doNothing().when(repo).updateJob(any(MJob.class), any(RepositoryTransaction.class));
- doNothing().when(repo).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
-
- repo.upgradeConnector(oldConnector, newConnector);
-
- InOrder repoOrder = inOrder(repo);
- InOrder txOrder = inOrder(tx);
- InOrder upgraderOrder = inOrder(upgrader);
- InOrder validatorOrder = inOrder(validator);
-
- repoOrder.verify(repo, times(1)).findConnectionsForConnector(anyLong());
- repoOrder.verify(repo, times(1)).findJobsForConnector(anyLong());
- repoOrder.verify(repo, times(1)).getTransaction();
- repoOrder.verify(repo, times(1)).deleteJobInputs(1, tx);
- repoOrder.verify(repo, times(1)).deleteJobInputs(2, tx);
- repoOrder.verify(repo, times(1)).deleteConnectionInputs(1, tx);
- repoOrder.verify(repo, times(1)).deleteConnectionInputs(2, tx);
- repoOrder.verify(repo, times(1)).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
- repoOrder.verify(repo, times(2)).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
- repoOrder.verify(repo, times(4)).updateJob(any(MJob.class), any(RepositoryTransaction.class));
+ doReturn(connectionList).when(repoSpy).findConnectionsForConnector(anyLong());
+ doReturn(jobList).when(repoSpy).findJobsForConnector(anyLong());
+ doNothing().when(repoSpy).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
+
+ repoSpy.upgradeConnector(oldConnector, newConnector);
+
+ InOrder repoOrder = inOrder(repoSpy);
+ InOrder txOrder = inOrder(repoTransactionMock);
+ InOrder upgraderOrder = inOrder(upgraderMock);
+ InOrder validatorOrder = inOrder(validatorMock);
+
+ repoOrder.verify(repoSpy, times(1)).findConnectionsForConnector(anyLong());
+ repoOrder.verify(repoSpy, times(1)).findJobsForConnector(anyLong());
+ repoOrder.verify(repoSpy, times(1)).getTransaction();
+ repoOrder.verify(repoSpy, times(1)).deleteJobInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteJobInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).updateConnector(any(MConnector.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(2)).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(4)).updateJob(any(MJob.class), any(RepositoryTransaction.class));
repoOrder.verifyNoMoreInteractions();
- txOrder.verify(tx, times(1)).begin();
- txOrder.verify(tx, times(1)).commit();
- txOrder.verify(tx, times(1)).close();
+ txOrder.verify(repoTransactionMock, times(1)).begin();
+ txOrder.verify(repoTransactionMock, times(1)).commit();
+ txOrder.verify(repoTransactionMock, times(1)).close();
txOrder.verifyNoMoreInteractions();
- upgraderOrder.verify(upgrader, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
- upgraderOrder.verify(upgrader, times(4)).upgrade(any(MJobForms.class), any(MJobForms.class));
+ upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
+ upgraderOrder.verify(upgraderMock, times(4)).upgrade(any(MJobForms.class), any(MJobForms.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validator, times(2)).validateConnection(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateConnection(anyObject());
// @TODO(Abe): Re-enable job validation?
- validatorOrder.verify(validator, times(0)).validateJob(anyObject());
+ validatorOrder.verify(validatorMock, times(0)).validateJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
}
@@ -321,49 +321,49 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeWithValidConnectionsAndJobs() {
MFramework newFramework = framework();
- when(validator.validateConnection(any(MConnection.class))).thenReturn(valid);
- when(validator.validateJob(any(MJob.class))).thenReturn(valid);
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
- when(frameworkMgr.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(frameworkMgr.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(frameworkMgrMock.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(frameworkMgrMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repo).findConnections();
- doReturn(jobList).when(repo).findJobs();
- doNothing().when(repo).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
- doNothing().when(repo).updateJob(any(MJob.class), any(RepositoryTransaction.class));
- doNothing().when(repo).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
-
- repo.upgradeFramework(newFramework);
-
- InOrder repoOrder = inOrder(repo);
- InOrder txOrder = inOrder(tx);
- InOrder upgraderOrder = inOrder(upgrader);
- InOrder validatorOrder = inOrder(validator);
-
- repoOrder.verify(repo, times(1)).findConnections();
- repoOrder.verify(repo, times(1)).findJobs();
- repoOrder.verify(repo, times(1)).getTransaction();
- repoOrder.verify(repo, times(1)).deleteJobInputs(1, tx);
- repoOrder.verify(repo, times(1)).deleteJobInputs(2, tx);
- repoOrder.verify(repo, times(1)).deleteConnectionInputs(1, tx);
- repoOrder.verify(repo, times(1)).deleteConnectionInputs(2, tx);
- repoOrder.verify(repo, times(1)).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
- repoOrder.verify(repo, times(2)).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
- repoOrder.verify(repo, times(2)).updateJob(any(MJob.class), any(RepositoryTransaction.class));
+ doReturn(connectionList).when(repoSpy).findConnections();
+ doReturn(jobList).when(repoSpy).findJobs();
+ doNothing().when(repoSpy).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
+
+ repoSpy.upgradeFramework(newFramework);
+
+ InOrder repoOrder = inOrder(repoSpy);
+ InOrder txOrder = inOrder(repoTransactionMock);
+ InOrder upgraderOrder = inOrder(upgraderMock);
+ InOrder validatorOrder = inOrder(validatorMock);
+
+ repoOrder.verify(repoSpy, times(1)).findConnections();
+ repoOrder.verify(repoSpy, times(1)).findJobs();
+ repoOrder.verify(repoSpy, times(1)).getTransaction();
+ repoOrder.verify(repoSpy, times(1)).deleteJobInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteJobInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(2)).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ repoOrder.verify(repoSpy, times(2)).updateJob(any(MJob.class), any(RepositoryTransaction.class));
repoOrder.verifyNoMoreInteractions();
- txOrder.verify(tx, times(1)).begin();
- txOrder.verify(tx, times(1)).commit();
- txOrder.verify(tx, times(1)).close();
+ txOrder.verify(repoTransactionMock, times(1)).begin();
+ txOrder.verify(repoTransactionMock, times(1)).commit();
+ txOrder.verify(repoTransactionMock, times(1)).close();
txOrder.verifyNoMoreInteractions();
- upgraderOrder.verify(upgrader, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
- upgraderOrder.verify(upgrader, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
+ upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
+ upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validator, times(2)).validateConnection(anyObject());
- validatorOrder.verify(validator, times(2)).validateJob(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateConnection(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
}
@@ -375,50 +375,50 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeWithInvalidConnectionsAndJobs() {
MFramework newFramework = framework();
- when(validator.validateConnection(any(MConnection.class))).thenReturn(invalid);
- when(validator.validateJob(any(MJob.class))).thenReturn(invalid);
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
- when(frameworkMgr.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(frameworkMgr.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(invalidRepoMock);
+ when(validatorMock.validateJob(any(MJob.class))).thenReturn(invalidRepoMock);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(frameworkMgrMock.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(frameworkMgrMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repo).findConnections();
- doReturn(jobList).when(repo).findJobs();
- doNothing().when(repo).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
- doNothing().when(repo).updateJob(any(MJob.class), any(RepositoryTransaction.class));
- doNothing().when(repo).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
+ doReturn(connectionList).when(repoSpy).findConnections();
+ doReturn(jobList).when(repoSpy).findJobs();
+ doNothing().when(repoSpy).updateConnection(any(MConnection.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
+ doNothing().when(repoSpy).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
try {
- repo.upgradeFramework(newFramework);
+ repoSpy.upgradeFramework(newFramework);
} catch (SqoopException ex) {
assertEquals(ex.getErrorCode(), RepositoryError.JDBCREPO_0027);
- InOrder repoOrder = inOrder(repo);
- InOrder txOrder = inOrder(tx);
- InOrder upgraderOrder = inOrder(upgrader);
- InOrder validatorOrder = inOrder(validator);
-
- repoOrder.verify(repo, times(1)).findConnections();
- repoOrder.verify(repo, times(1)).findJobs();
- repoOrder.verify(repo, times(1)).getTransaction();
- repoOrder.verify(repo, times(1)).deleteJobInputs(1, tx);
- repoOrder.verify(repo, times(1)).deleteJobInputs(2, tx);
- repoOrder.verify(repo, times(1)).deleteConnectionInputs(1, tx);
- repoOrder.verify(repo, times(1)).deleteConnectionInputs(2, tx);
- repoOrder.verify(repo, times(1)).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
+ InOrder repoOrder = inOrder(repoSpy);
+ InOrder txOrder = inOrder(repoTransactionMock);
+ InOrder upgraderOrder = inOrder(upgraderMock);
+ InOrder validatorOrder = inOrder(validatorMock);
+
+ repoOrder.verify(repoSpy, times(1)).findConnections();
+ repoOrder.verify(repoSpy, times(1)).findJobs();
+ repoOrder.verify(repoSpy, times(1)).getTransaction();
+ repoOrder.verify(repoSpy, times(1)).deleteJobInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteJobInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(1, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).deleteConnectionInputs(2, repoTransactionMock);
+ repoOrder.verify(repoSpy, times(1)).updateFramework(any(MFramework.class), any(RepositoryTransaction.class));
repoOrder.verifyNoMoreInteractions();
- txOrder.verify(tx, times(1)).begin();
- txOrder.verify(tx, times(1)).rollback();
- txOrder.verify(tx, times(1)).close();
+ txOrder.verify(repoTransactionMock, times(1)).begin();
+ txOrder.verify(repoTransactionMock, times(1)).rollback();
+ txOrder.verify(repoTransactionMock, times(1)).close();
txOrder.verifyNoMoreInteractions();
- upgraderOrder.verify(upgrader, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
- upgraderOrder.verify(upgrader, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
+ upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MConnectionForms.class), any(MConnectionForms.class));
+ upgraderOrder.verify(upgraderMock, times(2)).upgrade(any(MJobForms.class), any(MJobForms.class));
upgraderOrder.verifyNoMoreInteractions();
- validatorOrder.verify(validator, times(2)).validateConnection(anyObject());
- validatorOrder.verify(validator, times(2)).validateJob(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateConnection(anyObject());
+ validatorOrder.verify(validatorMock, times(2)).validateJob(anyObject());
validatorOrder.verifyNoMoreInteractions();
return ;
}
@@ -436,20 +436,20 @@ public class TestJdbcRepository {
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validator);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
- when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
+ when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"find connections for connector error.");
- doThrow(exception).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
try {
- repo.upgradeConnector(oldConnector, newConnector);
+ repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -466,24 +466,24 @@ public class TestJdbcRepository {
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validator);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
- when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
+ when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
- doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"find jobs for connector error.");
- doThrow(exception).when(repoHandler).findJobsForConnector(anyLong(), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
try {
- repo.upgradeConnector(oldConnector, newConnector);
+ repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -500,27 +500,27 @@ public class TestJdbcRepository {
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validator);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
- when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
+ when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobsForConnector(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"delete job inputs for connector error.");
- doThrow(exception).when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
try {
- repo.upgradeConnector(oldConnector, newConnector);
+ repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).deleteJobInputs(anyLong(), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).deleteJobInputs(anyLong(), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -537,29 +537,29 @@ public class TestJdbcRepository {
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validator);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
- when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
+ when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobsForConnector(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"delete connection inputs for connector error.");
- doThrow(exception).when(repoHandler).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
try {
- repo.upgradeConnector(oldConnector, newConnector);
+ repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -576,31 +576,31 @@ public class TestJdbcRepository {
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(sqconnector.getValidator()).thenReturn(validator);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
- when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
+ when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobsForConnector(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update connector error.");
- doThrow(exception).when(repoHandler).updateConnector(any(MConnector.class), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).updateConnector(any(MConnector.class), any(Connection.class));
try {
- repo.upgradeConnector(oldConnector, newConnector);
+ repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -617,39 +617,39 @@ public class TestJdbcRepository {
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validator.validateConnection(any(MConnection.class))).thenReturn(valid);
- when(validator.validateJob(any(MJob.class))).thenReturn(valid);
- when(sqconnector.getValidator()).thenReturn(validator);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
+ when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
- when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
+ when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobsForConnector(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteConnectionInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).updateConnector(any(MConnector.class), any(Connection.class));
- doReturn(true).when(repoHandler).existsConnection(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateConnector(any(MConnector.class), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsConnection(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update connection error.");
- doThrow(exception).when(repoHandler).updateConnection(any(MConnection.class), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).updateConnection(any(MConnection.class), any(Connection.class));
try {
- repo.upgradeConnector(oldConnector, newConnector);
+ repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
- verify(repoHandler, times(1)).existsConnection(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateConnection(any(MConnection.class), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
+ verify(repoHandlerMock, times(1)).existsConnection(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateConnection(any(MConnection.class), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -666,43 +666,43 @@ public class TestJdbcRepository {
MConnector oldConnector = connector(1);
SqoopConnector sqconnector = mock(SqoopConnector.class);
- when(validator.validateConnection(any(MConnection.class))).thenReturn(valid);
- when(validator.validateJob(any(MJob.class))).thenReturn(valid);
- when(sqconnector.getValidator()).thenReturn(validator);
- when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
+ when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(sqconnector.getValidator()).thenReturn(validatorMock);
+ when(sqconnector.getMetadataUpgrader()).thenReturn(upgraderMock);
when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(JobConfiguration.class);
- when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);
+ when(connectorMgrMock.getConnector(anyString())).thenReturn(sqconnector);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobsForConnector(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteConnectionInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).updateConnector(any(MConnector.class), any(Connection.class));
- doNothing().when(repoHandler).updateConnection(any(MConnection.class), any(Connection.class));
- doReturn(true).when(repoHandler).existsConnection(anyLong(), any(Connection.class));
- doReturn(true).when(repoHandler).existsJob(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnectionsForConnector(anyLong(), any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobsForConnector(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateConnector(any(MConnector.class), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateConnection(any(MConnection.class), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsConnection(anyLong(), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsJob(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update job error.");
- doThrow(exception).when(repoHandler).updateJob(any(MJob.class), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).updateJob(any(MJob.class), any(Connection.class));
try {
- repo.upgradeConnector(oldConnector, newConnector);
+ repoSpy.upgradeConnector(oldConnector, newConnector);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
- verify(repoHandler, times(2)).existsConnection(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).updateConnection(any(MConnection.class), any(Connection.class));
- verify(repoHandler, times(1)).existsJob(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateJob(any(MJob.class), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnectionsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobsForConnector(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateConnector(any(MConnector.class), any(Connection.class));
+ verify(repoHandlerMock, times(2)).existsConnection(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).updateConnection(any(MConnection.class), any(Connection.class));
+ verify(repoHandlerMock, times(1)).existsJob(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateJob(any(MJob.class), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -717,19 +717,19 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeHandlerFindConnectionsError() {
MFramework newFramework = framework();
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"find connections error.");
- doThrow(exception).when(repoHandler).findConnections(any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).findConnections(any(Connection.class));
try {
- repo.upgradeFramework(newFramework);
+ repoSpy.upgradeFramework(newFramework);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnections(any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -744,23 +744,23 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeHandlerFindJobsError() {
MFramework newFramework = framework();
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
- doReturn(connectionList).when(repoHandler).findConnections(any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"find jobs error.");
- doThrow(exception).when(repoHandler).findJobs(any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).findJobs(any(Connection.class));
try {
- repo.upgradeFramework(newFramework);
+ repoSpy.upgradeFramework(newFramework);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnections(any(Connection.class));
- verify(repoHandler, times(1)).findJobs(any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -775,26 +775,26 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeHandlerDeleteJobInputsError() {
MFramework newFramework = framework();
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnections(any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobs(any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"delete job inputs error.");
- doThrow(exception).when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
try {
- repo.upgradeFramework(newFramework);
+ repoSpy.upgradeFramework(newFramework);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnections(any(Connection.class));
- verify(repoHandler, times(1)).findJobs(any(Connection.class));
- verify(repoHandler, times(1)).deleteJobInputs(anyLong(), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
+ verify(repoHandlerMock, times(1)).deleteJobInputs(anyLong(), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -809,28 +809,28 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeHandlerDeleteConnectionInputsError() {
MFramework newFramework = framework();
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnections(any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobs(any(Connection.class));
- doNothing().when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"delete connection inputs error.");
- doThrow(exception).when(repoHandler).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
try {
- repo.upgradeFramework(newFramework);
+ repoSpy.upgradeFramework(newFramework);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnections(any(Connection.class));
- verify(repoHandler, times(1)).findJobs(any(Connection.class));
- verify(repoHandler, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -845,30 +845,30 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeHandlerUpdateFrameworkError() {
MFramework newFramework = framework();
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnections(any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobs(any(Connection.class));
- doNothing().when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update framework metadata error.");
- doThrow(exception).when(repoHandler).updateFramework(any(MFramework.class), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).updateFramework(any(MFramework.class), any(Connection.class));
try {
- repo.upgradeFramework(newFramework);
+ repoSpy.upgradeFramework(newFramework);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnections(any(Connection.class));
- verify(repoHandler, times(1)).findJobs(any(Connection.class));
- verify(repoHandler, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -883,38 +883,38 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeHandlerUpdateConnectionError() {
MFramework newFramework = framework();
- when(validator.validateConnection(any(MConnection.class))).thenReturn(valid);
- when(validator.validateJob(any(MJob.class))).thenReturn(valid);
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
- when(frameworkMgr.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(frameworkMgr.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(frameworkMgrMock.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(frameworkMgrMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnections(any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobs(any(Connection.class));
- doNothing().when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteConnectionInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).updateFramework(any(MFramework.class), any(Connection.class));
- doReturn(true).when(repoHandler).existsConnection(anyLong(), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateFramework(any(MFramework.class), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsConnection(anyLong(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update connection error.");
- doThrow(exception).when(repoHandler).updateConnection(any(MConnection.class), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).updateConnection(any(MConnection.class), any(Connection.class));
try {
- repo.upgradeFramework(newFramework);
+ repoSpy.upgradeFramework(newFramework);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnections(any(Connection.class));
- verify(repoHandler, times(1)).findJobs(any(Connection.class));
- verify(repoHandler, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
- verify(repoHandler, times(1)).existsConnection(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateConnection(any(MConnection.class), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
+ verify(repoHandlerMock, times(1)).existsConnection(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateConnection(any(MConnection.class), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
@@ -929,42 +929,42 @@ public class TestJdbcRepository {
public void testFrameworkUpgradeHandlerUpdateJobError() {
MFramework newFramework = framework();
- when(validator.validateConnection(any(MConnection.class))).thenReturn(valid);
- when(validator.validateJob(any(MJob.class))).thenReturn(valid);
- when(frameworkMgr.getValidator()).thenReturn(validator);
- when(frameworkMgr.getMetadataUpgrader()).thenReturn(upgrader);
- when(frameworkMgr.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
- when(frameworkMgr.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
+ when(validatorMock.validateConnection(any(MConnection.class))).thenReturn(validRepoMock);
+ when(validatorMock.validateJob(any(MJob.class))).thenReturn(validRepoMock);
+ when(frameworkMgrMock.getValidator()).thenReturn(validatorMock);
+ when(frameworkMgrMock.getMetadataUpgrader()).thenReturn(upgraderMock);
+ when(frameworkMgrMock.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
+ when(frameworkMgrMock.getJobConfigurationClass()).thenReturn(JobConfiguration.class);
List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
List<MJob> jobList = jobs(job(1,1,1,1,1), job(2,1,1,2,1));
- doReturn(connectionList).when(repoHandler).findConnections(any(Connection.class));
- doReturn(jobList).when(repoHandler).findJobs(any(Connection.class));
- doNothing().when(repoHandler).deleteJobInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).deleteConnectionInputs(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).updateFramework(any(MFramework.class), any(Connection.class));
- doReturn(true).when(repoHandler).existsConnection(anyLong(), any(Connection.class));
- doReturn(true).when(repoHandler).existsJob(anyLong(), any(Connection.class));
- doNothing().when(repoHandler).updateConnection(any(MConnection.class), any(Connection.class));
+ doReturn(connectionList).when(repoHandlerMock).findConnections(any(Connection.class));
+ doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteJobInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).deleteConnectionInputs(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateFramework(any(MFramework.class), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsConnection(anyLong(), any(Connection.class));
+ doReturn(true).when(repoHandlerMock).existsJob(anyLong(), any(Connection.class));
+ doNothing().when(repoHandlerMock).updateConnection(any(MConnection.class), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
"update job error.");
- doThrow(exception).when(repoHandler).updateJob(any(MJob.class), any(Connection.class));
+ doThrow(exception).when(repoHandlerMock).updateJob(any(MJob.class), any(Connection.class));
try {
- repo.upgradeFramework(newFramework);
+ repoSpy.upgradeFramework(newFramework);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
- verify(repoHandler, times(1)).findConnections(any(Connection.class));
- verify(repoHandler, times(1)).findJobs(any(Connection.class));
- verify(repoHandler, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
- verify(repoHandler, times(2)).existsConnection(anyLong(), any(Connection.class));
- verify(repoHandler, times(2)).updateConnection(any(MConnection.class), any(Connection.class));
- verify(repoHandler, times(1)).existsJob(anyLong(), any(Connection.class));
- verify(repoHandler, times(1)).updateJob(any(MJob.class), any(Connection.class));
- verifyNoMoreInteractions(repoHandler);
+ verify(repoHandlerMock, times(1)).findConnections(any(Connection.class));
+ verify(repoHandlerMock, times(1)).findJobs(any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteJobInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).deleteConnectionInputs(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateFramework(any(MFramework.class), any(Connection.class));
+ verify(repoHandlerMock, times(2)).existsConnection(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(2)).updateConnection(any(MConnection.class), any(Connection.class));
+ verify(repoHandlerMock, times(1)).existsJob(anyLong(), any(Connection.class));
+ verify(repoHandlerMock, times(1)).updateJob(any(MJob.class), any(Connection.class));
+ verifyNoMoreInteractions(repoHandlerMock);
return ;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java
new file mode 100644
index 0000000..2d53dd2
--- /dev/null
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRJobRequest.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.execution.mapreduce;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.sqoop.framework.JobRequest;
+
+/**
+ * Map-reduce specific submission request containing all extra information
+ * needed for bootstrapping map-reduce job.
+ */
+public class MRJobRequest extends JobRequest {
+
+ /**
+ * Map-reduce specific options.
+ */
+ Class<? extends InputFormat> inputFormatClass;
+ Class<? extends Mapper> mapperClass;
+ Class<? extends Writable> mapOutputKeyClass;
+ Class<? extends Writable> mapOutputValueClass;
+ Class<? extends OutputFormat> outputFormatClass;
+ Class<? extends Writable> outputKeyClass;
+ Class<? extends Writable> outputValueClass;
+
+ public MRJobRequest() {
+ super();
+ }
+
+ public Class<? extends InputFormat> getInputFormatClass() {
+ return inputFormatClass;
+ }
+
+ public void setInputFormatClass(Class<? extends InputFormat> inputFormatClass) {
+ this.inputFormatClass = inputFormatClass;
+ }
+
+ public Class<? extends Mapper> getMapperClass() {
+ return mapperClass;
+ }
+
+ public void setMapperClass(Class<? extends Mapper> mapperClass) {
+ this.mapperClass = mapperClass;
+ }
+
+ public Class<? extends Writable> getMapOutputKeyClass() {
+ return mapOutputKeyClass;
+ }
+
+ public void setMapOutputKeyClass(Class<? extends Writable> mapOutputKeyClass) {
+ this.mapOutputKeyClass = mapOutputKeyClass;
+ }
+
+ public Class<? extends Writable> getMapOutputValueClass() {
+ return mapOutputValueClass;
+ }
+
+ public void setMapOutputValueClass(Class<? extends Writable> mapOutputValueClass) {
+ this.mapOutputValueClass = mapOutputValueClass;
+ }
+
+ public Class<? extends OutputFormat> getOutputFormatClass() {
+ return outputFormatClass;
+ }
+
+ public void setOutputFormatClass(Class<? extends OutputFormat> outputFormatClass) {
+ this.outputFormatClass = outputFormatClass;
+ }
+
+ public Class<? extends Writable> getOutputKeyClass() {
+ return outputKeyClass;
+ }
+
+ public void setOutputKeyClass(Class<? extends Writable> outputKeyClass) {
+ this.outputKeyClass = outputKeyClass;
+ }
+
+ public Class<? extends Writable> getOutputValueClass() {
+ return outputValueClass;
+ }
+
+ public void setOutputValueClass(Class<? extends Writable> outputValueClass) {
+ this.outputValueClass = outputValueClass;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3d539dd4/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRSubmissionRequest.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRSubmissionRequest.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRSubmissionRequest.java
deleted file mode 100644
index 32d598c..0000000
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MRSubmissionRequest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.execution.mapreduce;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.sqoop.framework.SubmissionRequest;
-
-/**
- * Map-reduce specific submission request containing all extra information
- * needed for bootstrapping map-reduce job.
- */
-public class MRSubmissionRequest extends SubmissionRequest {
-
- /**
- * Map-reduce specific options.
- */
- Class<? extends InputFormat> inputFormatClass;
- Class<? extends Mapper> mapperClass;
- Class<? extends Writable> mapOutputKeyClass;
- Class<? extends Writable> mapOutputValueClass;
- Class<? extends OutputFormat> outputFormatClass;
- Class<? extends Writable> outputKeyClass;
- Class<? extends Writable> outputValueClass;
-
- public MRSubmissionRequest() {
- super();
- }
-
- public Class<? extends InputFormat> getInputFormatClass() {
- return inputFormatClass;
- }
-
- public void setInputFormatClass(Class<? extends InputFormat> inputFormatClass) {
- this.inputFormatClass = inputFormatClass;
- }
-
- public Class<? extends Mapper> getMapperClass() {
- return mapperClass;
- }
-
- public void setMapperClass(Class<? extends Mapper> mapperClass) {
- this.mapperClass = mapperClass;
- }
-
- public Class<? extends Writable> getMapOutputKeyClass() {
- return mapOutputKeyClass;
- }
-
- public void setMapOutputKeyClass(Class<? extends Writable> mapOutputKeyClass) {
- this.mapOutputKeyClass = mapOutputKeyClass;
- }
-
- public Class<? extends Writable> getMapOutputValueClass() {
- return mapOutputValueClass;
- }
-
- public void setMapOutputValueClass(Class<? extends Writable> mapOutputValueClass) {
- this.mapOutputValueClass = mapOutputValueClass;
- }
-
- public Class<? extends OutputFormat> getOutputFormatClass() {
- return outputFormatClass;
- }
-
- public void setOutputFormatClass(Class<? extends OutputFormat> outputFormatClass) {
- this.outputFormatClass = outputFormatClass;
- }
-
- public Class<? extends Writable> getOutputKeyClass() {
- return outputKeyClass;
- }
-
- public void setOutputKeyClass(Class<? extends Writable> outputKeyClass) {
- this.outputKeyClass = outputKeyClass;
- }
-
- public Class<? extends Writable> getOutputValueClass() {
- return outputValueClass;
- }
-
- public void setOutputValueClass(Class<? extends Writable> outputValueClass) {
- this.outputValueClass = outputValueClass;
- }
-}
[36/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
deleted file mode 100644
index a0a5dc2..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
+++ /dev/null
@@ -1,939 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import jline.ConsoleReader;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.lang.StringUtils;
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MEnumInput;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MIntegerInput;
-import org.apache.sqoop.model.MMapInput;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MNamedElement;
-import org.apache.sqoop.model.MStringInput;
-import org.apache.sqoop.model.MValidatedElement;
-import org.apache.sqoop.validation.Message;
-import org.apache.sqoop.validation.Status;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-
-/**
- * Convenient methods for retrieving user input and CLI options.
- */
-public final class FormFiller {
-
- /**
- * Internal input that will be reused for loading names for link and
- * job objects.
- */
- private static MStringInput nameInput = new MStringInput("object-name", false, (short)25);
-
- /**
- * Fill job object based on CLI options.
- *
- * @param line Associated console reader object
- * @param job Job that user is suppose to fill in
- * @return True if we filled all inputs, false if user has stopped processing
- * @throws IOException
- */
- public static boolean fillJob(CommandLine line,
- MJob job)
- throws IOException {
-
- job.setName(line.getOptionValue("name"));
-
- // Fill in data from user
- return fillForms(line,
- job.getConnectorPart(Direction.FROM).getForms(),
- job.getFrameworkPart().getForms());
- }
-
- /**
- * Fill job object based on user input.
- *
- * @param reader Associated console reader object
- * @param job Job that user is suppose to fill in
- * @param fromConnectorBundle Connector resource bundle
- * @param driverConfigBundle Driver config resource bundle
- * @return True if we filled all inputs, false if user has stopped processing
- * @throws IOException
- */
- public static boolean fillJob(ConsoleReader reader,
- MJob job,
- ResourceBundle fromConnectorBundle,
- ResourceBundle driverConfigBundle,
- ResourceBundle toConnectorBundle)
- throws IOException {
-
- job.setName(getName(reader, job.getName()));
-
- // Fill in data from user
- return fillForms(reader,
- job.getConnectorPart(Direction.FROM).getForms(),
- fromConnectorBundle,
- job.getFrameworkPart().getForms(),
- driverConfigBundle,
- job.getConnectorPart(Direction.TO).getForms(),
- toConnectorBundle);
- }
-
- /**
- * Fill link object based on CLI options.
- *
- * @param line Associated command line options
- * @param link Link that user is suppose to fill in
- * @return True if we filled all inputs, false if user has stopped processing
- * @throws IOException
- */
- public static boolean fillConnection(CommandLine line,
- MLink link)
- throws IOException {
-
- link.setName(line.getOptionValue("name"));
-
- // Fill in data from user
- return fillForms(line,
- link.getConnectorPart().getForms(),
- link.getFrameworkPart().getForms());
- }
-
- /**
- * Fill link object based on user input.
- *
- * @param reader Associated console reader object
- * @param link Link that user is suppose to fill in
- * @param connectorConfigBundle Connector resource bundle
- * @param driverConfigBundle Driver config resource bundle
- * @return True if we filled all inputs, false if user has stopped processing
- * @throws IOException
- */
- public static boolean fillLink(ConsoleReader reader,
- MLink link,
- ResourceBundle connectorConfigBundle,
- ResourceBundle driverConfigBundle)
- throws IOException {
-
- link.setName(getName(reader, link.getName()));
-
- // Fill in data from user
- return fillForms(reader,
- link.getConnectorPart().getForms(),
- connectorConfigBundle,
- link.getFrameworkPart().getForms(),
- driverConfigBundle);
- }
-
- /**
- * Load CLI options for framework forms and connector forms.
- *
- * @param line CLI options container
- * @param connectorForms Connector forms to read or edit
- * @param frameworkForms Framework forms to read or edit
- * @return
- * @throws IOException
- */
- public static boolean fillForms(CommandLine line,
- List<MForm> connectorForms,
- List<MForm> frameworkForms)
- throws IOException {
- // Query connector forms and framework forms
- return fillForms("connector", connectorForms, line)
- && fillForms("framework", frameworkForms, line);
- }
-
- /**
- * Load all CLI options for a list of forms.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param forms Forms to read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- public static boolean fillForms(String prefix,
- List<MForm> forms,
- CommandLine line)
- throws IOException {
- for (MForm form : forms) {
- if (!fillForm(prefix, form, line)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Load all CLI options for a particular form.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param form Form to read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- @SuppressWarnings("rawtypes")
- public static boolean fillForm(String prefix,
- MForm form,
- CommandLine line) throws IOException {
- for (MInput input : form.getInputs()) {
- if (!fillInput(prefix, input, line)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Load CLI option.
- * Chooses the appropriate 'fill' method to use based on input type.
- *
- * Keys for CLI options are automatically created from the 'prefix' argument
- * and 'input' argument: <prefix>-<form name>-<input name>
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- @SuppressWarnings("rawtypes")
- public static boolean fillInput(String prefix,
- MInput input,
- CommandLine line) throws IOException {
- // Based on the input type, let's perform specific load
- switch (input.getType()) {
- case STRING:
- return fillInputString(prefix, (MStringInput) input, line);
- case INTEGER:
- return fillInputInteger(prefix, (MIntegerInput) input, line);
- case BOOLEAN:
- return fillInputBoolean(prefix, (MBooleanInput) input, line);
- case MAP:
- return fillInputMap(prefix, (MMapInput) input, line);
- case ENUM:
- return fillInputEnum(prefix, (MEnumInput) input, line);
- default:
- println("Unsupported data type " + input.getType());
- return true;
- }
- }
-
- /**
- * Load CLI option for enum type.
- *
- * Currently only supports numeric values.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- private static boolean fillInputEnum(String prefix,
- MEnumInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- String value = line.getOptionValue(opt);
- int index = java.util.Arrays.asList(input.getValues()).indexOf(value);
-
- if(index < 0) {
- errorMessage(input, String.format("Invalid option %s. Please use one of %s.", value, StringUtils.join(input.getValues(), ", ")));
- return false;
- }
-
- input.setValue(value);
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- /**
- * Load CLI options for map type.
- *
- * Parses Key-Value pairs that take the form "<key>=<value>&<key>=<value>&...".
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- private static boolean fillInputMap(String prefix,
- MMapInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- String value = line.getOptionValue(opt);
- Map<String, String> values = new HashMap<String, String>();
- String[] entries = value.split("&");
- for (String entry : entries) {
- if (entry.contains("=")) {
- String[] keyValue = entry.split("=");
- values.put(keyValue[0], keyValue[1]);
- } else {
- errorMessage(input, "Don't know what to do with " + entry);
- return false;
- }
- }
- input.setValue(values);
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- /**
- * Load integer input from CLI option.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- private static boolean fillInputInteger(String prefix,
- MIntegerInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- try {
- input.setValue(Integer.valueOf(line.getOptionValue(FormOptions.getOptionKey(prefix, input))));
- } catch (NumberFormatException ex) {
- errorMessage(input, "Input is not valid integer number");
- return false;
- }
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- /**
- * Load string input from CLI option.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- public static boolean fillInputString(String prefix,
- MStringInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- String value = line.getOptionValue(FormOptions.getOptionKey(prefix, input));
- if(value.length() > input.getMaxLength()) {
- errorMessage(input, "Size of input exceeds allowance for this input"
- + " field. Maximal allowed size is " + input.getMaxLength());
- }
- input.setValue(value);
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- /**
- * Load boolean input from CLI option.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- public static boolean fillInputBoolean(String prefix,
- MBooleanInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- input.setValue(Boolean.valueOf(line.getOptionValue(FormOptions.getOptionKey(prefix, input))));
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- public static boolean fillForms(ConsoleReader reader,
- List<MForm> connectorForms,
- ResourceBundle connectorConfigBundle,
- List<MForm> frameworkForms,
- ResourceBundle driverConfigBundle) throws IOException {
-
-
- // Query connector forms
- if(!fillForms(connectorForms, reader, connectorConfigBundle)) {
- return false;
- }
-
- // Query framework forms
- if(!fillForms(frameworkForms, reader, driverConfigBundle)) {
- return false;
- }
- return true;
- }
-
- public static boolean fillForms(ConsoleReader reader,
- List<MForm> fromConnectorForms,
- ResourceBundle fromConnectorBundle,
- List<MForm> frameworkForms,
- ResourceBundle driverConfigBundle,
- List<MForm> toConnectorForms,
- ResourceBundle toConnectorBundle) throws IOException {
-
-
- // From connector forms
- if(!fillForms(fromConnectorForms, reader, fromConnectorBundle)) {
- return false;
- }
-
- // Query framework forms
- if(!fillForms(frameworkForms, reader, driverConfigBundle)) {
- return false;
- }
-
- // To connector forms
- if(!fillForms(toConnectorForms, reader, toConnectorBundle)) {
- return false;
- }
-
- return true;
- }
-
- public static boolean fillForms(List<MForm> forms,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- for (MForm form : forms) {
- if(!fillForm(form, reader, bundle)) {
- return false;
- }
- }
-
- return true;
- }
-
- @SuppressWarnings("rawtypes")
- public static boolean fillForm(MForm form,
- ConsoleReader reader,
- ResourceBundle bundle) throws IOException {
- println("");
- println(bundle.getString(form.getLabelKey()));
-
- // Print out form validation
- printValidationMessage(form, false);
- println("");
-
- for (MInput input : form.getInputs()) {
- if(!fillInput(input, reader, bundle)) {
- return false;
- }
- }
-
- return true;
- }
-
- @SuppressWarnings("rawtypes")
- public static boolean fillInput(MInput input,
- ConsoleReader reader,
- ResourceBundle bundle) throws IOException {
- // Print out validation
- printValidationMessage(input, false);
-
- // Based on the input type, let's perform specific load
- switch (input.getType()) {
- case STRING:
- return fillInputString((MStringInput) input, reader, bundle);
- case INTEGER:
- return fillInputInteger((MIntegerInput) input, reader, bundle);
- case BOOLEAN:
- return fillInputBoolean((MBooleanInput) input, reader, bundle);
- case MAP:
- return fillInputMap((MMapInput) input, reader, bundle);
- case ENUM:
- return fillInputEnum((MEnumInput) input, reader, bundle);
- default:
- println("Unsupported data type " + input.getType());
- return true;
- }
- }
-
- /**
- * Load user input for enum type.
- *
- * Print out numbered list of all available options and let user choose one
- * item from that.
- *
- * @param input Input that we should read or edit
- * @param reader Associated console reader
- * @param bundle Resource bundle
- * @return True if user with to continue with loading addtional inputs
- * @throws IOException
- */
- private static boolean fillInputEnum(MEnumInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- // Prompt in enum case
- println(bundle.getString(input.getLabelKey()) + ": ");
-
- // Indexes
- int i = -1;
- int lastChoice = -1;
-
- // Print out all values as a numbered list
- for(String value : input.getValues()) {
- i++;
-
- println(" " + i + " : " + value);
-
- // Only show last choice if not sensitive
- if(!input.isEmpty() && value.equals(input.getValue()) && !input.isSensitive()) {
- lastChoice = i;
- }
- }
-
- // Prompt
- reader.printString("Choose: ");
-
- // Fill previously filled index when available
- if(lastChoice != -1) {
- reader.putString(Integer.toString(lastChoice));
- }
-
- reader.flushConsole();
- String userTyped;
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if (userTyped == null) {
- return false;
- } else if (userTyped.isEmpty()) {
- input.setEmpty();
- } else {
- Integer index;
- try {
- index = Integer.valueOf(userTyped);
-
- if(index < 0 || index >= input.getValues().length) {
- errorMessage("Invalid index");
- return fillInputEnum(input, reader, bundle);
- }
-
- input.setValue(input.getValues()[index]);
- } catch (NumberFormatException ex) {
- errorMessage("Input is not valid integer number");
- return fillInputEnum(input, reader, bundle);
- }
- }
-
- return true;
- }
-
- /**
- * Load user input for map type.
- *
- * This implementation will load one map entry at the time. Current flows is
- * as follows: if user did not enter anything (empty input) finish loading
- * and return from function. If user specified input with equal sign (=),
- * lets add new key value pair. Otherwise consider entire input as a key name
- * and try to remove it from the map.
- *
- * Please note that following code do not supports equal sign in property
- * name. It's however perfectly fine to have equal sign in value.
- *
- * @param input Input that we should read or edit
- * @param reader Associated console reader
- * @param bundle Resource bundle
- * @return True if user wish to continue with loading additional inputs
- * @throws IOException
- */
- private static boolean fillInputMap(MMapInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- // Special prompt in Map case
- println(bundle.getString(input.getLabelKey()) + ": ");
-
- // Internal loading map
- Map<String, String> values = input.getValue();
- if(values == null) {
- values = new HashMap<String, String>();
- }
-
- String userTyped;
-
- while(true) {
- // Print all current items in each iteration
- // However do not printout if this input contains sensitive information.
- println("There are currently " + values.size() + " values in the map:");
- if (!input.isSensitive()) {
- for(Map.Entry<String, String> entry : values.entrySet()) {
- println(entry.getKey() + " = " + entry.getValue());
- }
- }
-
- // Special prompt for Map entry
- reader.printString("entry# ");
- reader.flushConsole();
-
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if(userTyped == null) {
- // Finish loading and return back to Sqoop shell
- return false;
- } else if(userTyped.isEmpty()) {
- // User has finished loading data to Map input, either set input empty
- // if there are no entries or propagate entries to the input
- if(values.size() == 0) {
- input.setEmpty();
- } else {
- input.setValue(values);
- }
- return true;
- } else {
- // User has specified regular input, let's check if it contains equals
- // sign. Save new entry (or update existing one) if it does. Otherwise
- // try to remove entry that user specified.
- if(userTyped.contains("=")) {
- String []keyValue = userTyped.split("=", 2);
- values.put(handleUserInput(keyValue[0]), handleUserInput(keyValue[1]));
- } else {
- String key = handleUserInput(userTyped);
- if(values.containsKey(key)) {
- values.remove(key);
- } else {
- errorMessage("Don't know what to do with " + userTyped);
- }
- }
- }
-
- }
- }
-
- /**
- * Handle special cases in user input.
- *
- * Preserve null and empty values, remove whitespace characters before and
- * after loaded string and de-quote the string if it's quoted (to preserve
- * spaces for example).
- *
- * @param input String loaded from user
- * @return Unquoted transformed string
- */
- private static String handleUserInput(String input) {
- // Preserve null and empty values
- if(input == null) {
- return null;
- }
- if(input.isEmpty()) {
- return input;
- }
-
- // Removes empty characters at the begging and end of loaded string
- input = input.trim();
-
- int lastIndex = input.length() - 1;
- char first = input.charAt(0);
- char last = input.charAt(lastIndex);
-
- // Remove quoting if present
- if(first == '\'' && last == '\'') {
- input = input.substring(1, lastIndex);
- } else if(first == '"' && last == '"') {
- input = input.substring(1, lastIndex);
- }
-
- // Return final string
- return input;
- }
-
- private static boolean fillInputInteger(MIntegerInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- generatePrompt(reader, bundle, input);
-
- // Fill already filled data when available
- // However do not printout if this input contains sensitive information.
- if(!input.isEmpty() && !input.isSensitive()) {
- reader.putString(input.getValue().toString());
- }
-
- // Get the data
- String userTyped;
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if (userTyped == null) {
- return false;
- } else if (userTyped.isEmpty()) {
- input.setEmpty();
- } else {
- Integer value;
- try {
- value = Integer.valueOf(userTyped);
- input.setValue(value);
- } catch (NumberFormatException ex) {
- errorMessage("Input is not valid integer number");
- return fillInputInteger(input, reader, bundle);
- }
-
- input.setValue(Integer.valueOf(userTyped));
- }
-
- return true;
- }
-
- /**
- * Load string input from the user.
- *
- * @param input Input that we should load in
- * @param reader Associated console reader
- * @param bundle Resource bundle for this input
- * @return
- * @throws IOException
- */
- public static boolean fillInputString(MStringInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- generatePrompt(reader, bundle, input);
-
- // Fill already filled data when available
- // However do not printout if this input contains sensitive information.
- if(!input.isEmpty() && !input.isSensitive()) {
- reader.putString(input.getValue());
- }
-
- // Get the data
- String userTyped;
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if (userTyped == null) {
- // Propagate end of loading process
- return false;
- } else if (userTyped.isEmpty()) {
- // Empty input in case that nothing was given
- input.setEmpty();
- } else {
- // Set value that user has entered
- input.setValue(userTyped);
-
- // Check that it did not exceeds maximal allowance for given input
- if(userTyped.length() > input.getMaxLength()) {
- errorMessage("Size of input exceeds allowance for this input"
- + " field. Maximal allowed size is " + input.getMaxLength());
- return fillInputString(input, reader, bundle);
- }
- }
-
- return true;
- }
-
- /**
- * Load boolean input from the user.
- *
- * @param input Input that we should load in
- * @param reader Associated console reader
- * @param bundle Resource bundle for this input
- * @return
- * @throws IOException
- */
- public static boolean fillInputBoolean(MBooleanInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- generatePrompt(reader, bundle, input);
-
- // Fill already filled data when available
- // However do not printout if this input contains sensitive information.
- if(!input.isEmpty() && !input.isSensitive()) {
- reader.putString(input.getValue().toString());
- }
-
- // Get the data
- String userTyped;
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if (userTyped == null) {
- // Propagate end of loading process
- return false;
- } else if (userTyped.isEmpty()) {
- // Empty input in case that nothing was given
- input.setEmpty();
- } else {
- // Set value that user has entered
- input.setValue(Boolean.valueOf(userTyped));
- }
-
- return true;
- }
-
- @SuppressWarnings("rawtypes")
- public static void generatePrompt(ConsoleReader reader,
- ResourceBundle bundle,
- MInput input)
- throws IOException {
- reader.printString(bundle.getString(input.getLabelKey()) + ": ");
- reader.flushConsole();
- }
-
- public static String getName(ConsoleReader reader,
- String name) throws IOException {
- if(name == null) {
- nameInput.setEmpty();
- } else {
- nameInput.setValue(name);
- }
-
- fillInputString(nameInput, reader, getResourceBundle());
-
- return nameInput.getValue();
- }
-
- /**
- * Print validation message in cases that it's not in state "FINE"
- *
- * @param element Validated element
- */
- public static void printValidationMessage(MValidatedElement element, boolean includeInputPrefix) {
- if(element.getValidationStatus() == Status.getDefault()) {
- return;
- }
-
- for(Message message : element.getValidationMessages())
- switch (message.getStatus()) {
- case UNACCEPTABLE:
- if (includeInputPrefix) {
- errorMessage(element, message.getMessage());
- } else {
- errorMessage(message.getMessage());
- }
- break;
- case ACCEPTABLE:
- if (includeInputPrefix) {
- warningMessage(element, message.getMessage());
- } else {
- warningMessage(message.getMessage());
- }
- break;
- default:
- // Simply ignore all other states for the moment
- break;
- }
- }
-
- public static void errorMessage(String message) {
- println("Error message: @|red " + message + " |@");
- }
-
- public static void errorMessage(MNamedElement input, String message) {
- print(input.getName());
- print(": ");
- errorMessage(message);
- }
-
- public static void warningMessage(String message) {
- println("Warning message: @|yellow " + message + " |@");
- }
-
- public static void warningMessage(MNamedElement input, String message) {
- print(input.getName());
- print(": ");
- warningMessage(message);
- }
-
- public static void errorIntroduction() {
- println();
- println("@|red There are issues with entered data, please revise your input:|@");
- }
-
- public static void printLinkValidationMessages(MLink link) {
- for (MForm form : link.getConnectorPart().getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- for (MForm form : link.getFrameworkPart().getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- }
-
- public static void printJobValidationMessages(MJob job) {
- for (MForm form : job.getConnectorPart(Direction.FROM).getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- for (MForm form : job.getFrameworkPart().getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- for (MForm form : job.getConnectorPart(Direction.TO).getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- }
-
- private FormFiller() {
- // Do not instantiate
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java
deleted file mode 100644
index efd002e..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MInputType;
-import org.apache.sqoop.shell.core.ShellError;
-
-/**
- * Utilities for automatically creating org.apache.commons.cli.Option objects.
- */
-public class FormOptions {
- /**
- * This method is used to automatically generate keys
- * for a particular input.
- *
- * @param prefix Prefix to prepend to CLI option keys
- * @param input
- * @return
- */
- @SuppressWarnings("rawtypes")
- public static String getOptionKey(String prefix, MInput input) {
- return prefix + "-" + input.getName().replace('.', '-');
- }
-
- /**
- * This method is used to automatically generate CLI options
- * for a list of forms.
- *
- * @param prefix Prefix to prepend to CLI option keys
- * @param forms Forms to get options for
- * @return
- */
- public static List<Option> getFormsOptions(String prefix, List<MForm> forms) {
- List<Option> options = new LinkedList<Option>();
- for (MForm form : forms) {
- List<Option> formOptions = getFormOptions(prefix, form);
- options.addAll(formOptions);
- }
- return options;
- }
-
- /**
- * This method is used to automatically generate CLI options
- * for a particular form.
- *
- * @param prefix Prefix to prepend to CLI option keys
- * @param form Form to get options for
- * @return List<Option>
- */
- @SuppressWarnings({ "rawtypes", "static-access" })
- public static List<Option> getFormOptions(String prefix, MForm form) {
- List<Option> options = new LinkedList<Option>();
- for (MInput input : form.getInputs()) {
- if (input.getType().equals(MInputType.BOOLEAN)) {
- options.add(OptionBuilder
- .withLongOpt(getOptionKey(prefix, input))
- .create());
- } else {
- options.add(OptionBuilder
- .withLongOpt(getOptionKey(prefix, input))
- .hasArg()
- .create());
- }
- }
- return options;
- }
-
- /**
- * Parses command line options.
- *
- * @param options parse arglist against these.
- * @param start beginning index in arglist.
- * @param arglist arguments to parse.
- * @param stopAtNonOption stop parsing when nonoption found in arglist.
- * @return CommandLine object
- */
- public static CommandLine parseOptions(Options options, int start, List<String> arglist, boolean stopAtNonOption) {
- String[] args = arglist.subList(start, arglist.size()).toArray(new String[arglist.size() - start]);
-
- CommandLineParser parser = new GnuParser();
- CommandLine line;
- try {
- line = parser.parse(options, args, stopAtNonOption);
- } catch (ParseException e) {
- throw new SqoopException(ShellError.SHELL_0003, e.getMessage(), e);
- }
- return line;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java
new file mode 100644
index 0000000..49b09c7
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.model.MJob;
+
+/**
+ * Automatically create dynamic options for jobs.
+ */
+@SuppressWarnings("serial")
+public class JobDynamicConfigOptions extends DynamicConfigOptions<MJob> {
+
+ @SuppressWarnings("static-access")
+ @Override
+ public void prepareOptions(MJob job) {
+ this.addOption(OptionBuilder
+ .withLongOpt("name")
+ .hasArg()
+ .create());
+ for (Option option : ConfigOptions.getConfigsOptions("from", job.getJobConfig(Direction.FROM).getConfigs())) {
+ this.addOption(option);
+ }
+ for (Option option : ConfigOptions.getConfigsOptions("driver", job.getDriverConfig().getConfigs())) {
+ this.addOption(option);
+ }
+ for (Option option : ConfigOptions.getConfigsOptions("to", job.getJobConfig(Direction.TO).getConfigs())) {
+ this.addOption(option);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java
deleted file mode 100644
index 87c0776..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MJob;
-
-/**
- * Automatically create dynamic options for jobs.
- */
-@SuppressWarnings("serial")
-public class JobDynamicFormOptions extends DynamicFormOptions<MJob> {
-
- @SuppressWarnings("static-access")
- @Override
- public void prepareOptions(MJob job) {
- this.addOption(OptionBuilder
- .withLongOpt("name")
- .hasArg()
- .create());
- for (Option option : FormOptions.getFormsOptions("connector", job.getConnectorPart(Direction.FROM).getForms())) {
- this.addOption(option);
- }
- for (Option option : FormOptions.getFormsOptions("framework", job.getFrameworkPart().getForms())) {
- this.addOption(option);
- }
- for (Option option : FormOptions.getFormsOptions("connector", job.getConnectorPart(Direction.TO).getForms())) {
- this.addOption(option);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java
new file mode 100644
index 0000000..ef279f7
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.model.MLink;
+
+/**
+ * Automatically create dynamic options for connections.
+ */
+@SuppressWarnings("serial")
+public class LinkDynamicConfigOptions extends DynamicConfigOptions<MLink> {
+
+ @SuppressWarnings("static-access")
+ @Override
+ public void prepareOptions(MLink link) {
+ this.addOption(OptionBuilder.withLongOpt("name").hasArg().create());
+ for (Option option : ConfigOptions.getConfigsOptions("link", link.getConnectorLinkConfig()
+ .getConfigs())) {
+ this.addOption(option);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
deleted file mode 100644
index 2952be2..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.model.MLink;
-
-/**
- * Automatically create dynamic options for connections.
- */
-@SuppressWarnings("serial")
-public class LinkDynamicFormOptions extends DynamicFormOptions<MLink> {
-
- @SuppressWarnings("static-access")
- @Override
- public void prepareOptions(MLink link) {
- this.addOption(OptionBuilder.withLongOpt("name").hasArg().create());
- for (Option option : FormOptions.getFormsOptions("connector", link.getConnectorPart()
- .getForms())) {
- this.addOption(option);
- }
- for (Option option : FormOptions.getFormsOptions("framework", link.getFrameworkPart()
- .getForms())) {
- this.addOption(option);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/resources/shell-resource.properties
----------------------------------------------------------------------
diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties
index c0f86f7..e2381dd 100644
--- a/shell/src/main/resources/shell-resource.properties
+++ b/shell/src/main/resources/shell-resource.properties
@@ -16,7 +16,7 @@
# Client Resources in default language (english)
############################
-# Security Form
+# Security Config
#############################
object-name.label = Name
object-name.help = Non unique name of the entity to help you remember \
@@ -200,22 +200,22 @@ table.header.status = Status
table.header.date = Last Update Date
table.header.enabled = Enabled
-#Form displayer resources
-formdisplayer.link = link
-formdisplayer.job = Job
-formdisplayer.form = form
-formdisplayer.name = Name
-formdisplayer.label = Label
-formdisplayer.help = Help
-formdisplayer.input = Input
-formdisplayer.type = Type
-formdisplayer.sensitive = Sensitive
-formdisplayer.size = Size
-formdisplayer.possible_values = Possible values
-formdisplayer.unsupported_datatype = Unsupported data type
-formdisplayer.input_sensitive = This input is sensitive
-
-formdisplayer.warning_message = There were warnings while create or update, but saved successfully.
+#Config displayer resources
+config.displayer.link = link
+config.displayer.job = Job
+config.displayer.config = config
+config.displayer.name = Name
+config.displayer.label = Label
+config.displayer.help = Help
+config.displayer.input = Input
+config.displayer.type = Type
+config.displayer.sensitive = Sensitive
+config.displayer.size = Size
+config.displayer.possible_values = Possible values
+config.displayer.unsupported_datatype = Unsupported data type
+config.displayer.input_sensitive = This input is sensitive
+
+config.displayer.warning_message = There were warnings while create or update, but saved successfully.
submission.submission_detail = Submission details
submission.job_id = Job ID
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java b/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
index e00b404..879e428 100644
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
@@ -18,8 +18,8 @@
*/
package org.apache.sqoop.connector.spi;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MLinkConfig;
/**
* Repository represents the sqoop entity store. Sqoop entities include
@@ -29,23 +29,23 @@ import org.apache.sqoop.model.MJobForms;
public abstract class RepositoryUpgrader {
/**
- * Upgrade the original connection and fill into the upgradeTarget. Note
- * that any metadata already in {@code upgradeTarget} maybe overwritten.
- * @param original - original connection metadata as in the repository
+ * Upgrade the original link config and fill into the upgradeTarget. Note
+ * that any data already in {@code upgradeTarget} maybe overwritten.
+ * @param original - original link config as in the repository
* @param upgradeTarget - the instance that will be filled in with the
- * upgraded metadata.
+ * upgraded link config.
*/
- public abstract void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget);
+ public abstract void upgrade(MLinkConfig original, MLinkConfig upgradeTarget);
/**
- * Upgrade the original job and fill into the upgradeTarget. Note
- * that any metadata already in {@code upgradeTarget} maybe overwritten.
- * This method must be called only after the connection metadata has
+ * Upgrade the original job config and fill into the upgradeTarget. Note
+ * that any config data already in {@code upgradeTarget} maybe overwritten.
+ * This method must be called only after the link config has
* already been upgraded.
- * @param original - original connection metadata as in the repository
+ * @param original - original job config as in the repository
* @param upgradeTarget - the instance that will be filled in with the
- * upgraded metadata.
+ * upgraded job config.
+ * NOTE(VB): This api will be revisited to accomodate from and to job config update
*/
- public abstract void upgrade(MJobForms original, MJobForms upgradeTarget);
+ public abstract void upgrade(MConfigList original, MConfigList upgradeTarget);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
index 7b78ba0..849f091 100644
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
@@ -58,12 +58,12 @@ public abstract class SqoopConnector {
}
/**
- * @return Get connection configuration class
+ * @return Get link configuration group class
*/
public abstract Class getLinkConfigurationClass();
/**
- * @return Get job configuration class for given type or null if not supported
+ * @return Get job configuration group class per direction type or null if not supported
*/
public abstract Class getJobConfigurationClass(Direction jobType);
@@ -79,12 +79,10 @@ public abstract class SqoopConnector {
/**
* Returns validation object that Sqoop can use to validate user
- * supplied forms before accepting them. This object will be used both for
- * connection and job forms.
- *
+ * supplied configs before accepting them. This object will be used both link and job configs
* @return Validator object
*/
- public abstract Validator getValidator();
+ public abstract Validator getConfigValidator();
/**
* Returns an {@linkplain RepositoryUpgrader} object that can upgrade the
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
index f965bdd..a133106 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
@@ -28,7 +28,7 @@ public abstract class Destroyer<LinkConfiguration, JobConfiguration> {
*
* @param context Destroyer context
* @param linkConfiguration link configuration object
- * @param jobConfiguration Job configuration object
+ * @param jobConfiguration job configuration object
*/
public abstract void destroy(DestroyerContext context,
LinkConfiguration linkConfiguration,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java b/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
index 34f02c9..d6c186d 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
@@ -27,8 +27,8 @@ public abstract class Extractor<LinkConfiguration, JobConfiguration, Partition>
* Extract data from source and pass them into the Sqoop.
*
* @param context Extractor context object
- * @param linkConfiguration link configuration
- * @param jobConfiguration Job configuration
+ * @param linkConfiguration link configuration object
+ * @param jobConfiguration job configuration object
* @param partition Partition that this extract should work on
*/
public abstract void extract(ExtractorContext context,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
index c9ae540..5c48fc3 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
@@ -34,28 +34,28 @@ public abstract class Initializer<LinkConfiguration, JobConfiguration> {
* promoted to all other part of the workflow automatically.
*
* @param context Initializer context object
- * @param linkConfiguration Connector's link configuration object
- * @param jobConfiguration Connector's job configuration object
+ * @param linkConfiguration link configuration object
+ * @param jobConfiguration job configuration object
*/
- public abstract void initialize(InitializerContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration);
+ public abstract void initialize(InitializerContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration);
- /**
- * Return list of all jars that this particular connector needs to operate
- * on following job. This method will be called after running initialize
- * method.
- *
- * @return
- */
- public List<String> getJars(InitializerContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration) {
- return new LinkedList<String>();
- }
+ /**
+ * Return list of all jars that this particular connector needs to operate
+ * on following job. This method will be called after running initialize
+ * method.
+ *
+ * @return
+ */
+ public List<String> getJars(InitializerContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration) {
+ return new LinkedList<String>();
+ }
- public abstract Schema getSchema(InitializerContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration);
+ public abstract Schema getSchema(InitializerContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java b/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
index 434dcf6..cc32ada 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
@@ -26,12 +26,12 @@ public abstract class Loader<LinkConfiguration, JobConfiguration> {
* Load data to target.
*
* @param context Loader context object
- * @param linkConfiguration link configuration
- * @param jobConfiguration Job configuration
+ * @param linkConfiguration link configuration object
+ * @param jobConfiguration job configuration object
* @throws Exception
*/
- public abstract void load(LoaderContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration) throws Exception;
+ public abstract void load(LoaderContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java b/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
index 8156762..57507df 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
@@ -31,12 +31,12 @@ public abstract class Partitioner<LinkConfiguration, JobConfiguration> {
* Each partition will be then processed in separate extractor.
*
* @param context Partitioner context object
- * @param linkConfiguration link configuration
- * @param jobConfiguration Job configuration
+ * @param linkConfiguration link configuration object
+ * @param jobConfiguration job configuration object
* @return
*/
- public abstract List<Partition> getPartitions(PartitionerContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration);
+ public abstract List<Partition> getPartitions(PartitionerContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/validation/Validator.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/validation/Validator.java b/spi/src/main/java/org/apache/sqoop/validation/Validator.java
index 894f412..2909905 100644
--- a/spi/src/main/java/org/apache/sqoop/validation/Validator.java
+++ b/spi/src/main/java/org/apache/sqoop/validation/Validator.java
@@ -29,21 +29,21 @@ public class Validator {
/**
* Validate link configuration object.
*
- * @param linkConfiguration Connection object to be validated
+ * @param linkConfiguration link config object to be validated
* @return Validation status
*/
- public Validation validateLink(Object linkConfiguration) {
- return new Validation(EmptyClass.class);
+ public ConfigValidator validateConfigForLink(Object linkConfiguration) {
+ return new ConfigValidator(EmptyClass.class);
}
/**
* Validate configuration object for job .
*
- * @param jobConfiguration Job to be validated
+ * @param jobConfiguration Job config to be validated
* @return Validation status
*/
- public Validation validateJob(Object jobConfiguration) {
- return new Validation(EmptyClass.class);
+ public ConfigValidator validateConfigForJob(Object jobConfiguration) {
+ return new ConfigValidator(EmptyClass.class);
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
index 25255ae..0c492ef 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
@@ -141,7 +141,7 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
* {@inheritDoc}
*/
@Override
- public boolean isExecutionEngineSupported(Class executionEngineClass) {
+ public boolean isExecutionEngineSupported(Class<?> executionEngineClass) {
return executionEngineClass == MapreduceExecutionEngine.class;
}
@@ -201,16 +201,16 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
try {
Job job = new Job(configuration);
- // And finally put all configuration objects to credentials cache
- ConfigurationUtils.setConnectorConnectionConfig(Direction.FROM, job, request.getConnectorLinkConfig(Direction.FROM));
- ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getConnectorJobConfig(Direction.FROM));
- ConfigurationUtils.setConnectorConnectionConfig(Direction.TO, job, request.getConnectorLinkConfig(Direction.TO));
- ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getConnectorJobConfig(Direction.TO));
+ // link configs
+ ConfigurationUtils.setConnectorLinkConfig(Direction.FROM, job, request.getConnectorLinkConfig(Direction.FROM));
+ ConfigurationUtils.setConnectorLinkConfig(Direction.TO, job, request.getConnectorLinkConfig(Direction.TO));
- ConfigurationUtils.setFrameworkConnectionConfig(Direction.FROM, job, request.getFrameworkLinkConfig(Direction.FROM));
- ConfigurationUtils.setFrameworkConnectionConfig(Direction.TO, job, request.getFrameworkLinkConfig(Direction.TO));
- ConfigurationUtils.setFrameworkJobConfig(job, request.getFrameworkJobConfig());
+ // from and to configs
+ ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getJobConfig(Direction.FROM));
+ ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getJobConfig(Direction.TO));
+ ConfigurationUtils.setDriverConfig(job, request.getDriverConfig());
+ // @TODO(Abe): Persist TO schema.
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, request.getSummary().getFromSchema());
ConfigurationUtils.setConnectorSchema(Direction.TO, job, request.getSummary().getToSchema());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
index 9ecc9da..7504e76 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
@@ -17,6 +17,12 @@
*/
package org.apache.sqoop.test.minicluster;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.conf.Configuration;
import org.codehaus.cargo.container.ContainerType;
import org.codehaus.cargo.container.InstalledLocalContainer;
@@ -25,16 +31,9 @@ import org.codehaus.cargo.container.configuration.LocalConfiguration;
import org.codehaus.cargo.container.deployable.WAR;
import org.codehaus.cargo.container.installer.Installer;
import org.codehaus.cargo.container.installer.ZipURLInstaller;
-import org.codehaus.cargo.container.property.GeneralPropertySet;
import org.codehaus.cargo.generic.DefaultContainerFactory;
import org.codehaus.cargo.generic.configuration.DefaultConfigurationFactory;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
/**
* Embedded tomcat Sqoop server mini cluster.
*
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
index 6074d36..af31769 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
@@ -17,15 +17,18 @@
*/
package org.apache.sqoop.test.testcases;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
import org.apache.log4j.Logger;
import org.apache.sqoop.client.SubmissionCallback;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.test.asserts.ProviderAsserts;
@@ -39,9 +42,6 @@ import org.apache.sqoop.validation.Status;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-
/**
* Base test case suitable for connector testing.
*
@@ -121,41 +121,50 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
}
/**
- * Fill link form based on currently active provider.
+ * Fill link config based on currently active provider.
*
* @param link MLink object to fill
*/
- protected void fillRdbmsLinkForm(MLink link) {
- MFormList forms = link.getConnectorPart();
- forms.getStringInput("link.jdbcDriver").setValue(provider.getJdbcDriver());
- forms.getStringInput("link.connectionString").setValue(provider.getConnectionUrl());
- forms.getStringInput("link.username").setValue(provider.getConnectionUsername());
- forms.getStringInput("link.password").setValue(provider.getConnectionPassword());
+ protected void fillRdbmsLinkConfig(MLink link) {
+ MConfigList configs = link.getConnectorLinkConfig();
+ configs.getStringInput("linkConfig.jdbcDriver").setValue(provider.getJdbcDriver());
+ configs.getStringInput("linkConfig.connectionString").setValue(provider.getConnectionUrl());
+ configs.getStringInput("linkConfig.username").setValue(provider.getConnectionUsername());
+ configs.getStringInput("linkConfig.password").setValue(provider.getConnectionPassword());
}
/**
- * Fill TO form with specific storage and output type.
+ * Fill TO config with specific storage and output type.
*
* @param job MJob object to fill
* @param output Output type that should be set
*/
- protected void fillToJobForm(MJob job, ToFormat output) {
- MFormList toForms = job.getConnectorPart(Direction.TO);
- toForms.getEnumInput("toJobConfig.outputFormat").setValue(output);
- toForms.getStringInput("toJobConfig.outputDirectory").setValue(getMapreduceDirectory());
+ protected void fillHdfsToConfig(MJob job, ToFormat output) {
+ MConfigList toConfig = job.getJobConfig(Direction.TO);
+ toConfig.getEnumInput("toJobConfig.outputFormat").setValue(output);
+ toConfig.getStringInput("toJobConfig.outputDirectory").setValue(getMapreduceDirectory());
}
/**
- * Fill FROM form
+ * Fill FROM config
*
* @param job MJob object to fill
*/
- protected void fillFromJobForm(MJob job) {
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.inputDirectory").setValue(getMapreduceDirectory());
+ protected void fillHdfsFromConfig(MJob job) {
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fromConfig.getStringInput("fromJobConfig.inputDirectory").setValue(getMapreduceDirectory());
}
/**
+ * Fill Driver config
+ * @param job
+ */
+ protected void fillDriverConfig(MJob job) {
+ job.getDriverConfig().getStringInput("throttlingConfig.numExtractors").setValue("3");
+ }
+
+
+ /**
* Create table cities.
*/
protected void createTableCities() {
@@ -186,7 +195,7 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
/**
* Assert row in testing table.
*
- * @param conditions Conditions in form that are expected by the database provider
+ * @param conditions Conditions in config that are expected by the database provider
* @param values Values that are expected in the table (with corresponding types)
*/
protected void assertRow(Object []conditions, Object ...values) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
index 63736ab..5e1d564 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
@@ -165,7 +165,7 @@ abstract public class TomcatTestCase {
* @throws IOException
*/
protected void assertTo(String... lines) throws IOException {
- // TODO(VB): fix this to be not directly dependent on mapreduce
+ // TODO(VB): fix this to be not directly dependent on hdfs/MR
HdfsAsserts.assertMapreduceOutput(hdfsClient, getMapreduceDirectory(), lines);
}
@@ -175,7 +175,7 @@ abstract public class TomcatTestCase {
* @param expectedFiles Expected number of files
*/
protected void assertToFiles(int expectedFiles) throws IOException {
- // TODO(VB): fix this to be not directly dependent on mapreduce
+ // TODO(VB): fix this to be not directly dependent on hdfs/MR
HdfsAsserts.assertMapreduceOutputFiles(hdfsClient, getMapreduceDirectory(), expectedFiles);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
index 5e1abc1..b1b3b16 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
@@ -19,8 +19,9 @@ package org.apache.sqoop.integration.connector.jdbc.generic;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
import org.junit.Test;
@@ -43,7 +44,7 @@ public class FromHDFSToRDBMSTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -53,10 +54,16 @@ public class FromHDFSToRDBMSTest extends ConnectorTestCase {
// Job creation
MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId());
- // Connector values
- MFormList toForms = job.getConnectorPart(Direction.TO);
- toForms.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- fillFromJobForm(job);
+ // set hdfs "FROM" config for the job, since the connector test case base class only has utilities for hdfs!
+ fillHdfsFromConfig(job);
+
+ // set the rdms "TO" config here
+ MConfigList toConfig = job.getJobConfig(Direction.TO);
+ toConfig.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+
+ // driver config
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
saveJob(job);
executeJob(job);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
index 2dc0613..36f7443 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
@@ -17,17 +17,18 @@
*/
package org.apache.sqoop.integration.connector.jdbc.generic;
+import static org.junit.Assert.assertTrue;
+
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.junit.Test;
-import static org.junit.Assert.assertTrue;
-
/**
* Import simple table with various configurations.
*/
@@ -39,7 +40,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsConnection);
+ fillRdbmsLinkConfig(rdbmsConnection);
saveLink(rdbmsConnection);
// HDFS link
@@ -49,11 +50,17 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
// Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());
- // Connector values
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
- fillToJobForm(job, ToFormat.TEXT_FILE);
+ // srt rdbms "FROM" config
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
+
+ // fill the hdfs "TO" config
+ fillHdfsToConfig(job, ToFormat.TEXT_FILE);
+ // driver config
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
+
saveJob(job);
executeJob(job);
@@ -76,7 +83,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -87,11 +94,11 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
// Connector values
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
- forms.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country"));
- fillToJobForm(job, ToFormat.TEXT_FILE);
+ MConfigList configs = job.getJobConfig(Direction.FROM);
+ configs.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ configs.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
+ configs.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country"));
+ fillHdfsToConfig(job, ToFormat.TEXT_FILE);
saveJob(job);
MSubmission submission = getClient().startSubmission(job.getPersistenceId());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
index 729f95e..824a51d 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
@@ -19,8 +19,9 @@ package org.apache.sqoop.integration.connector.jdbc.generic;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.apache.sqoop.test.utils.ParametrizedUtils;
@@ -59,8 +60,8 @@ public class PartitionerTest extends ConnectorTestCase {
}
private String partitionColumn;
- private int extractors;
private int maxOutputFiles;
+ private int extractors;
public PartitionerTest(String partitionColumn, int expectedOutputFiles, int extractors) {
this.partitionColumn = partitionColumn;
@@ -74,7 +75,7 @@ public class PartitionerTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -84,13 +85,17 @@ public class PartitionerTest extends ConnectorTestCase {
// Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
- // Connector values
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn));
- fillToJobForm(job, ToFormat.TEXT_FILE);
- forms = job.getFrameworkPart();
- forms.getIntegerInput("throttling.extractors").setValue(extractors);
+ // set the rdbms "FROM" config
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn));
+ // fill hdfs "TO" config
+ fillHdfsToConfig(job, ToFormat.TEXT_FILE);
+
+ // set driver config
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(extractors);
+
saveJob(job);
executeJob(job);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
index 562a6a6..f42fa32 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
@@ -17,16 +17,16 @@
*/
package org.apache.sqoop.integration.connector.jdbc.generic;
+import static org.junit.Assert.assertEquals;
+
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.test.data.Cities;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
/**
*
*/
@@ -46,7 +46,7 @@ public class TableStagedRDBMSTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -57,13 +57,19 @@ public class TableStagedRDBMSTest extends ConnectorTestCase {
MJob job = getClient().createJob(hdfsLink.getPersistenceId(),
rdbmsLink.getPersistenceId());
- // Connector values
- MFormList forms = job.getConnectorPart(Direction.TO);
- forms.getStringInput("toJobConfig.tableName").setValue(
+ // fill HDFS "FROM" config
+ fillHdfsFromConfig(job);
+
+ // fill rdbms "TO" config here
+ MConfigList configs = job.getJobConfig(Direction.TO);
+ configs.getStringInput("toJobConfig.tableName").setValue(
provider.escapeTableName(getTableName()));
- forms.getStringInput("toJobConfig.stageTableName").setValue(
+ configs.getStringInput("toJobConfig.stageTableName").setValue(
provider.escapeTableName(stageTableName));
- fillFromJobForm(job);
+ // driver config
+ MConfigList driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
+
saveJob(job);
executeJob(job);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
index 507ac53..055bc3d 100644
--- a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
@@ -23,7 +23,7 @@ import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
import org.apache.sqoop.driver.DriverError;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.junit.runner.RunWith;
@@ -67,7 +67,7 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -77,15 +77,21 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase {
// Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
- // Connector values
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
- // Framework values
- fillToJobForm(job, ToFormat.TEXT_FILE);
+ // rdms "FROM" config
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
+
+ // hdfs "TO" config
+ fillHdfsToConfig(job, ToFormat.TEXT_FILE);
+
+ // driver config
+ MConfigList driverConfig = job.getDriverConfig();
+ //driverConfig.getIntegerInput("throttlingConfig.extractors").setValue(3);
+
saveJob(job);
- // Disable model entities as per parametrized run
+ // Disable model entities as per parameterized run
getClient().enableLink(rdbmsLink.getPersistenceId(), enabledLink);
getClient().enableJob(job.getPersistenceId(), enabledJob);
[02/52] [abbrv] SQOOP-1374: From/To: Metadata upgrade
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index f603cc1..29da340 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -18,6 +18,7 @@
package org.apache.sqoop.repository.derby;
import junit.framework.TestCase;
+import org.apache.sqoop.common.Direction;
import org.apache.sqoop.framework.FrameworkManager;
import org.apache.sqoop.model.MConnection;
import org.apache.sqoop.model.MConnectionForms;
@@ -47,443 +48,648 @@ import static org.apache.sqoop.repository.derby.DerbySchemaQuery.*;
*/
abstract public class DerbyTestCase extends TestCase {
-// public static final String DERBY_DRIVER =
-// "org.apache.derby.jdbc.EmbeddedDriver";
-//
-// public static final String JDBC_URL =
-// "jdbc:derby:memory:myDB";
-//
-// private Connection connection;
-//
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// // Create connection to the database
-// Class.forName(DERBY_DRIVER).newInstance();
-// connection = DriverManager.getConnection(getStartJdbcUrl());
-// }
-//
-// @Override
-// public void tearDown() throws Exception {
-// // Close active connection
-// if(connection != null) {
-// connection.close();
-// }
-//
-// try {
-// // Drop in memory database
-// DriverManager.getConnection(getStopJdbcUrl());
-// } catch (SQLException ex) {
-// // Dropping Derby database leads always to exception
-// }
-//
-// // Call parent tear down
-// super.tearDown();
-// }
-//
-// /**
-// * Create derby schema.
-// *
-// * @throws Exception
-// */
-// protected void createSchema() throws Exception {
-// runQuery(QUERY_CREATE_SCHEMA_SQOOP);
-// runQuery(QUERY_CREATE_TABLE_SQ_CONNECTOR);
-// runQuery(QUERY_CREATE_TABLE_SQ_FORM);
-// runQuery(QUERY_CREATE_TABLE_SQ_INPUT);
-// runQuery(QUERY_CREATE_TABLE_SQ_CONNECTION);
-// runQuery(QUERY_CREATE_TABLE_SQ_JOB);
-// runQuery(QUERY_CREATE_TABLE_SQ_CONNECTION_INPUT);
-// runQuery(QUERY_CREATE_TABLE_SQ_JOB_INPUT);
-// runQuery(QUERY_CREATE_TABLE_SQ_SUBMISSION);
-// runQuery(QUERY_CREATE_TABLE_SQ_COUNTER_GROUP);
-// runQuery(QUERY_CREATE_TABLE_SQ_COUNTER);
-// runQuery(QUERY_CREATE_TABLE_SQ_COUNTER_SUBMISSION);
-// runQuery(QUERY_CREATE_TABLE_SQ_SYSTEM);
-// runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_ENABLED);
-// runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_ENABLED);
-// runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_CREATION_USER);
-// runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_UPDATE_USER);
-// runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_CREATION_USER);
-// runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_UPDATE_USER);
-// runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_CREATION_USER);
-// runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_UPDATE_USER);
-// runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) VALUES('version', '3')");
-// runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) " +
-// "VALUES('framework.version', '1')");
-// }
-//
-// /**
-// * Run arbitrary query on derby memory repository.
-// *
-// * @param query Query to execute
-// * @throws Exception
-// */
-// protected void runQuery(String query) throws Exception {
-// Statement stmt = null;
-// try {
-// stmt = getDerbyConnection().createStatement();
-//
-// stmt.execute(query);
-// } finally {
-// if (stmt != null) {
-// stmt.close();
-// }
-// }
-// }
-//
-// protected Connection getDerbyConnection() {
-// return connection;
-// }
-//
-// protected String getJdbcUrl() {
-// return JDBC_URL;
-// }
-//
-// protected String getStartJdbcUrl() {
-// return JDBC_URL + ";create=true";
-// }
-//
-// protected String getStopJdbcUrl() {
-// return JDBC_URL + ";drop=true";
-// }
-//
-// /**
-// * Load testing connector and framework metadata into repository.
-// *
-// * @throws Exception
-// */
-// protected void loadConnectorAndFramework() throws Exception {
-// // Connector entry
-// runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
-// + "VALUES('A', 'org.apache.sqoop.test.A', '1.0-test')");
-//
-// for(String connector : new String[] {"1", "NULL"}) {
-// // Form entries
-// for(String operation : new String[] {"null", "'IMPORT'", "'EXPORT'"}) {
-//
-// String type;
-// if(operation.equals("null")) {
-// type = "CONNECTION";
-// } else {
-// type = "JOB";
-// }
-//
-// runQuery("INSERT INTO SQOOP.SQ_FORM"
-// + "(SQF_CONNECTOR, SQF_OPERATION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
-// + "VALUES("
-// + connector + ", "
-// + operation
-// + ", 'F1', '"
-// + type
-// + "', 0)");
-// runQuery("INSERT INTO SQOOP.SQ_FORM"
-// + "(SQF_CONNECTOR, SQF_OPERATION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
-// + "VALUES("
-// + connector + ", "
-// + operation
-// + ", 'F2', '"
-// + type
-// + "', 1)");
-// }
-// }
-//
-// // Input entries
-// for(int x = 0; x < 2; x++ ) {
-// for(int i = 0; i < 3; i++) {
-// // First form
-// runQuery("INSERT INTO SQOOP.SQ_INPUT"
-// +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
-// + " VALUES('I1', " + (x * 6 + i * 2 + 1) + ", 0, 'STRING', false, 30)");
-// runQuery("INSERT INTO SQOOP.SQ_INPUT"
-// +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
-// + " VALUES('I2', " + (x * 6 + i * 2 + 1) + ", 1, 'MAP', false, 30)");
-//
-// // Second form
-// runQuery("INSERT INTO SQOOP.SQ_INPUT"
-// +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
-// + " VALUES('I3', " + (x * 6 + i * 2 + 2) + ", 0, 'STRING', false, 30)");
-// runQuery("INSERT INTO SQOOP.SQ_INPUT"
-// +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
-// + " VALUES('I4', " + (x * 6 + i * 2 + 2) + ", 1, 'MAP', false, 30)");
-// }
-// }
-// }
-//
-// /**
-// * Load testing connection objects into metadata repository.
-// *
-// * @throws Exception
-// */
-// public void loadConnections() throws Exception {
-// // Insert two connections - CA and CB
-// runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
-// + "VALUES('CA', 1)");
-// runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
-// + "VALUES('CB', 1)");
-//
-// for(String ci : new String[] {"1", "2"}) {
-// for(String i : new String[] {"1", "3", "13", "15"}) {
-// runQuery("INSERT INTO SQOOP.SQ_CONNECTION_INPUT"
-// + "(SQNI_CONNECTION, SQNI_INPUT, SQNI_VALUE) "
-// + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
-// }
-// }
-// }
-//
-// /**
-// * Load testing job objects into metadata repository.
-// *
-// * @throws Exception
-// */
-// public void loadJobs() throws Exception {
-// for(String type : new String[] {"IMPORT", "EXPORT"}) {
-// for(String name : new String[] {"JA", "JB"} ) {
-// runQuery("INSERT INTO SQOOP.SQ_JOB(SQB_NAME, SQB_CONNECTION, SQB_TYPE)"
-// + " VALUES('" + name + "', 1, '" + type + "')");
-// }
-// }
-//
-// // Import inputs
-// for(String ci : new String[] {"1", "2"}) {
-// for(String i : new String[] {"5", "7", "17", "19"}) {
-// runQuery("INSERT INTO SQOOP.SQ_JOB_INPUT"
-// + "(SQBI_JOB, SQBI_INPUT, SQBI_VALUE) "
-// + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
-// }
-// }
-//
-// // Export inputs
-// for(String ci : new String[] {"3", "4"}) {
-// for(String i : new String[] {"9", "11", "21", "23"}) {
-// runQuery("INSERT INTO SQOOP.SQ_JOB_INPUT"
-// + "(SQBI_JOB, SQBI_INPUT, SQBI_VALUE) "
-// + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
-// }
-// }
-// }
-//
-// /**
-// * Add a second connector for testing with multiple connectors
-// */
-// public void addConnector() throws Exception {
-// // Connector entry
-// runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
-// + "VALUES('B', 'org.apache.sqoop.test.B', '1.0-test')");
-// }
-//
-// /**
-// * Load testing submissions into the metadata repository.
-// *
-// * @throws Exception
-// */
-// public void loadSubmissions() throws Exception {
-// runQuery("INSERT INTO SQOOP.SQ_COUNTER_GROUP "
-// + "(SQG_NAME) "
-// + "VALUES"
-// + "('gA'), ('gB')"
-// );
-//
-// runQuery("INSERT INTO SQOOP.SQ_COUNTER "
-// + "(SQR_NAME) "
-// + "VALUES"
-// + "('cA'), ('cB')"
-// );
-//
-// runQuery("INSERT INTO SQOOP.SQ_SUBMISSION"
-// + "(SQS_JOB, SQS_STATUS, SQS_CREATION_DATE, SQS_UPDATE_DATE,"
-// + " SQS_EXTERNAL_ID, SQS_EXTERNAL_LINK, SQS_EXCEPTION,"
-// + " SQS_EXCEPTION_TRACE)"
-// + "VALUES "
-// + "(1, 'RUNNING', '2012-01-01 01:01:01', '2012-01-01 01:01:01', 'job_1',"
-// + "NULL, NULL, NULL),"
-// + "(2, 'SUCCEEDED', '2012-01-01 01:01:01', '2012-01-02 01:01:01', 'job_2',"
-// + " NULL, NULL, NULL),"
-// + "(3, 'FAILED', '2012-01-01 01:01:01', '2012-01-03 01:01:01', 'job_3',"
-// + " NULL, NULL, NULL),"
-// + "(4, 'UNKNOWN', '2012-01-01 01:01:01', '2012-01-04 01:01:01', 'job_4',"
-// + " NULL, NULL, NULL),"
-// + "(1, 'RUNNING', '2012-01-01 01:01:01', '2012-01-05 01:01:01', 'job_5',"
-// + " NULL, NULL, NULL)"
-// );
-//
-// runQuery("INSERT INTO SQOOP.SQ_COUNTER_SUBMISSION "
-// + "(SQRS_GROUP, SQRS_COUNTER, SQRS_SUBMISSION, SQRS_VALUE) "
-// + "VALUES"
-// + "(1, 1, 4, 300)"
-// );
-//
-// }
-//
-// protected MConnector getConnector() {
-// return new MConnector("A", "org.apache.sqoop.test.A", "1.0-test",
-// getConnectionForms(), getJobForms());
-// }
-//
-// protected MFramework getFramework() {
-// return new MFramework(getConnectionForms(), getJobForms(),
-// FrameworkManager.CURRENT_FRAMEWORK_VERSION);
-// }
-//
-// protected void fillConnection(MConnection connection) {
-// List<MForm> forms;
-//
-// forms = connection.getConnectorPart().getForms();
-// ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
-// ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
-//
-// forms = connection.getFrameworkPart().getForms();
-// ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value13");
-// ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value15");
-// }
-//
-// protected void fillJob(MJob job) {
-// List<MForm> forms;
-//
-// forms = job.getFromPart().getForms();
-// ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
-// ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
-//
-// forms = job.getFrameworkPart().getForms();
-// ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value13");
-// ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value15");
-// }
-//
-// protected List<MJobForms> getJobForms() {
-// List <MJobForms> jobForms = new LinkedList<MJobForms>();
-// jobForms.add(new MJobForms(MJob.Type.IMPORT, getForms()));
-// jobForms.add(new MJobForms(MJob.Type.EXPORT, getForms()));
-// return jobForms;
-// }
-//
-// protected MConnectionForms getConnectionForms() {
-// return new MConnectionForms(getForms());
-// }
-//
-// protected List<MForm> getForms() {
-// List<MForm> forms = new LinkedList<MForm>();
-//
-// List<MInput<?>> inputs;
-// MInput input;
-//
-// inputs = new LinkedList<MInput<?>>();
-// input = new MStringInput("I1", false, (short)30);
-// inputs.add(input);
-// input = new MMapInput("I2", false);
-// inputs.add(input);
-// forms.add(new MForm("F1", inputs));
-//
-// inputs = new LinkedList<MInput<?>>();
-// input = new MStringInput("I3", false, (short)30);
-// inputs.add(input);
-// input = new MMapInput("I4", false);
-// inputs.add(input);
-// forms.add(new MForm("F2", inputs));
-//
-// return forms;
-// }
-//
-// /**
-// * Find out number of entries in given table.
-// *
-// * @param table Table name
-// * @return Number of rows in the table
-// * @throws Exception
-// */
-// protected long countForTable(String table) throws Exception {
-// Statement stmt = null;
-// ResultSet rs = null;
-//
-// try {
-// stmt = getDerbyConnection().createStatement();
-//
-// rs = stmt.executeQuery("SELECT COUNT(*) FROM "+ table);
-// rs.next();
-//
-// return rs.getLong(1);
-// } finally {
-// if(stmt != null) {
-// stmt.close();
-// }
-// if(rs != null) {
-// rs.close();
-// }
-// }
-// }
-//
-// /**
-// * Assert row count for given table.
-// *
-// * @param table Table name
-// * @param expected Expected number of rows
-// * @throws Exception
-// */
-// protected void assertCountForTable(String table, long expected)
-// throws Exception {
-// long count = countForTable(table);
-// assertEquals(expected, count);
-// }
-//
-// /**
-// * Printout repository content for advance debugging.
-// *
-// * This method is currently unused, but might be helpful in the future, so
-// * I'm letting it here.
-// *
-// * @throws Exception
-// */
-// protected void generateDatabaseState() throws Exception {
-// for(String tbl : new String[] {"SQ_CONNECTOR", "SQ_FORM", "SQ_INPUT",
-// "SQ_CONNECTION", "SQ_CONNECTION_INPUT", "SQ_JOB", "SQ_JOB_INPUT"}) {
-// generateTableState("SQOOP." + tbl);
-// }
-// }
-//
-// /**
-// * Printout one single table.
-// *
-// * @param table Table name
-// * @throws Exception
-// */
-// protected void generateTableState(String table) throws Exception {
-// PreparedStatement ps = null;
-// ResultSet rs = null;
-// ResultSetMetaData rsmt = null;
-//
-// try {
-// ps = getDerbyConnection().prepareStatement("SELECT * FROM " + table);
-// rs = ps.executeQuery();
-//
-// rsmt = rs.getMetaData();
-//
-// StringBuilder sb = new StringBuilder();
-// System.out.println("Table " + table + ":");
-//
-// for(int i = 1; i <= rsmt.getColumnCount(); i++) {
-// sb.append("| ").append(rsmt.getColumnName(i)).append(" ");
-// }
-// sb.append("|");
-// System.out.println(sb.toString());
-//
-// while(rs.next()) {
-// sb = new StringBuilder();
-// for(int i = 1; i <= rsmt.getColumnCount(); i++) {
-// sb.append("| ").append(rs.getString(i)).append(" ");
-// }
-// sb.append("|");
-// System.out.println(sb.toString());
-// }
-//
-// System.out.println("");
-//
-// } finally {
-// if(rs != null) {
-// rs.close();
-// }
-// if(ps != null) {
-// ps.close();
-// }
-// }
-// }
+ private static int SYSTEM_VERSION = 4;
+
+ public static final String DERBY_DRIVER =
+ "org.apache.derby.jdbc.EmbeddedDriver";
+
+ public static final String JDBC_URL =
+ "jdbc:derby:memory:myDB";
+
+ private Connection connection;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ // Create connection to the database
+ Class.forName(DERBY_DRIVER).newInstance();
+ connection = DriverManager.getConnection(getStartJdbcUrl());
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ // Close active connection
+ if(connection != null) {
+ connection.close();
+ }
+
+ try {
+ // Drop in memory database
+ DriverManager.getConnection(getStopJdbcUrl());
+ } catch (SQLException ex) {
+ // Dropping Derby database leads always to exception
+ }
+
+ // Call parent tear down
+ super.tearDown();
+ }
+
+ /**
+ * Create derby schema.
+ *
+ * @throws Exception
+ */
+ protected void createSchema(int version) throws Exception {
+ if (version > 0) {
+ runQuery(QUERY_CREATE_SCHEMA_SQOOP);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONNECTOR);
+ runQuery(QUERY_CREATE_TABLE_SQ_FORM);
+ runQuery(QUERY_CREATE_TABLE_SQ_INPUT);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONNECTION);
+ runQuery(QUERY_CREATE_TABLE_SQ_JOB);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONNECTION_INPUT);
+ runQuery(QUERY_CREATE_TABLE_SQ_JOB_INPUT);
+ runQuery(QUERY_CREATE_TABLE_SQ_SUBMISSION);
+ runQuery(QUERY_CREATE_TABLE_SQ_COUNTER_GROUP);
+ runQuery(QUERY_CREATE_TABLE_SQ_COUNTER);
+ runQuery(QUERY_CREATE_TABLE_SQ_COUNTER_SUBMISSION);
+ }
+
+ if (version > 1) {
+ runQuery(QUERY_CREATE_TABLE_SQ_SYSTEM);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_ENABLED);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_ENABLED);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_CREATION_USER);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_UPDATE_USER);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_CREATION_USER);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_UPDATE_USER);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_CREATION_USER);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_UPDATE_USER);
+ }
+
+ if (version > 3) {
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_RENAME_COLUMN_SQF_OPERATION_TO_SQF_DIRECTION);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_CONNECTION_TO_SQB_FROM_CONNECTION);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_CONNECTION);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQN);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_FROM);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_TO);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE);
+ }
+
+ runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) VALUES('version', '" + version + "')");
+ runQuery("INSERT INTO SQOOP.SQ_SYSTEM(SQM_KEY, SQM_VALUE) " +
+ "VALUES('framework.version', '1')");
+ }
+
+ protected void createSchema() throws Exception {
+ createSchema(SYSTEM_VERSION);
+ }
+
+ /**
+ * Run arbitrary query on derby memory repository.
+ *
+ * @param query Query to execute
+ * @throws Exception
+ */
+ protected void runQuery(String query, String... args) throws Exception {
+ PreparedStatement stmt = null;
+ try {
+ stmt = getDerbyConnection().prepareStatement(query);
+
+ for (int i = 0; i < args.length; ++i) {
+ stmt.setString(i + 1, args[i]);
+ }
+
+ stmt.execute();
+ } finally {
+ if (stmt != null) {
+ stmt.close();
+ }
+ }
+ }
+
+ protected Connection getDerbyConnection() {
+ return connection;
+ }
+
+ protected String getJdbcUrl() {
+ return JDBC_URL;
+ }
+
+ protected String getStartJdbcUrl() {
+ return JDBC_URL + ";create=true";
+ }
+
+ protected String getStopJdbcUrl() {
+ return JDBC_URL + ";drop=true";
+ }
+
+ protected void loadConnectorAndFrameworkVersion2() throws Exception {
+ // Connector entry
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ + "VALUES('A', 'org.apache.sqoop.test.A', '1.0-test')");
+
+ String connector = "1";
+
+ // Connector form entries
+ for(String operation : new String[] {"null", "'IMPORT'", "'EXPORT'"}) {
+
+ String type;
+ if(operation.equals("null")) {
+ type = "CONNECTION";
+ } else {
+ type = "JOB";
+ }
+
+ runQuery("INSERT INTO SQOOP.SQ_FORM"
+ + "(SQF_CONNECTOR, SQF_OPERATION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ + "VALUES("
+ + connector + ", "
+ + operation
+ + ", 'F1', '"
+ + type
+ + "', 0)");
+ runQuery("INSERT INTO SQOOP.SQ_FORM"
+ + "(SQF_CONNECTOR, SQF_OPERATION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ + "VALUES("
+ + connector + ", "
+ + operation
+ + ", 'F2', '"
+ + type
+ + "', 1)");
+ }
+
+ // Framework form entries
+ runQuery("INSERT INTO SQOOP.SQ_FORM"
+ + "(SQF_CONNECTOR, SQF_OPERATION, SQF_NAME, SQF_TYPE, SQF_INDEX) VALUES"
+ + "(NULL, 'IMPORT', 'output', 'JOB', 0),"
+ + "(NULL, 'IMPORT', 'throttling', 'JOB', 1),"
+ + "(NULL, 'EXPORT', 'input', 'JOB', 0),"
+ + "(NULL, 'EXPORT', 'throttling', 'JOB', 1),"
+ + "(NULL, NULL, 'security', 'CONNECTION', 0)");
+
+ // Connector input entries
+ int x = 0;
+ for(int i = 0; i < 3; i++) {
+ // First form
+ runQuery("INSERT INTO SQOOP.SQ_INPUT"
+ +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + " VALUES('I1', " + (i * 2 + 1) + ", 0, 'STRING', false, 30)");
+ runQuery("INSERT INTO SQOOP.SQ_INPUT"
+ +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + " VALUES('I2', " + (i * 2 + 1) + ", 1, 'MAP', false, 30)");
+
+ // Second form
+ runQuery("INSERT INTO SQOOP.SQ_INPUT"
+ +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + " VALUES('I3', " + (i * 2 + 2) + ", 0, 'STRING', false, 30)");
+ runQuery("INSERT INTO SQOOP.SQ_INPUT"
+ +"(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + " VALUES('I4', " + (i * 2 + 2) + ", 1, 'MAP', false, 30)");
+ }
+
+ // Framework input entries.
+ runQuery("INSERT INTO SQOOP.SQ_INPUT (SQI_NAME, SQI_FORM, SQI_INDEX,"
+ + " SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH, SQI_ENUMVALS)"
+ +" VALUES ('security.maxConnections',11,0,'INTEGER','false',NULL,NULL),"
+ + "('input.inputDirectory',9,0,'STRING','false',255,NULL),"
+ + "('throttling.extractors',8,0,'INTEGER','false',NULL,NULL),"
+ + "('throttling.loaders',8,1,'INTEGER','false',NULL,NULL),"
+ + "('output.storageType',7,0,'ENUM','false',NULL,'HDFS'),"
+ + "('output.outputFormat',7,1,'ENUM','false',NULL,'TEXT_FILE,SEQUENCE_FILE'),"
+ + "('output.compression',7,2,'ENUM','false',NULL,'NONE,DEFAULT,DEFLATE,GZIP,BZIP2,LZO,LZ4,SNAPPY'),"
+ + "('output.outputDirectory',7,3,'STRING','false',255,NULL),"
+ + "('throttling.extractors',10,0,'INTEGER','false',NULL,NULL),"
+ + "('throttling.loaders',10,1,'INTEGER','false',NULL,NULL)");
+ }
+
+ protected void loadConnectorAndFrameworkVersion4() throws Exception {
+ // Connector entry
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ + "VALUES('A', 'org.apache.sqoop.test.A', '1.0-test')");
+
+ // Connector part
+ for (String connector : new String[]{"1"}) {
+ // Form entries
+ for (String direction : new String[]{"null", "'FROM'", "'TO'"}) {
+
+ String type;
+ if (direction.equals("null")) {
+ type = "CONNECTION";
+ } else {
+ type = "JOB";
+ }
+
+ runQuery("INSERT INTO SQOOP.SQ_FORM"
+ + "(SQF_CONNECTOR, SQF_DIRECTION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ + "VALUES("
+ + connector + ", "
+ + direction
+ + ", 'F1', '"
+ + type
+ + "', 0)");
+ runQuery("INSERT INTO SQOOP.SQ_FORM"
+ + "(SQF_CONNECTOR, SQF_DIRECTION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ + "VALUES("
+ + connector + ", "
+ + direction
+ + ", 'F2', '"
+ + type
+ + "', 1)");
+ }
+ }
+
+ // Framework part
+ for (String type : new String[]{"CONNECTION", "JOB"}) {
+ runQuery("INSERT INTO SQOOP.SQ_FORM"
+ + "(SQF_CONNECTOR, SQF_DIRECTION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ + "VALUES(NULL, NULL"
+ + ", 'F1', '"
+ + type
+ + "', 0)");
+ runQuery("INSERT INTO SQOOP.SQ_FORM"
+ + "(SQF_CONNECTOR, SQF_DIRECTION, SQF_NAME, SQF_TYPE, SQF_INDEX) "
+ + "VALUES(NULL, NULL"
+ + ", 'F2', '"
+ + type
+ + "', 1)");
+ }
+
+ // Input entries
+ // Connector connection parts: 0-3
+ // Connector job (FROM) parts: 4-7
+ // Connector job (TO) parts: 8-11
+ // Framework connection parts: 12-15
+ // Framework job parts: 16-19
+ for (int i = 0; i < 5; i++) {
+ // First form
+ runQuery("INSERT INTO SQOOP.SQ_INPUT"
+ + "(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + " VALUES('I1', " + (i * 2 + 1) + ", 0, 'STRING', false, 30)");
+ runQuery("INSERT INTO SQOOP.SQ_INPUT"
+ + "(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + " VALUES('I2', " + (i * 2 + 1) + ", 1, 'MAP', false, 30)");
+
+ // Second form
+ runQuery("INSERT INTO SQOOP.SQ_INPUT"
+ + "(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + " VALUES('I3', " + (i * 2 + 2) + ", 0, 'STRING', false, 30)");
+ runQuery("INSERT INTO SQOOP.SQ_INPUT"
+ + "(SQI_NAME, SQI_FORM, SQI_INDEX, SQI_TYPE, SQI_STRMASK, SQI_STRLENGTH)"
+ + " VALUES('I4', " + (i * 2 + 2) + ", 1, 'MAP', false, 30)");
+ }
+ }
+
+ /**
+ * Load testing connector and framework metadata into repository.
+ *
+ * @param version system version (2 or 4)
+ * @throws Exception
+ */
+ protected void loadConnectorAndFramework(int version) throws Exception {
+ switch(version) {
+ case 2:
+ loadConnectorAndFrameworkVersion2();
+ break;
+
+ case 4:
+ loadConnectorAndFrameworkVersion4();
+ break;
+
+ default:
+ throw new AssertionError("Invalid connector and framework version: " + version);
+ }
+ }
+
+ protected void loadConnectorAndFramework() throws Exception {
+ loadConnectorAndFramework(SYSTEM_VERSION);
+ }
+
+ /**
+ * Load testing connection objects into metadata repository.
+ *
+ * @param version system version (2 or 4)
+ * @throws Exception
+ */
+ public void loadConnections(int version) throws Exception {
+ switch (version) {
+ case 2:
+ // Insert two connections - CA and CB
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ + "VALUES('CA', 1)");
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ + "VALUES('CB', 1)");
+
+ for(String ci : new String[] {"1", "2"}) {
+ for(String i : new String[] {"1", "3", "13", "15"}) {
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTION_INPUT"
+ + "(SQNI_CONNECTION, SQNI_INPUT, SQNI_VALUE) "
+ + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
+ }
+ }
+ break;
+
+ case 4:
+ // Insert two connections - CA and CB
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ + "VALUES('CA', 1)");
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTION(SQN_NAME, SQN_CONNECTOR) "
+ + "VALUES('CB', 1)");
+
+ for (String ci : new String[]{"1", "2"}) {
+ for (String i : new String[]{"1", "3", "13", "15"}) {
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTION_INPUT"
+ + "(SQNI_CONNECTION, SQNI_INPUT, SQNI_VALUE) "
+ + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
+ }
+ }
+ break;
+
+ default:
+ throw new AssertionError("Invalid connector and framework version: " + version);
+ }
+ }
+
+ public void loadConnections() throws Exception {
+ loadConnections(SYSTEM_VERSION);
+ }
+
+ /**
+ * Load testing job objects into metadata repository.
+ *
+ * @param version system version (2 or 4)
+ * @throws Exception
+ */
+ public void loadJobs(int version) throws Exception {
+ switch (version) {
+ case 2:
+ for(String type : new String[] {"IMPORT", "EXPORT"}) {
+ for(String name : new String[] {"JA", "JB"} ) {
+ runQuery("INSERT INTO SQOOP.SQ_JOB(SQB_NAME, SQB_CONNECTION, SQB_TYPE)"
+ + " VALUES('" + name + "', 1, '" + type + "')");
+ }
+ }
+
+ // Import inputs
+ for(String ci : new String[] {"1", "2"}) {
+ for(String i : new String[] {"5", "7", "17", "19"}) {
+ runQuery("INSERT INTO SQOOP.SQ_JOB_INPUT"
+ + "(SQBI_JOB, SQBI_INPUT, SQBI_VALUE) "
+ + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
+ }
+ }
+
+ // Export inputs
+ for(String ci : new String[] {"3", "4"}) {
+ for(String i : new String[] {"9", "11"}) {
+ runQuery("INSERT INTO SQOOP.SQ_JOB_INPUT"
+ + "(SQBI_JOB, SQBI_INPUT, SQBI_VALUE) "
+ + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
+ }
+ }
+ break;
+
+
+ case 4:
+ for (String name : new String[]{"JA", "JB", "JC", "JD"}) {
+ runQuery("INSERT INTO SQOOP.SQ_JOB(SQB_NAME, SQB_FROM_CONNECTION, SQB_TO_CONNECTION)"
+ + " VALUES('" + name + "', 1, 1)");
+ }
+
+ // Odd IDs inputs have values
+ for (String ci : new String[]{"1", "2", "3", "4"}) {
+ for (String i : new String[]{"5", "9", "17"}) {
+ runQuery("INSERT INTO SQOOP.SQ_JOB_INPUT"
+ + "(SQBI_JOB, SQBI_INPUT, SQBI_VALUE) "
+ + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
+ }
+
+ for (String i : new String[]{"7", "11", "19"}) {
+ runQuery("INSERT INTO SQOOP.SQ_JOB_INPUT"
+ + "(SQBI_JOB, SQBI_INPUT, SQBI_VALUE) "
+ + "VALUES(" + ci + ", " + i + ", 'Value" + i + "')");
+ }
+ }
+ break;
+
+ default:
+ throw new AssertionError("Invalid connector and framework version: " + version);
+ }
+ }
+
+ public void loadJobs() throws Exception {
+ loadJobs(SYSTEM_VERSION);
+ }
+
+ /**
+ * Add a second connector for testing with multiple connectors
+ */
+ public void addConnector() throws Exception {
+ // Connector entry
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ + "VALUES('B', 'org.apache.sqoop.test.B', '1.0-test')");
+ }
+
+ /**
+ * Load testing submissions into the metadata repository.
+ *
+ * @throws Exception
+ */
+ public void loadSubmissions() throws Exception {
+ runQuery("INSERT INTO SQOOP.SQ_COUNTER_GROUP "
+ + "(SQG_NAME) "
+ + "VALUES"
+ + "('gA'), ('gB')"
+ );
+
+ runQuery("INSERT INTO SQOOP.SQ_COUNTER "
+ + "(SQR_NAME) "
+ + "VALUES"
+ + "('cA'), ('cB')"
+ );
+
+ runQuery("INSERT INTO SQOOP.SQ_SUBMISSION"
+ + "(SQS_JOB, SQS_STATUS, SQS_CREATION_DATE, SQS_UPDATE_DATE,"
+ + " SQS_EXTERNAL_ID, SQS_EXTERNAL_LINK, SQS_EXCEPTION,"
+ + " SQS_EXCEPTION_TRACE)"
+ + "VALUES "
+ + "(1, 'RUNNING', '2012-01-01 01:01:01', '2012-01-01 01:01:01', 'job_1',"
+ + "NULL, NULL, NULL),"
+ + "(2, 'SUCCEEDED', '2012-01-01 01:01:01', '2012-01-02 01:01:01', 'job_2',"
+ + " NULL, NULL, NULL),"
+ + "(3, 'FAILED', '2012-01-01 01:01:01', '2012-01-03 01:01:01', 'job_3',"
+ + " NULL, NULL, NULL),"
+ + "(4, 'UNKNOWN', '2012-01-01 01:01:01', '2012-01-04 01:01:01', 'job_4',"
+ + " NULL, NULL, NULL),"
+ + "(1, 'RUNNING', '2012-01-01 01:01:01', '2012-01-05 01:01:01', 'job_5',"
+ + " NULL, NULL, NULL)"
+ );
+
+ runQuery("INSERT INTO SQOOP.SQ_COUNTER_SUBMISSION "
+ + "(SQRS_GROUP, SQRS_COUNTER, SQRS_SUBMISSION, SQRS_VALUE) "
+ + "VALUES"
+ + "(1, 1, 4, 300)"
+ );
+
+ }
+
+ protected MConnector getConnector() {
+ return new MConnector("A", "org.apache.sqoop.test.A", "1.0-test",
+ getConnectionForms(), new MJobForms(getForms()), new MJobForms(getForms()));
+ }
+
+ protected MFramework getFramework() {
+ return new MFramework(getConnectionForms(), new MJobForms(getForms()),
+ FrameworkManager.CURRENT_FRAMEWORK_VERSION);
+ }
+
+ protected void fillConnection(MConnection connection) {
+ List<MForm> forms;
+
+ forms = connection.getConnectorPart().getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
+
+ forms = connection.getFrameworkPart().getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value13");
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value15");
+ }
+
+ protected void fillJob(MJob job) {
+ List<MForm> forms;
+
+ forms = job.getConnectorPart(Direction.FROM).getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
+
+ forms = job.getConnectorPart(Direction.TO).getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value1");
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value2");
+
+ forms = job.getFrameworkPart().getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Value13");
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Value15");
+ }
+
+ protected MConnectionForms getConnectionForms() {
+ return new MConnectionForms(getForms());
+ }
+
+ protected List<MForm> getForms() {
+ List<MForm> forms = new LinkedList<MForm>();
+
+ List<MInput<?>> inputs;
+ MInput input;
+
+ inputs = new LinkedList<MInput<?>>();
+ input = new MStringInput("I1", false, (short)30);
+ inputs.add(input);
+ input = new MMapInput("I2", false);
+ inputs.add(input);
+ forms.add(new MForm("F1", inputs));
+
+ inputs = new LinkedList<MInput<?>>();
+ input = new MStringInput("I3", false, (short)30);
+ inputs.add(input);
+ input = new MMapInput("I4", false);
+ inputs.add(input);
+ forms.add(new MForm("F2", inputs));
+
+ return forms;
+ }
+
+ /**
+ * Find out number of entries in given table.
+ *
+ * @param table Table name
+ * @return Number of rows in the table
+ * @throws Exception
+ */
+ protected long countForTable(String table) throws Exception {
+ Statement stmt = null;
+ ResultSet rs = null;
+
+ try {
+ stmt = getDerbyConnection().createStatement();
+
+ rs = stmt.executeQuery("SELECT COUNT(*) FROM "+ table);
+ rs.next();
+
+ return rs.getLong(1);
+ } finally {
+ if(stmt != null) {
+ stmt.close();
+ }
+ if(rs != null) {
+ rs.close();
+ }
+ }
+ }
+
+ /**
+ * Assert row count for given table.
+ *
+ * @param table Table name
+ * @param expected Expected number of rows
+ * @throws Exception
+ */
+ protected void assertCountForTable(String table, long expected)
+ throws Exception {
+ long count = countForTable(table);
+ assertEquals(expected, count);
+ }
+
+ /**
+ * Printout repository content for advance debugging.
+ *
+ * This method is currently unused, but might be helpful in the future, so
+ * I'm letting it here.
+ *
+ * @throws Exception
+ */
+ protected void generateDatabaseState() throws Exception {
+ for(String tbl : new String[] {"SQ_CONNECTOR", "SQ_FORM", "SQ_INPUT",
+ "SQ_CONNECTION", "SQ_CONNECTION_INPUT", "SQ_JOB", "SQ_JOB_INPUT"}) {
+ generateTableState("SQOOP." + tbl);
+ }
+ }
+
+ /**
+ * Printout one single table.
+ *
+ * @param table Table name
+ * @throws Exception
+ */
+ protected void generateTableState(String table) throws Exception {
+ PreparedStatement ps = null;
+ ResultSet rs = null;
+ ResultSetMetaData rsmt = null;
+
+ try {
+ ps = getDerbyConnection().prepareStatement("SELECT * FROM " + table);
+ rs = ps.executeQuery();
+
+ rsmt = rs.getMetaData();
+
+ StringBuilder sb = new StringBuilder();
+ System.out.println("Table " + table + ":");
+
+ for(int i = 1; i <= rsmt.getColumnCount(); i++) {
+ sb.append("| ").append(rsmt.getColumnName(i)).append(" ");
+ }
+ sb.append("|");
+ System.out.println(sb.toString());
+
+ while(rs.next()) {
+ sb = new StringBuilder();
+ for(int i = 1; i <= rsmt.getColumnCount(); i++) {
+ sb.append("| ").append(rs.getString(i)).append(" ");
+ }
+ sb.append("|");
+ System.out.println(sb.toString());
+ }
+
+ System.out.println("");
+
+ } finally {
+ if(rs != null) {
+ rs.close();
+ }
+ if(ps != null) {
+ ps.close();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java
index bdd3c05..f9e9217 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectionHandling.java
@@ -33,213 +33,213 @@ import java.util.Map;
*/
public class TestConnectionHandling extends DerbyTestCase {
-// DerbyRepositoryHandler handler;
-//
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// handler = new DerbyRepositoryHandler();
-//
-// // We always needs schema for this test case
-// createSchema();
-//
-// // We always needs connector and framework structures in place
-// loadConnectorAndFramework();
-// }
-//
-// public void testFindConnection() throws Exception {
-// // Let's try to find non existing connection
-// try {
-// handler.findConnection(1, getDerbyConnection());
-// fail();
-// } catch(SqoopException ex) {
-// assertEquals(DerbyRepoError.DERBYREPO_0024, ex.getErrorCode());
-// }
-//
-// // Load prepared connections into database
-// loadConnections();
-//
-// MConnection connA = handler.findConnection(1, getDerbyConnection());
-// assertNotNull(connA);
-// assertEquals(1, connA.getPersistenceId());
-// assertEquals("CA", connA.getName());
-//
-// List<MForm> forms;
-//
-// // Check connector part
-// forms = connA.getConnectorPart().getForms();
-// assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Value3", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-//
-// // Check framework part
-// forms = connA.getFrameworkPart().getForms();
-// assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-// }
-//
-// public void testFindConnections() throws Exception {
-// List<MConnection> list;
-//
-// // Load empty list on empty repository
-// list = handler.findConnections(getDerbyConnection());
-// assertEquals(0, list.size());
-//
-// loadConnections();
-//
-// // Load all two connections on loaded repository
-// list = handler.findConnections(getDerbyConnection());
-// assertEquals(2, list.size());
-//
-// assertEquals("CA", list.get(0).getName());
-// assertEquals("CB", list.get(1).getName());
-// }
-//
-// public void testExistsConnection() throws Exception {
-// // There shouldn't be anything on empty repository
-// assertFalse(handler.existsConnection(1, getDerbyConnection()));
-// assertFalse(handler.existsConnection(2, getDerbyConnection()));
-// assertFalse(handler.existsConnection(3, getDerbyConnection()));
-//
-// loadConnections();
-//
-// assertTrue(handler.existsConnection(1, getDerbyConnection()));
-// assertTrue(handler.existsConnection(2, getDerbyConnection()));
-// assertFalse(handler.existsConnection(3, getDerbyConnection()));
-// }
-//
-// public void testCreateConnection() throws Exception {
-// MConnection connection = getConnection();
-//
-// // Load some data
-// fillConnection(connection);
-//
-// handler.createConnection(connection, getDerbyConnection());
-//
-// assertEquals(1, connection.getPersistenceId());
-// assertCountForTable("SQOOP.SQ_CONNECTION", 1);
-// assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
-//
-// MConnection retrieved = handler.findConnection(1, getDerbyConnection());
-// assertEquals(1, retrieved.getPersistenceId());
-//
-// List<MForm> forms;
-// forms = connection.getConnectorPart().getForms();
-// assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Value2", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-//
-// forms = connection.getFrameworkPart().getForms();
-// assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-//
-// // Let's create second connection
-// connection = getConnection();
-// fillConnection(connection);
-//
-// handler.createConnection(connection, getDerbyConnection());
-//
-// assertEquals(2, connection.getPersistenceId());
-// assertCountForTable("SQOOP.SQ_CONNECTION", 2);
-// assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 8);
-// }
-//
-// public void testInUseConnection() throws Exception {
-// loadConnections();
-//
-// assertFalse(handler.inUseConnection(1, getDerbyConnection()));
-//
-// loadJobs();
-//
-// assertTrue(handler.inUseConnection(1, getDerbyConnection()));
-// }
-//
-// public void testUpdateConnection() throws Exception {
-// loadConnections();
-//
-// MConnection connection = handler.findConnection(1, getDerbyConnection());
-//
-// List<MForm> forms;
-//
-// forms = connection.getConnectorPart().getForms();
-// ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
-// ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
-// ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
-// ((MMapInput)forms.get(1).getInputs().get(1)).setValue(null);
-//
-// forms = connection.getFrameworkPart().getForms();
-// ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
-// ((MMapInput)forms.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
-// ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
-// ((MMapInput)forms.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
-//
-// connection.setName("name");
-//
-// handler.updateConnection(connection, getDerbyConnection());
-//
-// assertEquals(1, connection.getPersistenceId());
-// assertCountForTable("SQOOP.SQ_CONNECTION", 2);
-// assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 10);
-//
-// MConnection retrieved = handler.findConnection(1, getDerbyConnection());
-// assertEquals("name", connection.getName());
-//
-// forms = retrieved.getConnectorPart().getForms();
-// assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
-// assertNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
-// assertNull(forms.get(1).getInputs().get(1).getValue());
-//
-// forms = retrieved.getFrameworkPart().getForms();
-// assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
-// assertNotNull(forms.get(0).getInputs().get(1).getValue());
-// assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
-// assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
-// assertNotNull(forms.get(1).getInputs().get(1).getValue());
-// assertEquals(((Map)forms.get(1).getInputs().get(1).getValue()).size(), 0);
-// }
-//
-// public void testEnableAndDisableConnection() throws Exception {
-// loadConnections();
-//
-// // disable connection 1
-// handler.enableConnection(1, false, getDerbyConnection());
-//
-// MConnection retrieved = handler.findConnection(1, getDerbyConnection());
-// assertNotNull(retrieved);
-// assertEquals(false, retrieved.getEnabled());
-//
-// // enable connection 1
-// handler.enableConnection(1, true, getDerbyConnection());
-//
-// retrieved = handler.findConnection(1, getDerbyConnection());
-// assertNotNull(retrieved);
-// assertEquals(true, retrieved.getEnabled());
-// }
-//
-// public void testDeleteConnection() throws Exception {
-// loadConnections();
-//
-// handler.deleteConnection(1, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_CONNECTION", 1);
-// assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
-//
-// handler.deleteConnection(2, getDerbyConnection());
-// assertCountForTable("SQOOP.SQ_CONNECTION", 0);
-// assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 0);
-// }
-//
-// public MConnection getConnection() {
-// return new MConnection(1,
-// handler.findConnector("A", getDerbyConnection()).getConnectionForms(),
-// handler.findFramework(getDerbyConnection()).getConnectionForms()
-// );
-// }
+ DerbyRepositoryHandler handler;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+
+ // We always needs connector and framework structures in place
+ loadConnectorAndFramework();
+ }
+
+ public void testFindConnection() throws Exception {
+ // Let's try to find non existing connection
+ try {
+ handler.findConnection(1, getDerbyConnection());
+ fail();
+ } catch(SqoopException ex) {
+ assertEquals(DerbyRepoError.DERBYREPO_0024, ex.getErrorCode());
+ }
+
+ // Load prepared connections into database
+ loadConnections();
+
+ MConnection connA = handler.findConnection(1, getDerbyConnection());
+ assertNotNull(connA);
+ assertEquals(1, connA.getPersistenceId());
+ assertEquals("CA", connA.getName());
+
+ List<MForm> forms;
+
+ // Check connector part
+ forms = connA.getConnectorPart().getForms();
+ assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value3", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ // Check framework part
+ forms = connA.getFrameworkPart().getForms();
+ assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+ }
+
+ public void testFindConnections() throws Exception {
+ List<MConnection> list;
+
+ // Load empty list on empty repository
+ list = handler.findConnections(getDerbyConnection());
+ assertEquals(0, list.size());
+
+ loadConnections();
+
+ // Load all two connections on loaded repository
+ list = handler.findConnections(getDerbyConnection());
+ assertEquals(2, list.size());
+
+ assertEquals("CA", list.get(0).getName());
+ assertEquals("CB", list.get(1).getName());
+ }
+
+ public void testExistsConnection() throws Exception {
+ // There shouldn't be anything on empty repository
+ assertFalse(handler.existsConnection(1, getDerbyConnection()));
+ assertFalse(handler.existsConnection(2, getDerbyConnection()));
+ assertFalse(handler.existsConnection(3, getDerbyConnection()));
+
+ loadConnections();
+
+ assertTrue(handler.existsConnection(1, getDerbyConnection()));
+ assertTrue(handler.existsConnection(2, getDerbyConnection()));
+ assertFalse(handler.existsConnection(3, getDerbyConnection()));
+ }
+
+ public void testCreateConnection() throws Exception {
+ MConnection connection = getConnection();
+
+ // Load some data
+ fillConnection(connection);
+
+ handler.createConnection(connection, getDerbyConnection());
+
+ assertEquals(1, connection.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 1);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
+
+ MConnection retrieved = handler.findConnection(1, getDerbyConnection());
+ assertEquals(1, retrieved.getPersistenceId());
+
+ List<MForm> forms;
+ forms = connection.getConnectorPart().getForms();
+ assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value2", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ forms = connection.getFrameworkPart().getForms();
+ assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ // Let's create second connection
+ connection = getConnection();
+ fillConnection(connection);
+
+ handler.createConnection(connection, getDerbyConnection());
+
+ assertEquals(2, connection.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 2);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 8);
+ }
+
+ public void testInUseConnection() throws Exception {
+ loadConnections();
+
+ assertFalse(handler.inUseConnection(1, getDerbyConnection()));
+
+ loadJobs();
+
+ assertTrue(handler.inUseConnection(1, getDerbyConnection()));
+ }
+
+ public void testUpdateConnection() throws Exception {
+ loadConnections();
+
+ MConnection connection = handler.findConnection(1, getDerbyConnection());
+
+ List<MForm> forms;
+
+ forms = connection.getConnectorPart().getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(1).getInputs().get(1)).setValue(null);
+
+ forms = connection.getFrameworkPart().getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+
+ connection.setName("name");
+
+ handler.updateConnection(connection, getDerbyConnection());
+
+ assertEquals(1, connection.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 2);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 10);
+
+ MConnection retrieved = handler.findConnection(1, getDerbyConnection());
+ assertEquals("name", connection.getName());
+
+ forms = retrieved.getConnectorPart().getForms();
+ assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ forms = retrieved.getFrameworkPart().getForms();
+ assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
+ assertNotNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
+ assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
+ assertNotNull(forms.get(1).getInputs().get(1).getValue());
+ assertEquals(((Map)forms.get(1).getInputs().get(1).getValue()).size(), 0);
+ }
+
+ public void testEnableAndDisableConnection() throws Exception {
+ loadConnections();
+
+ // disable connection 1
+ handler.enableConnection(1, false, getDerbyConnection());
+
+ MConnection retrieved = handler.findConnection(1, getDerbyConnection());
+ assertNotNull(retrieved);
+ assertEquals(false, retrieved.getEnabled());
+
+ // enable connection 1
+ handler.enableConnection(1, true, getDerbyConnection());
+
+ retrieved = handler.findConnection(1, getDerbyConnection());
+ assertNotNull(retrieved);
+ assertEquals(true, retrieved.getEnabled());
+ }
+
+ public void testDeleteConnection() throws Exception {
+ loadConnections();
+
+ handler.deleteConnection(1, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 1);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
+
+ handler.deleteConnection(2, getDerbyConnection());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 0);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 0);
+ }
+
+ public MConnection getConnection() {
+ return new MConnection(1,
+ handler.findConnector("A", getDerbyConnection()).getConnectionForms(),
+ handler.findFramework(getDerbyConnection()).getConnectionForms()
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
index 54ae726..745e128 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestConnectorHandling.java
@@ -26,70 +26,70 @@ import java.util.List;
*/
public class TestConnectorHandling extends DerbyTestCase {
-// DerbyRepositoryHandler handler;
-//
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// handler = new DerbyRepositoryHandler();
-//
-// // We always needs schema for this test case
-// createSchema();
-// }
-//
-// public void testFindConnector() throws Exception {
-// // On empty repository, no connectors should be there
-// assertNull(handler.findConnector("A", getDerbyConnection()));
-// assertNull(handler.findConnector("B", getDerbyConnection()));
-//
-// // Load connector into repository
-// loadConnectorAndFramework();
-//
-// // Retrieve it
-// MConnector connector = handler.findConnector("A", getDerbyConnection());
-// assertNotNull(connector);
-//
-// // Get original structure
-// MConnector original = getConnector();
-//
-// // And compare them
-// assertEquals(original, connector);
-// }
-//
-// public void testFindAllConnectors() throws Exception {
-// // No connectors in an empty repository, we expect an empty list
-// assertEquals(handler.findConnectors(getDerbyConnection()).size(),0);
-//
-// loadConnectorAndFramework();
-// addConnector();
-//
-// // Retrieve connectors
-// List<MConnector> connectors = handler.findConnectors(getDerbyConnection());
-// assertNotNull(connectors);
-// assertEquals(connectors.size(),2);
-// assertEquals(connectors.get(0).getUniqueName(),"A");
-// assertEquals(connectors.get(1).getUniqueName(),"B");
-//
-//
-// }
-//
-// public void testRegisterConnector() throws Exception {
-// MConnector connector = getConnector();
-//
-// handler.registerConnector(connector, getDerbyConnection());
-//
-// // Connector should get persistence ID
-// assertEquals(1, connector.getPersistenceId());
-//
-// // Now check content in corresponding tables
-// assertCountForTable("SQOOP.SQ_CONNECTOR", 1);
-// assertCountForTable("SQOOP.SQ_FORM", 6);
-// assertCountForTable("SQOOP.SQ_INPUT", 12);
-//
-// // Registered connector should be easily recovered back
-// MConnector retrieved = handler.findConnector("A", getDerbyConnection());
-// assertNotNull(retrieved);
-// assertEquals(connector, retrieved);
-// }
+ DerbyRepositoryHandler handler;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+ }
+
+ public void testFindConnector() throws Exception {
+ // On empty repository, no connectors should be there
+ assertNull(handler.findConnector("A", getDerbyConnection()));
+ assertNull(handler.findConnector("B", getDerbyConnection()));
+
+ // Load connector into repository
+ loadConnectorAndFramework();
+
+ // Retrieve it
+ MConnector connector = handler.findConnector("A", getDerbyConnection());
+ assertNotNull(connector);
+
+ // Get original structure
+ MConnector original = getConnector();
+
+ // And compare them
+ assertEquals(original, connector);
+ }
+
+ public void testFindAllConnectors() throws Exception {
+ // No connectors in an empty repository, we expect an empty list
+ assertEquals(handler.findConnectors(getDerbyConnection()).size(),0);
+
+ loadConnectorAndFramework();
+ addConnector();
+
+ // Retrieve connectors
+ List<MConnector> connectors = handler.findConnectors(getDerbyConnection());
+ assertNotNull(connectors);
+ assertEquals(connectors.size(),2);
+ assertEquals(connectors.get(0).getUniqueName(),"A");
+ assertEquals(connectors.get(1).getUniqueName(),"B");
+
+
+ }
+
+ public void testRegisterConnector() throws Exception {
+ MConnector connector = getConnector();
+
+ handler.registerConnector(connector, getDerbyConnection());
+
+ // Connector should get persistence ID
+ assertEquals(1, connector.getPersistenceId());
+
+ // Now check content in corresponding tables
+ assertCountForTable("SQOOP.SQ_CONNECTOR", 1);
+ assertCountForTable("SQOOP.SQ_FORM", 6);
+ assertCountForTable("SQOOP.SQ_INPUT", 12);
+
+ // Registered connector should be easily recovered back
+ MConnector retrieved = handler.findConnector("A", getDerbyConnection());
+ assertNotNull(retrieved);
+ assertEquals(connector, retrieved);
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java
index 8b3326d..006ec9c 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestFrameworkHandling.java
@@ -29,102 +29,101 @@ import java.sql.SQLException;
*/
public class TestFrameworkHandling extends DerbyTestCase {
-// DerbyRepositoryHandler handler;
-//
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// handler = new DerbyRepositoryHandler();
-//
-// // We always needs schema for this test case
-// createSchema();
-// }
-//
-// public void testFindFramework() throws Exception {
-// // On empty repository, no framework should be there
-// assertNull(handler.findFramework(getDerbyConnection()));
-//
-// // Load framework into repository
-// loadConnectorAndFramework();
-//
-// // Retrieve it
-// MFramework framework = handler.findFramework(getDerbyConnection());
-// assertNotNull(framework);
-//
-// // Get original structure
-// MFramework original = getFramework();
-//
-// // And compare them
-// assertEquals(original, framework);
-// }
-//
-// public void testRegisterConnector() throws Exception {
-// MFramework framework = getFramework();
-//
-// handler.registerFramework(framework, getDerbyConnection());
-//
-// // Connector should get persistence ID
-// assertEquals(1, framework.getPersistenceId());
-//
-// // Now check content in corresponding tables
-// assertCountForTable("SQOOP.SQ_CONNECTOR", 0);
-// assertCountForTable("SQOOP.SQ_FORM", 6);
-// assertCountForTable("SQOOP.SQ_INPUT", 12);
-//
-// // Registered framework should be easily recovered back
-// MFramework retrieved = handler.findFramework(getDerbyConnection());
-// assertNotNull(retrieved);
-// assertEquals(framework, retrieved);
-// assertEquals(framework.getVersion(), retrieved.getVersion());
-// }
-//
-// private String getFrameworkVersion() throws Exception {
-// final String frameworkVersionQuery =
-// "SELECT SQM_VALUE FROM SQOOP.SQ_SYSTEM WHERE SQM_KEY=?";
-// String retVal = null;
-// PreparedStatement preparedStmt = null;
-// ResultSet resultSet = null;
-// try {
-// preparedStmt =
-// getDerbyConnection().prepareStatement(frameworkVersionQuery);
-// preparedStmt.setString(1, DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION);
-// resultSet = preparedStmt.executeQuery();
-// if(resultSet.next())
-// retVal = resultSet.getString(1);
-// return retVal;
-// } finally {
-// if(preparedStmt !=null) {
-// try {
-// preparedStmt.close();
-// } catch(SQLException e) {
-// }
-// }
-// if(resultSet != null) {
-// try {
-// resultSet.close();
-// } catch(SQLException e) {
-// }
-// }
-// }
-// }
-//
-// public void testFrameworkVersion() throws Exception {
-// handler.registerFramework(getFramework(), getDerbyConnection());
-//
-// final String lowerVersion = Integer.toString(
-// Integer.parseInt(FrameworkManager.CURRENT_FRAMEWORK_VERSION) - 1);
-// assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, getFrameworkVersion());
-// runQuery("UPDATE SQOOP.SQ_SYSTEM SET SQM_VALUE='" + lowerVersion +
-// "' WHERE SQM_KEY = '" + DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION + "'");
-// assertEquals(lowerVersion, getFrameworkVersion());
-//
-// MFramework framework = getFramework();
-// handler.updateFramework(framework, getDerbyConnection());
-//
-// assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, framework.getVersion());
-//
-// assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, getFrameworkVersion());
-// }
+ DerbyRepositoryHandler handler;
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+ }
+
+ public void testFindFramework() throws Exception {
+ // On empty repository, no framework should be there
+ assertNull(handler.findFramework(getDerbyConnection()));
+
+ // Load framework into repository
+ loadConnectorAndFramework();
+
+ // Retrieve it
+ MFramework framework = handler.findFramework(getDerbyConnection());
+ assertNotNull(framework);
+
+ // Get original structure
+ MFramework original = getFramework();
+
+ // And compare them
+ assertEquals(original, framework);
+ }
+
+ public void testRegisterConnector() throws Exception {
+ MFramework framework = getFramework();
+
+ handler.registerFramework(framework, getDerbyConnection());
+
+ // Connector should get persistence ID
+ assertEquals(1, framework.getPersistenceId());
+
+ // Now check content in corresponding tables
+ assertCountForTable("SQOOP.SQ_CONNECTOR", 0);
+ assertCountForTable("SQOOP.SQ_FORM", 4);
+ assertCountForTable("SQOOP.SQ_INPUT", 8);
+
+ // Registered framework should be easily recovered back
+ MFramework retrieved = handler.findFramework(getDerbyConnection());
+ assertNotNull(retrieved);
+ assertEquals(framework, retrieved);
+ assertEquals(framework.getVersion(), retrieved.getVersion());
+ }
+
+ private String getFrameworkVersion() throws Exception {
+ final String frameworkVersionQuery =
+ "SELECT SQM_VALUE FROM SQOOP.SQ_SYSTEM WHERE SQM_KEY=?";
+ String retVal = null;
+ PreparedStatement preparedStmt = null;
+ ResultSet resultSet = null;
+ try {
+ preparedStmt =
+ getDerbyConnection().prepareStatement(frameworkVersionQuery);
+ preparedStmt.setString(1, DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION);
+ resultSet = preparedStmt.executeQuery();
+ if(resultSet.next())
+ retVal = resultSet.getString(1);
+ return retVal;
+ } finally {
+ if(preparedStmt !=null) {
+ try {
+ preparedStmt.close();
+ } catch(SQLException e) {
+ }
+ }
+ if(resultSet != null) {
+ try {
+ resultSet.close();
+ } catch(SQLException e) {
+ }
+ }
+ }
+ }
+
+ public void testFrameworkVersion() throws Exception {
+ handler.registerFramework(getFramework(), getDerbyConnection());
+
+ final String lowerVersion = Integer.toString(
+ Integer.parseInt(FrameworkManager.CURRENT_FRAMEWORK_VERSION) - 1);
+ assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, getFrameworkVersion());
+ runQuery("UPDATE SQOOP.SQ_SYSTEM SET SQM_VALUE='" + lowerVersion +
+ "' WHERE SQM_KEY = '" + DerbyRepoConstants.SYSKEY_FRAMEWORK_VERSION + "'");
+ assertEquals(lowerVersion, getFrameworkVersion());
+
+ MFramework framework = getFramework();
+ handler.updateFramework(framework, getDerbyConnection());
+
+ assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, framework.getVersion());
+
+ assertEquals(FrameworkManager.CURRENT_FRAMEWORK_VERSION, getFrameworkVersion());
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
index 5d3807d..15f9539 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
@@ -40,107 +40,107 @@ import java.util.Map;
*/
public class TestInputTypes extends DerbyTestCase {
-// DerbyRepositoryHandler handler;
-//
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// handler = new DerbyRepositoryHandler();
-//
-// // We always needs schema for this test case
-// createSchema();
-// }
-//
-// /**
-// * Ensure that metadata with all various data types can be successfully
-// * serialized into repository and retrieved back.
-// */
-// public void testMetadataSerialization() throws Exception {
-// MConnector connector = getConnector();
-//
-// // Serialize the connector with all data types into repository
-// handler.registerConnector(connector, getDerbyConnection());
-//
-// // Successful serialization should update the ID
-// assertNotSame(connector.getPersistenceId(), MPersistableEntity.PERSISTANCE_ID_DEFAULT);
-//
-// // Retrieve registered connector
-// MConnector retrieved = handler.findConnector(connector.getUniqueName(), getDerbyConnection());
-// assertNotNull(retrieved);
-//
-// // Original and retrieved connectors should be the same
-// assertEquals(connector, retrieved);
-// }
-//
-// /**
-// * Test that serializing actual data is not an issue.
-// */
-// public void testDataSerialization() throws Exception {
-// MConnector connector = getConnector();
-// MFramework framework = getFramework();
-//
-// // Register metadata for everything and our new connector
-// handler.registerConnector(connector, getDerbyConnection());
-// handler.registerFramework(framework, getDerbyConnection());
-//
-// // Inserted values
-// Map<String, String> map = new HashMap<String, String>();
-// map.put("A", "B");
-//
-// // Connection object with all various values
-// MConnection connection = new MConnection(connector.getPersistenceId(), connector.getConnectionForms(), framework.getConnectionForms());
-// MConnectionForms forms = connection.getConnectorPart();
-// forms.getStringInput("f.String").setValue("A");
-// forms.getMapInput("f.Map").setValue(map);
-// forms.getIntegerInput("f.Integer").setValue(1);
-// forms.getBooleanInput("f.Boolean").setValue(true);
-// forms.getEnumInput("f.Enum").setValue("YES");
-//
-// // Create the connection in repository
-// handler.createConnection(connection, getDerbyConnection());
-// assertNotSame(connection.getPersistenceId(), MPersistableEntity.PERSISTANCE_ID_DEFAULT);
-//
-// // Retrieve created connection
-// MConnection retrieved = handler.findConnection(connection.getPersistenceId(), getDerbyConnection());
-// forms = retrieved.getConnectorPart();
-// assertEquals("A", forms.getStringInput("f.String").getValue());
-// assertEquals(map, forms.getMapInput("f.Map").getValue());
-// assertEquals(1, (int)forms.getIntegerInput("f.Integer").getValue());
-// assertEquals(true, (boolean)forms.getBooleanInput("f.Boolean").getValue());
-// assertEquals("YES", forms.getEnumInput("f.Enum").getValue());
-// }
-//
-// /**
-// * Overriding parent method to get forms with all supported data types.
-// *
-// * @return Forms with all data types
-// */
-// @Override
-// protected List<MForm> getForms() {
-// List<MForm> forms = new LinkedList<MForm>();
-//
-// List<MInput<?>> inputs;
-// MInput input;
-//
-// inputs = new LinkedList<MInput<?>>();
-//
-// input = new MStringInput("f.String", false, (short)30);
-// inputs.add(input);
-//
-// input = new MMapInput("f.Map", false);
-// inputs.add(input);
-//
-// input = new MIntegerInput("f.Integer", false);
-// inputs.add(input);
-//
-// input = new MBooleanInput("f.Boolean", false);
-// inputs.add(input);
-//
-// input = new MEnumInput("f.Enum", false, new String[] {"YES", "NO"});
-// inputs.add(input);
-//
-// forms.add(new MForm("f", inputs));
-// return forms;
-// }
+ DerbyRepositoryHandler handler;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+ }
+
+ /**
+ * Ensure that metadata with all various data types can be successfully
+ * serialized into repository and retrieved back.
+ */
+ public void testMetadataSerialization() throws Exception {
+ MConnector connector = getConnector();
+
+ // Serialize the connector with all data types into repository
+ handler.registerConnector(connector, getDerbyConnection());
+
+ // Successful serialization should update the ID
+ assertNotSame(connector.getPersistenceId(), MPersistableEntity.PERSISTANCE_ID_DEFAULT);
+
+ // Retrieve registered connector
+ MConnector retrieved = handler.findConnector(connector.getUniqueName(), getDerbyConnection());
+ assertNotNull(retrieved);
+
+ // Original and retrieved connectors should be the same
+ assertEquals(connector, retrieved);
+ }
+
+ /**
+ * Test that serializing actual data is not an issue.
+ */
+ public void testDataSerialization() throws Exception {
+ MConnector connector = getConnector();
+ MFramework framework = getFramework();
+
+ // Register metadata for everything and our new connector
+ handler.registerConnector(connector, getDerbyConnection());
+ handler.registerFramework(framework, getDerbyConnection());
+
+ // Inserted values
+ Map<String, String> map = new HashMap<String, String>();
+ map.put("A", "B");
+
+ // Connection object with all various values
+ MConnection connection = new MConnection(connector.getPersistenceId(), connector.getConnectionForms(), framework.getConnectionForms());
+ MConnectionForms forms = connection.getConnectorPart();
+ forms.getStringInput("f.String").setValue("A");
+ forms.getMapInput("f.Map").setValue(map);
+ forms.getIntegerInput("f.Integer").setValue(1);
+ forms.getBooleanInput("f.Boolean").setValue(true);
+ forms.getEnumInput("f.Enum").setValue("YES");
+
+ // Create the connection in repository
+ handler.createConnection(connection, getDerbyConnection());
+ assertNotSame(connection.getPersistenceId(), MPersistableEntity.PERSISTANCE_ID_DEFAULT);
+
+ // Retrieve created connection
+ MConnection retrieved = handler.findConnection(connection.getPersistenceId(), getDerbyConnection());
+ forms = retrieved.getConnectorPart();
+ assertEquals("A", forms.getStringInput("f.String").getValue());
+ assertEquals(map, forms.getMapInput("f.Map").getValue());
+ assertEquals(1, (int)forms.getIntegerInput("f.Integer").getValue());
+ assertEquals(true, (boolean)forms.getBooleanInput("f.Boolean").getValue());
+ assertEquals("YES", forms.getEnumInput("f.Enum").getValue());
+ }
+
+ /**
+ * Overriding parent method to get forms with all supported data types.
+ *
+ * @return Forms with all data types
+ */
+ @Override
+ protected List<MForm> getForms() {
+ List<MForm> forms = new LinkedList<MForm>();
+
+ List<MInput<?>> inputs;
+ MInput input;
+
+ inputs = new LinkedList<MInput<?>>();
+
+ input = new MStringInput("f.String", false, (short)30);
+ inputs.add(input);
+
+ input = new MMapInput("f.Map", false);
+ inputs.add(input);
+
+ input = new MIntegerInput("f.Integer", false);
+ inputs.add(input);
+
+ input = new MBooleanInput("f.Boolean", false);
+ inputs.add(input);
+
+ input = new MEnumInput("f.Enum", false, new String[] {"YES", "NO"});
+ inputs.add(input);
+
+ forms.add(new MForm("f", inputs));
+ return forms;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
index 0d93348..913439b 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInternals.java
@@ -17,31 +17,53 @@
*/
package org.apache.sqoop.repository.derby;
+import java.sql.Connection;
+
/**
*
*/
public class TestInternals extends DerbyTestCase {
-// DerbyRepositoryHandler handler;
-//
-// @Override
-// public void setUp() throws Exception {
-// super.setUp();
-//
-// handler = new DerbyRepositoryHandler();
-// }
-//
-// public void testSuitableInternals() throws Exception {
-// assertFalse(handler.haveSuitableInternals(getDerbyConnection()));
-// createSchema(); // Test code is building the structures
-// assertTrue(handler.haveSuitableInternals(getDerbyConnection()));
-// }
-//
-// public void testCreateorUpdateInternals() throws Exception {
-// assertFalse(handler.haveSuitableInternals(getDerbyConnection()));
-// handler.createOrUpdateInternals(getDerbyConnection());
-// assertTrue(handler.haveSuitableInternals(getDerbyConnection()));
-// }
+ DerbyRepositoryHandler handler;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new TestDerbyRepositoryHandler();
+ }
+
+ public void testSuitableInternals() throws Exception {
+ assertFalse(handler.haveSuitableInternals(getDerbyConnection()));
+ createSchema(); // Test code is building the structures
+ assertTrue(handler.haveSuitableInternals(getDerbyConnection()));
+ }
+
+ public void testCreateorUpdateInternals() throws Exception {
+ assertFalse(handler.haveSuitableInternals(getDerbyConnection()));
+ handler.createOrUpdateInternals(getDerbyConnection());
+ assertTrue(handler.haveSuitableInternals(getDerbyConnection()));
+ }
+ public void testUpgradeVersion2ToVersion4() throws Exception {
+ createSchema(2);
+ assertFalse(handler.haveSuitableInternals(getDerbyConnection()));
+ loadConnectorAndFramework(2);
+ loadConnections(2);
+ loadJobs(2);
+ handler.createOrUpdateInternals(getDerbyConnection());
+ assertTrue(handler.haveSuitableInternals(getDerbyConnection()));
+ }
+ private class TestDerbyRepositoryHandler extends DerbyRepositoryHandler {
+ protected long registerHdfsConnector(Connection conn) {
+ try {
+ runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)"
+ + "VALUES('hdfs-connector', 'org.apache.sqoop.test.B', '1.0-test')");
+ return 2L;
+ } catch(Exception e) {
+ return -1L;
+ }
+ }
+ }
}
[52/52] [abbrv] git commit: SQOOP-1574: Sqoop2: From/To: Rebase
against Sqoop2 branch
Posted by ab...@apache.org.
SQOOP-1574: Sqoop2: From/To: Rebase against Sqoop2 branch
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d7e04904
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d7e04904
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d7e04904
Branch: refs/heads/SQOOP-1367
Commit: d7e04904300ddef91cb70d65efbc16493b206815
Parents: fc8eb9b
Author: Abraham Elmahrek <ab...@elmahrek.com>
Authored: Thu Oct 9 19:06:32 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 19:39:41 2014 -0700
----------------------------------------------------------------------
.../apache/sqoop/tools/tool/JSONConstants.java | 4 +-
.../sqoop/tools/tool/RepositoryDumpTool.java | 15 +-
.../sqoop/tools/tool/RepositoryLoadTool.java | 190 +++++++++----------
3 files changed, 95 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d7e04904/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
index 288cba3..9bec0d0 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
@@ -28,9 +28,9 @@ public final class JSONConstants {
public static final String CONNECTOR_NAME = "connector-name";
public static final String ALL = "all";
public static final String NAME = "name";
- public static final String CONNECTION_ID = "connection-id";
+ public static final String LINK_ID = "link-id";
public static final String JOB_ID = "job-id";
- public static final String CONNECTIONS = "connections";
+ public static final String LINKS = "links";
public static final String JOBS = "jobs";
public static final String SUBMISSIONS = "submissions";
public static final String METADATA = "metadata";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d7e04904/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
index d41b0d2..f89c546 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
@@ -25,25 +25,21 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.ParseException;
import org.apache.log4j.Logger;
import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.json.ConnectionBean;
import org.apache.sqoop.json.JobBean;
+import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.SubmissionBean;
-import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.repository.Repository;
import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.tools.ConfiguredTool;
import org.apache.sqoop.common.VersionInfo;
-import static org.apache.sqoop.json.util.FormSerialization.ALL;
+import static org.apache.sqoop.json.util.ConfigSerialization.ALL;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
-import java.util.HashMap;
import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
/**
* Write user-created content of Sqoop repository to JSON formatted file
@@ -106,8 +102,8 @@ public class RepositoryDumpTool extends ConfiguredTool {
JSONObject result = new JSONObject();
LOG.info("Dumping Connections with skipSensitive=" + String.valueOf(skipSensitive));
- ConnectionBean connections = new ConnectionBean(repository.findConnections());
- result.put(JSONConstants.CONNECTIONS, addConnectorName(connections.extract(skipSensitive)));
+ LinkBean links = new LinkBean(repository.findLinks());
+ result.put(JSONConstants.LINKS, addConnectorName(links.extract(skipSensitive)));
LOG.info("Dumping Jobs with skipSensitive=" + String.valueOf(skipSensitive));
JobBean jobs = new JobBean(repository.findJobs());
@@ -134,7 +130,6 @@ public class RepositoryDumpTool extends ConfiguredTool {
}
private JSONObject addConnectorName(JSONObject json) {
- Repository repository = RepositoryManager.getInstance().getRepository();
ConnectorManager connectorManager = ConnectorManager.getInstance();
JSONArray results = (JSONArray) json.get(ALL);
@@ -144,7 +139,7 @@ public class RepositoryDumpTool extends ConfiguredTool {
while (iterator.hasNext()) {
JSONObject result = iterator.next();
Long connectorId = (Long) result.get(JSONConstants.CONNECTOR_ID);
- result.put(JSONConstants.CONNECTOR_NAME, connectorManager.getConnectorMetadata(connectorId).getUniqueName());
+ result.put(JSONConstants.CONNECTOR_NAME, connectorManager.getConnectorConfig(connectorId).getUniqueName());
}
return json;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d7e04904/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
index c6124da..76ebd3b 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
@@ -26,24 +26,27 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.Charsets;
import org.apache.log4j.Logger;
+import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.VersionInfo;
import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.connector.spi.MetadataUpgrader;
+import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.framework.FrameworkManager;
-import org.apache.sqoop.json.ConnectionBean;
+import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.json.JobBean;
+import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.SubmissionBean;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MDriver;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.repository.Repository;
import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.tools.ConfiguredTool;
@@ -58,9 +61,9 @@ import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.sqoop.utils.ClassUtils;
+import org.apache.sqoop.validation.ConfigValidationResult;
+import org.apache.sqoop.validation.ConfigValidationRunner;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
-import org.apache.sqoop.validation.Validator;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
@@ -139,28 +142,25 @@ public class RepositoryLoadTool extends ConfiguredTool {
ConnectorManager.getInstance().initialize();
ConnectorManager connectorManager = ConnectorManager.getInstance();
- FrameworkManager.getInstance().initialize();
- FrameworkManager frameworkManager = FrameworkManager.getInstance();
-
LOG.info("Loading Connections");
- JSONObject jsonConns = (JSONObject) repo.get(JSONConstants.CONNECTIONS);
+ JSONObject jsonConns = (JSONObject) repo.get(JSONConstants.LINKS);
if (jsonConns == null) {
- LOG.error("Malformed JSON file. Key "+ JSONConstants.CONNECTIONS + " not found.");
+ LOG.error("Malformed JSON file. Key "+ JSONConstants.LINKS + " not found.");
return false;
}
- ConnectionBean connectionBean = new ConnectionBean();
- connectionBean.restore(updateConnectorIDUsingName(jsonConns));
+ LinkBean linkBean = new LinkBean();
+ linkBean.restore(updateConnectorIDUsingName(jsonConns));
HashMap<Long,Long> connectionIds = new HashMap<Long, Long>();
- for (MConnection connection : connectionBean.getConnections()) {
- long oldId = connection.getPersistenceId();
- long newId = loadConnection(connection);
- if (newId == connection.PERSISTANCE_ID_DEFAULT) {
- LOG.error("loading connection " + connection.getName() + " with previous ID " + oldId + " failed. Aborting repository load. Check log for details.");
+ for (MLink link : linkBean.getLinks()) {
+ long oldId = link.getPersistenceId();
+ long newId = loadLink(link);
+ if (newId == link.PERSISTANCE_ID_DEFAULT) {
+ LOG.error("loading connection " + link.getName() + " with previous ID " + oldId + " failed. Aborting repository load. Check log for details.");
return false;
}
connectionIds.put(oldId,newId);
@@ -176,7 +176,7 @@ public class RepositoryLoadTool extends ConfiguredTool {
}
JobBean jobBean = new JobBean();
- jobBean.restore(updateIdUsingMap(updateConnectorIDUsingName(jsonJobs), connectionIds,JSONConstants.CONNECTION_ID));
+ jobBean.restore(updateIdUsingMap(updateConnectorIDUsingName(jsonJobs), connectionIds,JSONConstants.LINK_ID));
HashMap<Long,Long> jobIds = new HashMap<Long, Long>();
for (MJob job: jobBean.getJobs()) {
@@ -242,117 +242,103 @@ public class RepositoryLoadTool extends ConfiguredTool {
return true;
}
- private long loadConnection(MConnection connection) {
+ private long loadLink(MLink link) {
//starting by pretending we have a brand new connection
- resetPersistenceId(connection);
+ resetPersistenceId(link);
- MetadataUpgrader upgrader = FrameworkManager.getInstance().getMetadataUpgrader();
- MFramework framework = FrameworkManager.getInstance().getFramework();
+ RepositoryUpgrader upgrader = Driver.getInstance().getDriverConfigRepositoryUpgrader();
Repository repository = RepositoryManager.getInstance().getRepository();
- List<MForm> frameworkForms = framework.getConnectionForms().clone(false).getForms();
- MConnectionForms newConnectionFrameworkForms = new MConnectionForms(frameworkForms);
-
- MConnector mConnector = ConnectorManager.getInstance().getConnectorMetadata(connection.getConnectorId());
- List<MForm> connectorForms = mConnector.getConnectionForms().clone(false).getForms();
- MConnectionForms newConnectionConnectorForms = new MConnectionForms(connectorForms);
+ MConnector mConnector = ConnectorManager.getInstance().getConnectorConfig(link.getConnectorId());
+ List<MConfig> connectorConfigs = mConnector.getLinkConfig().clone(false).getConfigs();
+ MLinkConfig newLinkConfigs = new MLinkConfig(connectorConfigs);
// upgrading the forms to make sure they match the current repository
- upgrader.upgrade(connection.getFrameworkPart(), newConnectionFrameworkForms);
- upgrader.upgrade(connection.getConnectorPart(), newConnectionConnectorForms);
- MConnection newConnection = new MConnection(connection, newConnectionConnectorForms, newConnectionFrameworkForms);
+ upgrader.upgrade(link.getConnectorLinkConfig(), newLinkConfigs);
+ MLink newLink = new MLink(link, newLinkConfigs);
- // Transform form structures to objects for validations
+ // Transform config structures to objects for validations
SqoopConnector connector =
- ConnectorManager.getInstance().getConnector(connection.getConnectorId());
+ ConnectorManager.getInstance().getConnector(link.getConnectorId());
Object connectorConfig = ClassUtils.instantiate(
- connector.getConnectionConfigurationClass());
- Object frameworkConfig = ClassUtils.instantiate(
- FrameworkManager.getInstance().getConnectionConfigurationClass());
+ connector.getLinkConfigurationClass());
- FormUtils.fromForms(
- connection.getConnectorPart().getForms(), connectorConfig);
- FormUtils.fromForms(
- connection.getFrameworkPart().getForms(), frameworkConfig);
+ ConfigUtils.fromConfigs(
+ link.getConnectorLinkConfig().getConfigs(), connectorConfig);
- Validator connectorValidator = connector.getValidator();
- Validator frameworkValidator = FrameworkManager.getInstance().getValidator();
+ ConfigValidationRunner validationRunner = new ConfigValidationRunner();
+ ConfigValidationResult result = validationRunner.validate(connectorConfig);
- Validation connectorValidation =
- connectorValidator.validateConnection(connectorConfig);
- Validation frameworkValidation =
- frameworkValidator.validateConnection(frameworkConfig);
-
- Status finalStatus = Status.getWorstStatus(connectorValidation.getStatus(),
- frameworkValidation.getStatus());
+ Status finalStatus = Status.getWorstStatus(result.getStatus());
if (finalStatus.canProceed()) {
- repository.createConnection(newConnection);
+ repository.createLink(newLink);
} else {
- LOG.error("Failed to load connection:" + connection.getName());
- LOG.error("Status of connector forms:" + connectorValidation.getStatus().toString());
- LOG.error("Status of framework forms:" + frameworkValidation.getStatus().toString());
+ LOG.error("Failed to load link:" + link.getName());
+ LOG.error("Status of connector configs:" + result.getStatus().toString());
}
- return newConnection.getPersistenceId();
+ return newLink.getPersistenceId();
}
private long loadJob(MJob job) {
//starting by pretending we have a brand new job
resetPersistenceId(job);
-
- MetadataUpgrader upgrader = FrameworkManager.getInstance().getMetadataUpgrader();
- MFramework framework = FrameworkManager.getInstance().getFramework();
+ RepositoryUpgrader upgrader = Driver.getInstance().getDriverConfigRepositoryUpgrader();
+ MDriver driver = Driver.getInstance().getDriver();
Repository repository = RepositoryManager.getInstance().getRepository();
- MJob.Type jobType = job.getType();
- List<MForm> frameworkForms = framework.getJobForms(job.getType()).clone(false).getForms();
- MJobForms newJobFrameworkForms = new MJobForms(jobType,frameworkForms);
-
- MConnector mConnector = ConnectorManager.getInstance().getConnectorMetadata(job.getConnectorId());
- List<MForm> connectorForms = mConnector.getJobForms(jobType).clone(false).getForms();
- MJobForms newJobConnectorForms = new MJobForms(jobType,connectorForms);
-
- // upgrading the forms to make sure they match the current repository
- upgrader.upgrade(job.getFrameworkPart(), newJobFrameworkForms);
- upgrader.upgrade(job.getConnectorPart(), newJobConnectorForms);
- MJob newJob = new MJob(job, newJobConnectorForms, newJobFrameworkForms);
-
- // Transform form structures to objects for validations
- SqoopConnector connector =
- ConnectorManager.getInstance().getConnector(job.getConnectorId());
-
- Object connectorConfig = ClassUtils.instantiate(
- connector.getJobConfigurationClass(jobType));
- Object frameworkConfig = ClassUtils.instantiate(
- FrameworkManager.getInstance().getJobConfigurationClass(jobType));
-
- FormUtils.fromForms(
- job.getConnectorPart().getForms(), connectorConfig);
- FormUtils.fromForms(
- job.getFrameworkPart().getForms(), frameworkConfig);
-
- Validator connectorValidator = connector.getValidator();
- Validator frameworkValidator = FrameworkManager.getInstance().getValidator();
-
- Validation connectorValidation =
- connectorValidator.validateJob(jobType,connectorConfig);
- Validation frameworkValidation =
- frameworkValidator.validateJob(jobType,frameworkConfig);
-
- Status finalStatus = Status.getWorstStatus(connectorValidation.getStatus(),
- frameworkValidation.getStatus());
+ MDriverConfig driverConfigs = driver.getDriverConfig();
+ MFromConfig fromConfigs = job.getFromJobConfig();
+ MToConfig toConfigs = job.getToJobConfig();
+
+ // upgrading the configs to make sure they match the current repository
+ upgrader.upgrade(job.getDriverConfig(), driverConfigs);
+ upgrader.upgrade(job.getFromJobConfig(), fromConfigs);
+ upgrader.upgrade(job.getToJobConfig(), toConfigs);
+ MJob newJob = new MJob(job, fromConfigs, toConfigs, driverConfigs);
+
+ // Transform config structures to objects for validations
+ SqoopConnector fromConnector =
+ ConnectorManager.getInstance().getConnector(
+ job.getConnectorId(Direction.FROM));
+ SqoopConnector toConnector =
+ ConnectorManager.getInstance().getConnector(
+ job.getConnectorId(Direction.TO));
+
+ Object fromConnectorConfig = ClassUtils.instantiate(
+ fromConnector.getJobConfigurationClass(Direction.FROM));
+ Object toConnectorConfig = ClassUtils.instantiate(
+ toConnector.getJobConfigurationClass(Direction.TO));
+ Object driverConfig = ClassUtils.instantiate(
+ Driver.getInstance().getDriverConfigurationGroupClass());
+
+ ConfigUtils.fromConfigs(
+ job.getFromJobConfig().getConfigs(), fromConnectorConfig);
+ ConfigUtils.fromConfigs(
+ job.getToJobConfig().getConfigs(), toConnectorConfig);
+ ConfigUtils.fromConfigs(
+ job.getDriverConfig().getConfigs(), driverConfig);
+
+ ConfigValidationRunner validationRunner = new ConfigValidationRunner();
+ ConfigValidationResult fromConnectorConfigResult = validationRunner.validate(fromConnectorConfig);
+ ConfigValidationResult toConnectorConfigResult = validationRunner.validate(toConnectorConfig);
+ ConfigValidationResult driverConfigResult = validationRunner.validate(driverConfig);
+
+ Status finalStatus = Status.getWorstStatus(fromConnectorConfigResult.getStatus(),
+ toConnectorConfigResult.getStatus(), driverConfigResult.getStatus());
if (finalStatus.canProceed()) {
repository.createJob(newJob);
} else {
LOG.error("Failed to load job:" + job.getName());
- LOG.error("Status of connector forms:" + connectorValidation.getStatus().toString());
- LOG.error("Status of framework forms:" + frameworkValidation.getStatus().toString());
+ LOG.error("Status of from connector configs:" + fromConnectorConfigResult.getStatus().toString());
+ LOG.error("Status of to connector configs:" + toConnectorConfigResult.getStatus().toString());
+ LOG.error("Status of driver configs:" + driverConfigResult.getStatus().toString());
}
return newJob.getPersistenceId();
[32/52] [abbrv] git commit: SQOOP-1479: Sqoop2: Validations: Cache
Validator objects in ValidationRunner
Posted by ab...@apache.org.
SQOOP-1479: Sqoop2: Validations: Cache Validator objects in ValidationRunner
(Jarek Jarcec Cecho via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/e540668b
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/e540668b
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/e540668b
Branch: refs/heads/SQOOP-1367
Commit: e540668b5569753165d812522e1c5bc24cc1fe5b
Parents: c1e53e5
Author: Abraham Elmahrek <ab...@elmahrek.com>
Authored: Thu Oct 2 21:26:41 2014 -0700
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:59:25 2014 -0700
----------------------------------------------------------------------
.../sqoop/validation/ValidationError.java | 3 ++
.../sqoop/validation/ValidationRunner.java | 35 ++++++++++++++++++--
2 files changed, 35 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e540668b/common/src/main/java/org/apache/sqoop/validation/ValidationError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationError.java b/common/src/main/java/org/apache/sqoop/validation/ValidationError.java
index 264d6e3..ec64f10 100644
--- a/common/src/main/java/org/apache/sqoop/validation/ValidationError.java
+++ b/common/src/main/java/org/apache/sqoop/validation/ValidationError.java
@@ -31,6 +31,9 @@ public enum ValidationError implements ErrorCode {
VALIDATION_0002("Usage of missing field"),
VALIDATION_0003("Invalid representation of form and input field"),
+
+ VALIDATION_0004("Can't find validator class"),
+
;
private final String message;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/e540668b/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java b/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
index f36faf2..8ffc0d4 100644
--- a/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
+++ b/common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java
@@ -17,6 +17,7 @@
*/
package org.apache.sqoop.validation;
+import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.FormClass;
@@ -27,6 +28,8 @@ import org.apache.sqoop.utils.ClassUtils;
import org.apache.sqoop.validation.validators.AbstractValidator;
import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
/**
* Validation runner that will run validators associated with given configuration
@@ -39,12 +42,23 @@ import java.lang.reflect.Field;
* Which means that form validator don't have to repeat it's input validators as it will
* be never called if the input's are not valid. Similarly Class validators won't be called
* unless all forms will pass validators.
- *
- * TODO: Cache the validators instances, so that we don't have create new instance every time
*/
public class ValidationRunner {
/**
+ * Private cache of instantiated validators.
+ *
+ * We're expecting that this cache will be very small as the number of possible validators
+ * is driven to high extent by the number of connectors and hence we don't have a cache
+ * eviction at the moment.
+ */
+ private Map<Class<? extends AbstractValidator>, AbstractValidator> cache;
+
+ public ValidationRunner() {
+ cache = new HashMap<Class<? extends AbstractValidator>, AbstractValidator>();
+ }
+
+ /**
* Validate given configuration instance.
*
* @param config Configuration instance
@@ -137,7 +151,22 @@ public class ValidationRunner {
* @return
*/
private AbstractValidator executeValidator(Object object, Validator validator) {
- AbstractValidator instance = (AbstractValidator) ClassUtils.instantiate(validator.value());
+ // Try to get validator instance from the cache
+ AbstractValidator instance = cache.get(validator.value());
+
+ if(instance == null) {
+ instance = (AbstractValidator) ClassUtils.instantiate(validator.value());
+
+ // This could happen if we would be missing some connector's jars on our classpath
+ if(instance == null) {
+ throw new SqoopException(ValidationError.VALIDATION_0004, validator.value().getName());
+ }
+
+ cache.put(validator.value(), instance);
+ } else {
+ instance.reset();
+ }
+
instance.setStringArgument(validator.strArg());
instance.validate(object);
return instance;
[40/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index 5dd7970..39702ca 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -28,31 +28,38 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
-import org.apache.log4j.Logger;
import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.DirectionError;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.ConnectorHandler;
import org.apache.sqoop.connector.ConnectorManagerUtils;
import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MEnumInput;
-import org.apache.sqoop.model.MIntegerInput;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConfigType;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MFormType;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MEnumInput;
+import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MInputType;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.repository.JdbcRepositoryContext;
import org.apache.sqoop.repository.JdbcRepositoryHandler;
import org.apache.sqoop.submission.SubmissionStatus;
@@ -93,74 +100,69 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
mc.getUniqueName());
}
mc.setPersistenceId(getConnectorId(mc, conn));
- insertFormsForConnector(mc, conn);
+ insertConfigsForConnector(mc, conn);
}
/**
- * Helper method to insert the forms from the into the
- * repository. The job and connector forms within <code>mc</code> will get
- * updated with the id of the forms when this function returns.
- * @param mDriverConfig The MFramework instance to use to upgrade.
- * @param conn JDBC link to use for updating the forms
+ * Helper method to insert the configs from the into the
+ * repository.
+ * @param mDriver The driver instance to use to upgrade.
+ * @param conn JDBC link to use for updating the configs
*/
- private void insertFormsForFramework(MDriverConfig mDriverConfig, Connection conn) {
- PreparedStatement baseFormStmt = null;
+ private void insertConfigsForDriver(MDriver mDriver, Connection conn) {
+ PreparedStatement baseConfigStmt = null;
PreparedStatement baseInputStmt = null;
try{
- baseFormStmt = conn.prepareStatement(STMT_INSERT_FORM_BASE,
+ baseConfigStmt = conn.prepareStatement(STMT_INSERT_CONFIG_BASE,
Statement.RETURN_GENERATED_KEYS);
baseInputStmt = conn.prepareStatement(STMT_INSERT_INPUT_BASE,
Statement.RETURN_GENERATED_KEYS);
- // Register connector forms
- registerForms(null, null, mDriverConfig.getConnectionForms().getForms(),
- MFormType.CONNECTION.name(), baseFormStmt, baseInputStmt);
-
- // Register job forms
- registerForms(null, null, mDriverConfig.getJobForms().getForms(),
- MFormType.JOB.name(), baseFormStmt, baseInputStmt);
+ // Register the job config type, since driver config is per job
+ registerConfigs(null, null, mDriver.getDriverConfig().getConfigs(),
+ MConfigType.JOB.name(), baseConfigStmt, baseInputStmt);
} catch (SQLException ex) {
- throw new SqoopException(DerbyRepoError.DERBYREPO_0014, mDriverConfig.toString(), ex);
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0014, mDriver.toString(), ex);
} finally {
- closeStatements(baseFormStmt, baseInputStmt);
+ closeStatements(baseConfigStmt, baseInputStmt);
}
}
/**
- * Helper method to insert the forms from the MConnector into the
- * repository. The job and connector forms within <code>mc</code> will get
- * updated with the id of the forms when this function returns.
- * @param mc The connector to use for updating forms
- * @param conn JDBC link to use for updating the forms
+ * Helper method to insert the configs from the MConnector into the
+ * repository. The job and connector configs within <code>mc</code> will get
+ * updated with the id of the configs when this function returns.
+ * @param mc The connector to use for updating configs
+ * @param conn JDBC link to use for updating the configs
*/
- private void insertFormsForConnector (MConnector mc, Connection conn) {
+ private void insertConfigsForConnector (MConnector mc, Connection conn) {
long connectorId = mc.getPersistenceId();
- PreparedStatement baseFormStmt = null;
+ PreparedStatement baseConfigStmt = null;
PreparedStatement baseInputStmt = null;
try{
- baseFormStmt = conn.prepareStatement(STMT_INSERT_FORM_BASE,
+ baseConfigStmt = conn.prepareStatement(STMT_INSERT_CONFIG_BASE,
Statement.RETURN_GENERATED_KEYS);
baseInputStmt = conn.prepareStatement(STMT_INSERT_INPUT_BASE,
Statement.RETURN_GENERATED_KEYS);
- // Register connector forms
- registerForms(connectorId, null, mc.getConnectionForms().getForms(),
- MFormType.CONNECTION.name(), baseFormStmt, baseInputStmt);
+ // Register link type config
+ registerConfigs(connectorId, null, mc.getLinkConfig().getConfigs(),
+ MConfigType.LINK.name(), baseConfigStmt, baseInputStmt);
- // Register all jobs
- registerForms(connectorId, Direction.FROM, mc.getJobForms(Direction.FROM).getForms(),
- MFormType.JOB.name(), baseFormStmt, baseInputStmt);
- registerForms(connectorId, Direction.TO, mc.getJobForms(Direction.TO).getForms(),
- MFormType.JOB.name(), baseFormStmt, baseInputStmt);
+ // Register both from/to job type config
+ registerConfigs(connectorId, Direction.FROM, mc.getConfig(Direction.FROM).getConfigs(),
+ MConfigType.JOB.name(), baseConfigStmt, baseInputStmt);
+ registerConfigs(connectorId, Direction.TO, mc.getConfig(Direction.TO).getConfigs(),
+ MConfigType.JOB.name(), baseConfigStmt, baseInputStmt);
} catch (SQLException ex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0014,
mc.toString(), ex);
} finally {
- closeStatements(baseFormStmt, baseInputStmt);
+ closeStatements(baseConfigStmt, baseInputStmt);
}
}
@@ -239,7 +241,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
} else {
LOG.warn("Even though embedded Derby driver was loaded, the connect "
- + "URL is of an unexpected form: " + connectUrl + ". Therefore no "
+ + "URL is of an unexpected config: " + connectUrl + ". Therefore no "
+ "attempt will be made to shutdown embedded Derby instance.");
}
}
@@ -335,9 +337,9 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
/**
* Create or update driver version
* @param conn Connection to the the repository
- * @param mDriverConfig
+ * @param mDriver
*/
- private void createOrUpdateDriverVersion(Connection conn, MDriverConfig mDriverConfig) {
+ private void createOrUpdateDriverSystemVersion(Connection conn, String version) {
ResultSet rs = null;
PreparedStatement stmt = null;
try {
@@ -348,7 +350,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
stmt = conn.prepareStatement(STMT_INSERT_SYSTEM);
stmt.setString(1, DerbyRepoConstants.SYSKEY_DRIVER_VERSION);
- stmt.setString(2, mDriverConfig.getVersion());
+ stmt.setString(2, version);
stmt.executeUpdate();
} catch (SQLException e) {
logException(e);
@@ -369,11 +371,11 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
if(version <= 0) {
runQuery(QUERY_CREATE_SCHEMA_SQOOP, conn);
runQuery(QUERY_CREATE_TABLE_SQ_CONNECTOR, conn);
- runQuery(QUERY_CREATE_TABLE_SQ_FORM, conn);
+ runQuery(QUERY_CREATE_TABLE_SQ_CONFIG, conn);
runQuery(QUERY_CREATE_TABLE_SQ_INPUT, conn);
- runQuery(QUERY_CREATE_TABLE_SQ_CONNECTION, conn);
+ runQuery(QUERY_CREATE_TABLE_SQ_LINK, conn);
runQuery(QUERY_CREATE_TABLE_SQ_JOB, conn);
- runQuery(QUERY_CREATE_TABLE_SQ_CONNECTION_INPUT, conn);
+ runQuery(QUERY_CREATE_TABLE_SQ_LINK_INPUT, conn);
runQuery(QUERY_CREATE_TABLE_SQ_JOB_INPUT, conn);
runQuery(QUERY_CREATE_TABLE_SQ_SUBMISSION, conn);
runQuery(QUERY_CREATE_TABLE_SQ_COUNTER_GROUP, conn);
@@ -382,10 +384,10 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
if(version <= 1) {
runQuery(QUERY_CREATE_TABLE_SQ_SYSTEM, conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_ENABLED, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_ENABLED, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_ENABLED, conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_CREATION_USER, conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTION_ADD_COLUMN_UPDATE_USER, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_CREATION_USER, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_COLUMN_UPDATE_USER, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_CREATION_USER, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_UPDATE_USER, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_CREATION_USER, conn);
@@ -397,17 +399,17 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
if(version <= 3) {
// Schema modifications
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_RENAME_COLUMN_SQF_OPERATION_TO_SQF_DIRECTION, conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_CONNECTION_TO_SQB_FROM_CONNECTION, conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_CONNECTION, conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQN, conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_FROM, conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_TO, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_RENAME_COLUMN_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_LINK_TO_SQB_FROM_LINK, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_LINK, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQ_LNK, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_FROM, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQ_LNK_TO, conn);
// Data modifications only for non-fresh install.
if (version > 0) {
// Register HDFS connector
- updateJobData(conn, registerHdfsConnector(conn));
+ updteJobInternals(conn, registerHdfsConnector(conn));
}
// Wait to remove SQB_TYPE (IMPORT/EXPORT) until we update data.
@@ -440,84 +442,84 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* Upgrade job data from IMPORT/EXPORT to FROM/TO.
* Since the framework is no longer responsible for HDFS,
* the HDFS connector/link must be added.
- * Also, the framework forms are moved around such that
- * they belong to the added HDFS connector. Any extra forms
+ * Also, the framework configs are moved around such that
+ * they belong to the added HDFS connector. Any extra configs
* are removed.
- * NOTE: Connector forms should have a direction (FROM/TO),
- * but framework forms should not.
+ * NOTE: Connector configs should have a direction (FROM/TO),
+ * but framework configs should not.
*
* Here's a brief list describing the data migration process.
- * 1. Change SQ_FORM.SQF_DIRECTION from IMPORT to FROM.
- * 2. Change SQ_FORM.SQF_DIRECTION from EXPORT to TO.
- * 3. Change EXPORT to TO in newly existing SQF_DIRECTION.
- * This should affect connectors only since Connector forms
- * should have had a value for SQF_OPERATION.
- * 4. Change IMPORT to FROM in newly existing SQF_DIRECTION.
- * This should affect connectors only since Connector forms
- * should have had a value for SQF_OPERATION.
+ * 1. Change SQ_CONFIG.SQ_CFG_DIRECTION from IMPORT to FROM.
+ * 2. Change SQ_CONFIG.SQ_CFG_DIRECTION from EXPORT to TO.
+ * 3. Change EXPORT to TO in newly existing SQ_CFG_DIRECTION.
+ * This should affect connectors only since Connector configs
+ * should have had a value for SQ_CFG_OPERATION.
+ * 4. Change IMPORT to FROM in newly existing SQ_CFG_DIRECTION.
+ * This should affect connectors only since Connector configs
+ * should have had a value for SQ_CFG_OPERATION.
* 5. Add HDFS connector for jobs to reference.
- * 6. Set 'input' and 'output' forms connector.
+ * 6. Set 'input' and 'output' configs connector.
* to HDFS connector.
- * 7. Throttling form was originally the second form in
- * the framework. It should now be the first form.
- * 8. Remove the EXPORT throttling form and ensure all of
- * its dependencies point to the IMPORT throttling form.
- * Then make sure the throttling form does not have a direction.
- * Framework forms should not have a direction.
+ * 7. Throttling config was originally the second config in
+ * the framework. It should now be the first config.
+ * 8. Remove the EXPORT throttling config and ensure all of
+ * its dependencies point to the IMPORT throttling config.
+ * Then make sure the throttling config does not have a direction.
+ * Framework configs should not have a direction.
* 9. Create an HDFS link to reference and update
* jobs to reference that link. IMPORT jobs
* should have TO HDFS connector, EXPORT jobs should have
* FROM HDFS connector.
- * 10. Update 'table' form names to 'fromJobConfig' and 'toTable'.
+ * 10. Update 'table' config names to 'fromJobConfig' and 'toTable'.
* Also update the relevant inputs as well.
* @param conn
*/
- private void updateJobData(Connection conn, long connectorId) {
+ private void updteJobInternals(Connection conn, long connectorId) {
if (LOG.isTraceEnabled()) {
LOG.trace("Updating existing data for generic connectors.");
}
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_OPERATION_TO_SQF_DIRECTION, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION, conn,
Direction.FROM.toString(), "IMPORT");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_OPERATION_TO_SQF_DIRECTION, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_SQ_CFG_OPERATION_TO_SQ_CFG_DIRECTION, conn,
Direction.TO.toString(), "EXPORT");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR_HDFS_FORM_DIRECTION, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_CONNECTOR_HDFS_CONFIG_DIRECTION, conn,
Direction.FROM.toString(),
"input");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR_HDFS_FORM_DIRECTION, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_CONNECTOR_HDFS_CONFIG_DIRECTION, conn,
Direction.TO.toString(),
"output");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_CONNECTOR, conn,
new Long(connectorId), "input", "output");
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_UPDATE_THROTTLING_FORM_INPUTS, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_UPDATE_THROTTLING_CONFIG_INPUTS, conn,
"IMPORT", "EXPORT");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_REMOVE_EXTRA_FORM_INPUTS, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_REMOVE_EXTRA_CONFIG_INPUTS, conn,
"throttling", "EXPORT");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_REMOVE_EXTRA_FRAMEWORK_FORM, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_REMOVE_EXTRA_DRIVER_CONFIG, conn,
"throttling", "EXPORT");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_DIRECTION_TO_NULL, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_DIRECTION_TO_NULL, conn,
"throttling");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_FRAMEWORK_INDEX, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_DRIVER_INDEX, conn,
new Long(0), "throttling");
- MLink hdfsConnection = createHdfsConnection(conn);
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION_COPY_SQB_FROM_CONNECTION, conn,
+ MLink hdfsLink = createHdfsLink(conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_LINK_COPY_SQB_FROM_LINK, conn,
"EXPORT");
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_FROM_CONNECTION, conn,
- new Long(hdfsConnection.getPersistenceId()), "EXPORT");
- runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION, conn,
- new Long(hdfsConnection.getPersistenceId()), "IMPORT");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_FROM_LINK, conn,
+ new Long(hdfsLink.getPersistenceId()), "EXPORT");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_LINK, conn,
+ new Long(hdfsLink.getPersistenceId()), "IMPORT");
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_NAME, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_SQ_CFG_NAME, conn,
"fromJobConfig", "table", Direction.FROM.toString());
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_TABLE_INPUT_NAMES, conn,
Direction.FROM.toString().toLowerCase(), "fromJobConfig", Direction.FROM.toString());
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_NAME, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_SQ_CFG_NAME, conn,
"toJobConfig", "table", Direction.TO.toString());
- runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES, conn,
+ runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_UPDATE_TABLE_INPUT_NAMES, conn,
Direction.TO.toString().toLowerCase(), "toJobConfig", Direction.TO.toString());
if (LOG.isTraceEnabled()) {
@@ -526,7 +528,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
/**
- * Pre-register HDFS Connector so that metadata upgrade will work.
+ * Pre-register HDFS Connector so that config upgrade will work.
*/
protected long registerHdfsConnector(Connection conn) {
if (LOG.isTraceEnabled()) {
@@ -536,7 +538,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs();
if (LOG.isInfoEnabled()) {
- LOG.info("Connector config urls: " + connectorConfigs);
+ LOG.info("Connector configs: " + connectorConfigs);
}
ConnectorHandler handler = null;
@@ -582,24 +584,22 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
*
* NOTE: Upgrade path only!
*/
- private MLink createHdfsConnection(Connection conn) {
+ private MLink createHdfsLink(Connection conn) {
if (LOG.isTraceEnabled()) {
LOG.trace("Creating HDFS link.");
}
MConnector hdfsConnector = this.findConnector(CONNECTOR_HDFS, conn);
- MDriverConfig driverConfig = findDriverConfig(conn);
- MLink hdfsConnection = new MLink(
+ MLink hdfsLink = new MLink(
hdfsConnector.getPersistenceId(),
- hdfsConnector.getConnectionForms(),
- driverConfig.getConnectionForms());
- this.createLink(hdfsConnection, conn);
+ hdfsConnector.getLinkConfig());
+ this.createLink(hdfsLink, conn);
if (LOG.isTraceEnabled()) {
LOG.trace("Created HDFS link.");
}
- return hdfsConnection;
+ return hdfsLink;
}
/**
@@ -675,92 +675,80 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public void registerDriverConfig(MDriverConfig mDriverConfig, Connection conn) {
- if (mDriverConfig.hasPersistenceId()) {
+ public void registerDriver(MDriver mDriver, Connection conn) {
+ if (mDriver.hasPersistenceId()) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0011,
- "Framework metadata");
+ "Driver");
}
- PreparedStatement baseFormStmt = null;
+ PreparedStatement baseConfigStmt = null;
PreparedStatement baseInputStmt = null;
try {
- baseFormStmt = conn.prepareStatement(STMT_INSERT_FORM_BASE,
+ baseConfigStmt = conn.prepareStatement(STMT_INSERT_CONFIG_BASE,
Statement.RETURN_GENERATED_KEYS);
baseInputStmt = conn.prepareStatement(STMT_INSERT_INPUT_BASE,
Statement.RETURN_GENERATED_KEYS);
- // Register connector forms
- registerForms(null, null, mDriverConfig.getConnectionForms().getForms(),
- MFormType.CONNECTION.name(), baseFormStmt, baseInputStmt);
-
- // Register all jobs
- registerForms(null, null, mDriverConfig.getJobForms().getForms(),
- MFormType.JOB.name(), baseFormStmt, baseInputStmt);
+ // Register a driver config as a job type with no owner/connector and direction
+ registerConfigs(null/* owner*/, null /*direction*/, mDriver.getDriverConfig().getConfigs(),
+ MConfigType.JOB.name(), baseConfigStmt, baseInputStmt);
- // We're using hardcoded value for framework metadata as they are
+ // We're using hardcoded value for driver config as they are
// represented as NULL in the database.
- mDriverConfig.setPersistenceId(1);
+ mDriver.setPersistenceId(1);
} catch (SQLException ex) {
- logException(ex, mDriverConfig);
+ logException(ex, mDriver);
throw new SqoopException(DerbyRepoError.DERBYREPO_0014, ex);
} finally {
- closeStatements(baseFormStmt, baseInputStmt);
+ closeStatements(baseConfigStmt, baseInputStmt);
}
- createOrUpdateDriverVersion(conn, mDriverConfig);
+ createOrUpdateDriverSystemVersion(conn, mDriver.getVersion());
}
/**
* {@inheritDoc}
*/
@Override
- public MDriverConfig findDriverConfig(Connection conn) {
- LOG.debug("Looking up driver config");
- MDriverConfig mDriverConfig = null;
- PreparedStatement formFetchStmt = null;
- PreparedStatement inputFetchStmt = null;
+ public MDriver findDriver(Connection conn) {
+ LOG.debug("Looking up Driver config to create a driver ");
+ MDriver mDriver = null;
+ PreparedStatement driverConfigFetchStmt = null;
+ PreparedStatement driverConfigInputFetchStmt = null;
try {
- formFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_FRAMEWORK);
- inputFetchStmt = conn.prepareStatement(STMT_FETCH_INPUT);
-
- List<MForm> connectionForms = new ArrayList<MForm>();
- List<MForm> jobForms = new ArrayList<MForm>();
+ driverConfigFetchStmt = conn.prepareStatement(STMT_FETCH_CONFIG_DRIVER);
+ driverConfigInputFetchStmt = conn.prepareStatement(STMT_FETCH_INPUT);
+ List<MConfig> driverConfigs = new ArrayList<MConfig>();
+ loadDriverConfigs(driverConfigs, driverConfigFetchStmt, driverConfigInputFetchStmt, 1);
- loadFrameworkForms(connectionForms, jobForms, formFetchStmt, inputFetchStmt, 1);
-
- // Return nothing If there aren't any framework metadata
- if(connectionForms.isEmpty() && jobForms.isEmpty()) {
+ if(driverConfigs.isEmpty()) {
return null;
}
- mDriverConfig = new MDriverConfig(new MConnectionForms(connectionForms),
- new MJobForms(jobForms), detectDriverVersion(conn));
-
- // We're using hardcoded value for driver config as they are
- // represented as NULL in the database.
- mDriverConfig.setPersistenceId(1);
+ mDriver = new MDriver(new MDriverConfig(driverConfigs), detectDriverVersion(conn));
+ mDriver.setPersistenceId(1);
} catch (SQLException ex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0004,
"Driver config", ex);
} finally {
- if (formFetchStmt != null) {
+ if (driverConfigFetchStmt != null) {
try {
- formFetchStmt.close();
+ driverConfigFetchStmt.close();
} catch (SQLException ex) {
- LOG.error("Unable to close form fetch statement", ex);
+ LOG.error("Unable to close config fetch statement", ex);
}
}
- if (inputFetchStmt != null) {
+ if (driverConfigInputFetchStmt != null) {
try {
- inputFetchStmt.close();
+ driverConfigInputFetchStmt.close();
} catch (SQLException ex) {
LOG.error("Unable to close input fetch statement", ex);
}
}
}
- LOG.debug("Looking up driver config found:" + mDriverConfig);
- return mDriverConfig;
+ LOG.debug("Looking up Driver config and created driver:" + mDriver);
+ return mDriver;
}
/**
@@ -779,7 +767,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
PreparedStatement stmt = null;
int result;
try {
- stmt = conn.prepareStatement(STMT_INSERT_CONNECTION,
+ stmt = conn.prepareStatement(STMT_INSERT_LINK,
Statement.RETURN_GENERATED_KEYS);
stmt.setString(1, link.getName());
stmt.setLong(2, link.getConnectorId());
@@ -803,15 +791,10 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
long connectionId = rsetConnectionId.getLong(1);
- createInputValues(STMT_INSERT_CONNECTION_INPUT,
+ createInputValues(STMT_INSERT_LINK_INPUT,
connectionId,
- link.getConnectorPart().getForms(),
+ link.getConnectorLinkConfig().getConfigs(),
conn);
- createInputValues(STMT_INSERT_CONNECTION_INPUT,
- connectionId,
- link.getFrameworkPart().getForms(),
- conn);
-
link.setPersistenceId(connectionId);
} catch (SQLException ex) {
@@ -831,12 +814,12 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
PreparedStatement updateStmt = null;
try {
// Firstly remove old values
- deleteStmt = conn.prepareStatement(STMT_DELETE_CONNECTION_INPUT);
+ deleteStmt = conn.prepareStatement(STMT_DELETE_LINK_INPUT);
deleteStmt.setLong(1, link.getPersistenceId());
deleteStmt.executeUpdate();
- // Update CONNECTION table
- updateStmt = conn.prepareStatement(STMT_UPDATE_CONNECTION);
+ // Update LINK_CONFIG table
+ updateStmt = conn.prepareStatement(STMT_UPDATE_LINK);
updateStmt.setString(1, link.getName());
updateStmt.setString(2, link.getLastUpdateUser());
updateStmt.setTimestamp(3, new Timestamp(new Date().getTime()));
@@ -845,13 +828,9 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
updateStmt.executeUpdate();
// And reinsert new values
- createInputValues(STMT_INSERT_CONNECTION_INPUT,
- link.getPersistenceId(),
- link.getConnectorPart().getForms(),
- conn);
- createInputValues(STMT_INSERT_CONNECTION_INPUT,
+ createInputValues(STMT_INSERT_LINK_INPUT,
link.getPersistenceId(),
- link.getFrameworkPart().getForms(),
+ link.getConnectorLinkConfig().getConfigs(),
conn);
} catch (SQLException ex) {
@@ -870,7 +849,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
PreparedStatement stmt = null;
ResultSet rs = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_CONNECTION_CHECK);
+ stmt = conn.prepareStatement(STMT_SELECT_LINK_CHECK);
stmt.setLong(1, id);
rs = stmt.executeQuery();
@@ -893,7 +872,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
ResultSet rs = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_JOBS_FOR_CONNECTION_CHECK);
+ stmt = conn.prepareStatement(STMT_SELECT_JOBS_FOR_LINK_CHECK);
stmt.setLong(1, connectionId);
rs = stmt.executeQuery();
@@ -916,7 +895,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
PreparedStatement enableConn = null;
try {
- enableConn = conn.prepareStatement(STMT_ENABLE_CONNECTION);
+ enableConn = conn.prepareStatement(STMT_ENABLE_LINK);
enableConn.setBoolean(1, enabled);
enableConn.setLong(2, connectionId);
enableConn.executeUpdate();
@@ -937,7 +916,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
try {
deleteLinkInputs(id, conn);
- dltConn = conn.prepareStatement(STMT_DELETE_CONNECTION);
+ dltConn = conn.prepareStatement(STMT_DELETE_LINK);
dltConn.setLong(1, id);
dltConn.executeUpdate();
} catch (SQLException ex) {
@@ -955,7 +934,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
public void deleteLinkInputs(long id, Connection conn) {
PreparedStatement dltConnInput = null;
try {
- dltConnInput = conn.prepareStatement(STMT_DELETE_CONNECTION_INPUT);
+ dltConnInput = conn.prepareStatement(STMT_DELETE_LINK_INPUT);
dltConnInput.setLong(1, id);
dltConnInput.executeUpdate();
} catch (SQLException ex) {
@@ -973,7 +952,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
public MLink findLink(long id, Connection conn) {
PreparedStatement stmt = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_CONNECTION_SINGLE);
+ stmt = conn.prepareStatement(STMT_SELECT_LINK_SINGLE);
stmt.setLong(1, id);
List<MLink> connections = loadLinks(stmt, conn);
@@ -1001,7 +980,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
public List<MLink> findLinks(Connection conn) {
PreparedStatement stmt = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_CONNECTION_ALL);
+ stmt = conn.prepareStatement(STMT_SELECT_LINK_ALL);
return loadLinks(stmt, conn);
@@ -1023,7 +1002,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
public List<MLink> findLinksForConnector(long connectorID, Connection conn) {
PreparedStatement stmt = null;
try {
- stmt = conn.prepareStatement(STMT_SELECT_CONNECTION_FOR_CONNECTOR);
+ stmt = conn.prepareStatement(STMT_SELECT_LINK_FOR_CONNECTOR);
stmt.setLong(1, connectorID);
return loadLinks(stmt, conn);
@@ -1042,12 +1021,12 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
@Override
public void updateConnector(MConnector mConnector, Connection conn) {
PreparedStatement updateConnectorStatement = null;
- PreparedStatement deleteForm = null;
+ PreparedStatement deleteConfig = null;
PreparedStatement deleteInput = null;
try {
updateConnectorStatement = conn.prepareStatement(STMT_UPDATE_CONNECTOR);
deleteInput = conn.prepareStatement(STMT_DELETE_INPUTS_FOR_CONNECTOR);
- deleteForm = conn.prepareStatement(STMT_DELETE_FORMS_FOR_CONNECTOR);
+ deleteConfig = conn.prepareStatement(STMT_DELETE_CONFIGS_FOR_CONNECTOR);
updateConnectorStatement.setString(1, mConnector.getUniqueName());
updateConnectorStatement.setString(2, mConnector.getClassName());
updateConnectorStatement.setString(3, mConnector.getVersion());
@@ -1057,17 +1036,17 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
throw new SqoopException(DerbyRepoError.DERBYREPO_0038);
}
deleteInput.setLong(1, mConnector.getPersistenceId());
- deleteForm.setLong(1, mConnector.getPersistenceId());
+ deleteConfig.setLong(1, mConnector.getPersistenceId());
deleteInput.executeUpdate();
- deleteForm.executeUpdate();
+ deleteConfig.executeUpdate();
} catch (SQLException e) {
logException(e, mConnector);
throw new SqoopException(DerbyRepoError.DERBYREPO_0038, e);
} finally {
- closeStatements(updateConnectorStatement, deleteForm, deleteInput);
+ closeStatements(updateConnectorStatement, deleteConfig, deleteInput);
}
- insertFormsForConnector(mConnector, conn);
+ insertConfigsForConnector(mConnector, conn);
}
@@ -1075,25 +1054,24 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* {@inheritDoc}
*/
@Override
- public void updateDriverConfig(MDriverConfig mDriverConfig, Connection conn) {
- PreparedStatement deleteForm = null;
+ public void updateDriver(MDriver mDriver, Connection conn) {
+ PreparedStatement deleteConfig = null;
PreparedStatement deleteInput = null;
try {
- deleteInput = conn.prepareStatement(STMT_DELETE_FRAMEWORK_INPUTS);
- deleteForm = conn.prepareStatement(STMT_DELETE_FRAMEWORK_FORMS);
+ deleteInput = conn.prepareStatement(STMT_DELETE_DRIVER_INPUTS);
+ deleteConfig = conn.prepareStatement(STMT_DELETE_DRIVER_CONFIGS);
deleteInput.executeUpdate();
- deleteForm.executeUpdate();
+ deleteConfig.executeUpdate();
} catch (SQLException e) {
- logException(e, mDriverConfig);
+ logException(e, mDriver);
throw new SqoopException(DerbyRepoError.DERBYREPO_0044, e);
} finally {
- closeStatements(deleteForm, deleteInput);
+ closeStatements(deleteConfig, deleteInput);
}
- createOrUpdateDriverVersion(conn, mDriverConfig);
- insertFormsForFramework(mDriverConfig, conn);
-
+ createOrUpdateDriverSystemVersion(conn, mDriver.getVersion());
+ insertConfigsForDriver(mDriver, conn);
}
/**
@@ -1104,8 +1082,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
PreparedStatement stmt = null;
int result;
try {
- stmt = conn.prepareStatement(STMT_INSERT_JOB,
- Statement.RETURN_GENERATED_KEYS);
+ stmt = conn.prepareStatement(STMT_INSERT_JOB, Statement.RETURN_GENERATED_KEYS);
stmt.setString(1, job.getName());
stmt.setLong(2, job.getLinkId(Direction.FROM));
stmt.setLong(3, job.getLinkId(Direction.TO));
@@ -1129,17 +1106,20 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
long jobId = rsetJobId.getLong(1);
+ // from config for the job
createInputValues(STMT_INSERT_JOB_INPUT,
jobId,
- job.getConnectorPart(Direction.FROM).getForms(),
+ job.getJobConfig(Direction.FROM).getConfigs(),
conn);
+ // to config for the job
createInputValues(STMT_INSERT_JOB_INPUT,
jobId,
- job.getConnectorPart(Direction.TO).getForms(),
+ job.getJobConfig(Direction.TO).getConfigs(),
conn);
+ // driver config per job
createInputValues(STMT_INSERT_JOB_INPUT,
jobId,
- job.getFrameworkPart().getForms(),
+ job.getDriverConfig().getConfigs(),
conn);
job.setPersistenceId(jobId);
@@ -1177,15 +1157,15 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
// And reinsert new values
createInputValues(STMT_INSERT_JOB_INPUT,
job.getPersistenceId(),
- job.getConnectorPart(Direction.FROM).getForms(),
+ job.getJobConfig(Direction.FROM).getConfigs(),
conn);
createInputValues(STMT_INSERT_JOB_INPUT,
job.getPersistenceId(),
- job.getConnectorPart(Direction.TO).getForms(),
+ job.getJobConfig(Direction.TO).getConfigs(),
conn);
createInputValues(STMT_INSERT_JOB_INPUT,
job.getPersistenceId(),
- job.getFrameworkPart().getForms(),
+ job.getDriverConfig().getConfigs(),
conn);
} catch (SQLException ex) {
@@ -1788,13 +1768,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
private List<MConnector> loadConnectors(PreparedStatement stmt,Connection conn) throws SQLException {
List<MConnector> connectors = new ArrayList<MConnector>();
ResultSet rsConnectors = null;
- PreparedStatement formFetchStmt = null;
- PreparedStatement inputFetchStmt = null;
+ PreparedStatement connectorConfigFetchStmt = null;
+ PreparedStatement connectorConfigInputFetchStmt = null;
try {
rsConnectors = stmt.executeQuery();
- formFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_CONNECTOR);
- inputFetchStmt = conn.prepareStatement(STMT_FETCH_INPUT);
+ connectorConfigFetchStmt = conn.prepareStatement(STMT_FETCH_CONFIG_CONNECTOR);
+ connectorConfigInputFetchStmt = conn.prepareStatement(STMT_FETCH_INPUT);
while(rsConnectors.next()) {
long connectorId = rsConnectors.getLong(1);
@@ -1802,26 +1782,26 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
String connectorClassName = rsConnectors.getString(3);
String connectorVersion = rsConnectors.getString(4);
- formFetchStmt.setLong(1, connectorId);
+ connectorConfigFetchStmt.setLong(1, connectorId);
- List<MForm> connectionForms = new ArrayList<MForm>();
- List<MForm> fromJobForms = new ArrayList<MForm>();
- List<MForm> toJobForms = new ArrayList<MForm>();
+ List<MConfig> linkConfig = new ArrayList<MConfig>();
+ List<MConfig> fromConfig = new ArrayList<MConfig>();
+ List<MConfig> toConfig = new ArrayList<MConfig>();
- loadConnectorForms(connectionForms, fromJobForms, toJobForms,
- formFetchStmt, inputFetchStmt, 1);
+ loadConfigTypes(linkConfig, fromConfig, toConfig,
+ connectorConfigFetchStmt, connectorConfigInputFetchStmt, 1);
MConnector mc = new MConnector(connectorName, connectorClassName, connectorVersion,
- new MConnectionForms(connectionForms),
- new MJobForms(fromJobForms),
- new MJobForms(toJobForms));
+ new MLinkConfig(linkConfig),
+ new MFromConfig(fromConfig),
+ new MToConfig(toConfig));
mc.setPersistenceId(connectorId);
connectors.add(mc);
}
} finally {
closeResultSets(rsConnectors);
- closeStatements(formFetchStmt,inputFetchStmt);
+ closeStatements(connectorConfigFetchStmt,connectorConfigInputFetchStmt);
}
return connectors;
@@ -1832,16 +1812,15 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
throws SQLException {
List<MLink> links = new ArrayList<MLink>();
ResultSet rsConnection = null;
- PreparedStatement formConnectorFetchStmt = null;
- PreparedStatement formFrameworkFetchStmt = null;
- PreparedStatement inputFetchStmt = null;
+ PreparedStatement connectorConfigFetchStatement = null;
+ PreparedStatement connectorConfigInputStatement = null;
try {
rsConnection = stmt.executeQuery();
- formConnectorFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_CONNECTOR);
- formFrameworkFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_FRAMEWORK);
- inputFetchStmt = conn.prepareStatement(STMT_FETCH_CONNECTION_INPUT);
+ //
+ connectorConfigFetchStatement = conn.prepareStatement(STMT_FETCH_CONFIG_CONNECTOR);
+ connectorConfigInputStatement = conn.prepareStatement(STMT_FETCH_LINK_INPUT);
while(rsConnection.next()) {
long id = rsConnection.getLong(1);
@@ -1853,26 +1832,17 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
String updateUser = rsConnection.getString(7);
Date lastUpdateDate = rsConnection.getTimestamp(8);
- formConnectorFetchStmt.setLong(1, connectorId);
-
- inputFetchStmt.setLong(1, id);
- //inputFetchStmt.setLong(2, XXX); // Will be filled by loadFrameworkForms
- inputFetchStmt.setLong(3, id);
-
- List<MForm> connectorConnForms = new ArrayList<MForm>();
- List<MForm> frameworkConnForms = new ArrayList<MForm>();
- List<MForm> frameworkJobForms = new ArrayList<MForm>();
- List<MForm> fromJobForms = new ArrayList<MForm>();
- List<MForm> toJobForms = new ArrayList<MForm>();
+ connectorConfigFetchStatement.setLong(1, connectorId);
+ connectorConfigInputStatement.setLong(1, id);
+ connectorConfigInputStatement.setLong(3, id);
- loadConnectorForms(connectorConnForms, fromJobForms, toJobForms,
- formConnectorFetchStmt, inputFetchStmt, 2);
- loadFrameworkForms(frameworkConnForms, frameworkJobForms,
- formFrameworkFetchStmt, inputFetchStmt, 2);
+ List<MConfig> connectorLinkConfig = new ArrayList<MConfig>();
+ List<MConfig> fromConfig = new ArrayList<MConfig>();
+ List<MConfig> toConfig = new ArrayList<MConfig>();
- MLink link = new MLink(connectorId,
- new MConnectionForms(connectorConnForms),
- new MConnectionForms(frameworkConnForms));
+ loadConfigTypes(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement,
+ connectorConfigInputStatement, 2);
+ MLink link = new MLink(connectorId, new MLinkConfig(connectorLinkConfig));
link.setPersistenceId(id);
link.setName(name);
@@ -1886,8 +1856,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
} finally {
closeResultSets(rsConnection);
- closeStatements(formConnectorFetchStmt,
- formFrameworkFetchStmt, inputFetchStmt);
+ closeStatements(connectorConfigFetchStatement, connectorConfigInputStatement);
}
return links;
@@ -1898,20 +1867,21 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
throws SQLException {
List<MJob> jobs = new ArrayList<MJob>();
ResultSet rsJob = null;
- PreparedStatement toFormConnectorFetchStmt = null;
- PreparedStatement fromFormConnectorFetchStmt = null;
- PreparedStatement formFrameworkFetchStmt = null;
- PreparedStatement inputFetchStmt = null;
+ PreparedStatement fromConfigFetchStmt = null;
+ PreparedStatement toConfigFetchStmt = null;
+ PreparedStatement driverConfigfetchStmt = null;
+ PreparedStatement jobInputFetchStmt = null;
try {
rsJob = stmt.executeQuery();
- toFormConnectorFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_CONNECTOR);
- fromFormConnectorFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_CONNECTOR);
- formFrameworkFetchStmt = conn.prepareStatement(STMT_FETCH_FORM_FRAMEWORK);
- inputFetchStmt = conn.prepareStatement(STMT_FETCH_JOB_INPUT);
+ fromConfigFetchStmt = conn.prepareStatement(STMT_FETCH_CONFIG_CONNECTOR);
+ toConfigFetchStmt = conn.prepareStatement(STMT_FETCH_CONFIG_CONNECTOR);
+ driverConfigfetchStmt = conn.prepareStatement(STMT_FETCH_CONFIG_DRIVER);
+ jobInputFetchStmt = conn.prepareStatement(STMT_FETCH_JOB_INPUT);
while(rsJob.next()) {
+ // why use connector? why cant it be link id?
long fromConnectorId = rsJob.getLong(1);
long toConnectorId = rsJob.getLong(2);
long id = rsJob.getLong(3);
@@ -1924,48 +1894,40 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
String updateBy = rsJob.getString(10);
Date lastUpdateDate = rsJob.getTimestamp(11);
- fromFormConnectorFetchStmt.setLong(1, fromConnectorId);
- toFormConnectorFetchStmt.setLong(1,toConnectorId);
+ fromConfigFetchStmt.setLong(1, fromConnectorId);
+ toConfigFetchStmt.setLong(1,toConnectorId);
- inputFetchStmt.setLong(1, id);
- //inputFetchStmt.setLong(1, XXX); // Will be filled by loadFrameworkForms
- inputFetchStmt.setLong(3, id);
+ jobInputFetchStmt.setLong(1, id);
+ //inputFetchStmt.setLong(1, XXX); // Will be filled by loadFrameworkConfigs
+ jobInputFetchStmt.setLong(3, id);
- List<MForm> toConnectorConnForms = new ArrayList<MForm>();
- List<MForm> fromConnectorConnForms = new ArrayList<MForm>();
+ // FROM entity configs
+ List<MConfig> fromConnectorLinkConfig = new ArrayList<MConfig>();
+ List<MConfig> fromConnectorFromJobConfig = new ArrayList<MConfig>();
+ List<MConfig> fromConnectorToJobConfig = new ArrayList<MConfig>();
- List<MForm> frameworkConnForms = new ArrayList<MForm>();
- List<MForm> frameworkJobForms = new ArrayList<MForm>();
+ loadConfigTypes(fromConnectorLinkConfig, fromConnectorFromJobConfig, fromConnectorToJobConfig,
+ fromConfigFetchStmt, jobInputFetchStmt, 2);
- // This looks confusing but our job has 2 connectors, each connector has two job forms
- // To define the job, we need to TO job form of the TO connector
- // and the FROM job form of the FROM connector
- List<MForm> fromConnectorFromJobForms = new ArrayList<MForm>();
- List<MForm> fromConnectorToJobForms = new ArrayList<MForm>();
- List<MForm> toConnectorFromJobForms = new ArrayList<MForm>();
- List<MForm> toConnectorToJobForms = new ArrayList<MForm>();
+ // TO entity configs
+ List<MConfig> toConnectorLinkConfig = new ArrayList<MConfig>();
+ List<MConfig> toConnectorFromJobConfig = new ArrayList<MConfig>();
+ List<MConfig> toConnectorToJobConfig = new ArrayList<MConfig>();
+ // ?? dont we need 2 different driver configs for the from/to?
+ List<MConfig> driverConfig = new ArrayList<MConfig>();
- loadConnectorForms(fromConnectorConnForms,
- fromConnectorFromJobForms,
- fromConnectorToJobForms,
- fromFormConnectorFetchStmt,
- inputFetchStmt,
- 2);
- loadConnectorForms(toConnectorConnForms,
- toConnectorFromJobForms,
- toConnectorToJobForms,
- toFormConnectorFetchStmt, inputFetchStmt, 2);
+ loadConfigTypes(toConnectorLinkConfig, toConnectorFromJobConfig, toConnectorToJobConfig,
+ toConfigFetchStmt, jobInputFetchStmt, 2);
- loadFrameworkForms(frameworkConnForms, frameworkJobForms,
- formFrameworkFetchStmt, inputFetchStmt, 2);
+ loadDriverConfigs(driverConfig, driverConfigfetchStmt, jobInputFetchStmt, 2);
MJob job = new MJob(
fromConnectorId, toConnectorId,
fromLinkId, toLinkId,
- new MJobForms(fromConnectorFromJobForms),
- new MJobForms(toConnectorToJobForms),
- new MJobForms(frameworkJobForms));
+ new MFromConfig(fromConnectorFromJobConfig),
+ new MToConfig(toConnectorToJobConfig),
+ new MDriverConfig(driverConfig));
job.setPersistenceId(id);
job.setName(name);
@@ -1979,65 +1941,65 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
} finally {
closeResultSets(rsJob);
- closeStatements(fromFormConnectorFetchStmt, toFormConnectorFetchStmt, formFrameworkFetchStmt, inputFetchStmt);
+ closeStatements(fromConfigFetchStmt, toConfigFetchStmt, driverConfigfetchStmt, jobInputFetchStmt);
}
return jobs;
}
/**
- * Register forms in derby database. This method will insert the ids
- * generated by the repository into the forms passed in itself.
+ * Register configs in derby database. This method will insert the ids
+ * generated by the repository into the configs passed in itself.
*
- * Use given prepared statements to create entire form structure in database.
+ * Use given prepared statements to create entire config structure in database.
*
* @param connectorId
- * @param forms
+ * @param configs
* @param type
- * @param baseFormStmt
+ * @param baseConfigStmt
* @param baseInputStmt
- * @return short number of forms registered.
+ * @return short number of configs registered.
* @throws SQLException
*/
- private short registerForms(Long connectorId, Direction direction,
- List<MForm> forms, String type, PreparedStatement baseFormStmt,
+ private short registerConfigs(Long connectorId, Direction direction,
+ List<MConfig> configs, String type, PreparedStatement baseConfigStmt,
PreparedStatement baseInputStmt)
throws SQLException {
- short formIndex = 0;
+ short configIndex = 0;
- for (MForm form : forms) {
+ for (MConfig config : configs) {
if(connectorId == null) {
- baseFormStmt.setNull(1, Types.BIGINT);
+ baseConfigStmt.setNull(1, Types.BIGINT);
} else {
- baseFormStmt.setLong(1, connectorId);
+ baseConfigStmt.setLong(1, connectorId);
}
if(direction == null) {
- baseFormStmt.setNull(2, Types.VARCHAR);
+ baseConfigStmt.setNull(2, Types.VARCHAR);
} else {
- baseFormStmt.setString(2, direction.name());
+ baseConfigStmt.setString(2, direction.name());
}
- baseFormStmt.setString(3, form.getName());
- baseFormStmt.setString(4, type);
- baseFormStmt.setShort(5, formIndex++);
+ baseConfigStmt.setString(3, config.getName());
+ baseConfigStmt.setString(4, type);
+ baseConfigStmt.setShort(5, configIndex++);
- int baseFormCount = baseFormStmt.executeUpdate();
- if (baseFormCount != 1) {
+ int baseConfigCount = baseConfigStmt.executeUpdate();
+ if (baseConfigCount != 1) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0015,
- Integer.toString(baseFormCount));
+ Integer.toString(baseConfigCount));
}
- ResultSet rsetFormId = baseFormStmt.getGeneratedKeys();
- if (!rsetFormId.next()) {
+ ResultSet rsetConfigId = baseConfigStmt.getGeneratedKeys();
+ if (!rsetConfigId.next()) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0016);
}
- long formId = rsetFormId.getLong(1);
- form.setPersistenceId(formId);
+ long configId = rsetConfigId.getLong(1);
+ config.setPersistenceId(configId);
// Insert all the inputs
- List<MInput<?>> inputs = form.getInputs();
- registerFormInputs(formId, inputs, baseInputStmt);
+ List<MInput<?>> inputs = config.getInputs();
+ registerConfigInputs(configId, inputs, baseInputStmt);
}
- return formIndex;
+ return configIndex;
}
/**
@@ -2045,17 +2007,17 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
*
* Use given prepare statement to save all inputs into repository.
*
- * @param formId Identifier for corresponding form
+ * @param configId Identifier for corresponding config
* @param inputs List of inputs that needs to be saved
* @param baseInputStmt Statement that we can utilize
* @throws SQLException In case of any failure on Derby side
*/
- private void registerFormInputs(long formId, List<MInput<?>> inputs,
+ private void registerConfigInputs(long configId, List<MInput<?>> inputs,
PreparedStatement baseInputStmt) throws SQLException {
short inputIndex = 0;
for (MInput<?> input : inputs) {
baseInputStmt.setString(1, input.getName());
- baseInputStmt.setLong(2, formId);
+ baseInputStmt.setLong(2, configId);
baseInputStmt.setShort(3, inputIndex++);
baseInputStmt.setString(4, input.getType().name());
baseInputStmt.setBoolean(5, input.isSensitive());
@@ -2128,43 +2090,42 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
/**
- * Load forms and corresponding inputs from Derby database.
+ * Load configs and corresponding inputs from Derby database.
*
- * Use given prepared statements to load all forms and corresponding inputs
+ * Use given prepared statements to load all configs and corresponding inputs
* from Derby.
*
- * @param connectionForms List of link forms that will be filled up
- * @param jobForms Map with job forms that will be filled up
- * @param formFetchStmt Prepared statement for fetching forms
+ * @param driverConfig List of driver configs that will be filled up
+ * @param configFetchStatement Prepared statement for fetching configs
* @param inputFetchStmt Prepare statement for fetching inputs
+ * @param configPosition position of the config
* @throws SQLException In case of any failure on Derby side
*/
- public void loadFrameworkForms(List<MForm> connectionForms,
- List<MForm> jobForms,
- PreparedStatement formFetchStmt,
+ public void loadDriverConfigs(List<MConfig> driverConfig,
+ PreparedStatement configFetchStatement,
PreparedStatement inputFetchStmt,
- int formPosition) throws SQLException {
+ int configPosition) throws SQLException {
// Get list of structures from database
- ResultSet rsetForm = formFetchStmt.executeQuery();
- while (rsetForm.next()) {
- long formId = rsetForm.getLong(1);
- Long formConnectorId = rsetForm.getLong(2);
- String formName = rsetForm.getString(4);
- String formType = rsetForm.getString(5);
- int formIndex = rsetForm.getInt(6);
- List<MInput<?>> formInputs = new ArrayList<MInput<?>>();
+ ResultSet rsetConfig = configFetchStatement.executeQuery();
+ while (rsetConfig.next()) {
+ long configId = rsetConfig.getLong(1);
+ Long fromConnectorId = rsetConfig.getLong(2);
+ String configName = rsetConfig.getString(4);
+ String configTYpe = rsetConfig.getString(5);
+ int configIndex = rsetConfig.getInt(6);
+ List<MInput<?>> configInputs = new ArrayList<MInput<?>>();
- MForm mDriverConfig = new MForm(formName, formInputs);
- mDriverConfig.setPersistenceId(formId);
+ MConfig mDriverConfig = new MConfig(configName, configInputs);
+ mDriverConfig.setPersistenceId(configId);
- inputFetchStmt.setLong(formPosition, formId);
+ inputFetchStmt.setLong(configPosition, configId);
ResultSet rsetInput = inputFetchStmt.executeQuery();
while (rsetInput.next()) {
long inputId = rsetInput.getLong(1);
String inputName = rsetInput.getString(2);
- long inputForm = rsetInput.getLong(3);
+ long inputConfig = rsetInput.getLong(3);
short inputIndex = rsetInput.getShort(4);
String inputType = rsetInput.getString(5);
boolean inputSensitivity = rsetInput.getBoolean(6);
@@ -2194,7 +2155,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
default:
throw new SqoopException(DerbyRepoError.DERBYREPO_0006,
"input-" + inputName + ":" + inputId + ":"
- + "form-" + inputForm + ":" + mit.name());
+ + "config-" + inputConfig + ":" + mit.name());
}
// Set persistent ID
@@ -2209,7 +2170,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
if (mDriverConfig.getInputs().size() != inputIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0009,
- "form: " + mDriverConfig
+ "config: " + mDriverConfig
+ "; input: " + input
+ "; index: " + inputIndex
+ "; expected: " + mDriverConfig.getInputs().size()
@@ -2221,83 +2182,69 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
if (mDriverConfig.getInputs().size() == 0) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0008,
- "connector-" + formConnectorId
- + "; form: " + mDriverConfig
+ "owner-" + fromConnectorId
+ + "; config: " + mDriverConfig
);
}
- MFormType mDriverConfigt = MFormType.valueOf(formType);
- switch (mDriverConfigt) {
- case CONNECTION:
- if (connectionForms.size() != formIndex) {
- throw new SqoopException(DerbyRepoError.DERBYREPO_0010,
- "connector-" + formConnectorId
- + "; form: " + mDriverConfig
- + "; index: " + formIndex
- + "; expected: " + connectionForms.size()
- );
- }
- connectionForms.add(mDriverConfig);
- break;
+ MConfigType configType = MConfigType.valueOf(configTYpe);
+ switch (configType) {
case JOB:
- if (jobForms.size() != formIndex) {
+ if (driverConfig.size() != configIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0010,
- "connector-" + formConnectorId
- + "; form: " + mDriverConfig
- + "; index: " + formIndex
- + "; expected: " + jobForms.size()
+ "owner-" + fromConnectorId
+ + "; config: " + configType
+ + "; index: " + configIndex
+ + "; expected: " + driverConfig.size()
);
}
- jobForms.add(mDriverConfig);
+ driverConfig.add(mDriverConfig);
break;
default:
throw new SqoopException(DerbyRepoError.DERBYREPO_0007,
- "connector-" + formConnectorId + ":" + mDriverConfig);
+ "connector-" + fromConnectorId + ":" + configType);
}
}
}
/**
- * Load forms and corresponding inputs from Derby database.
+ * Load configs and corresponding inputs from Derby database.
*
- * Use given prepared statements to load all forms and corresponding inputs
+ * Use given prepared statements to load all configs and corresponding inputs
* from Derby.
*
- * @param connectionForms List of link forms that will be filled up
- * @param fromJobForms FROM job forms that will be filled up
- * @param toJobForms TO job forms that will be filled up
- * @param formFetchStmt Prepared statement for fetching forms
+ * @param linkConfig List of link configs that will be filled up
+ * @param fromConfig FROM job configs that will be filled up
+ * @param toConfig TO job configs that will be filled up
+ * @param configFetchStmt Prepared statement for fetching configs
* @param inputFetchStmt Prepare statement for fetching inputs
* @throws SQLException In case of any failure on Derby side
*/
- public void loadConnectorForms(List<MForm> connectionForms,
- List<MForm> fromJobForms,
- List<MForm> toJobForms,
- PreparedStatement formFetchStmt,
- PreparedStatement inputFetchStmt,
- int formPosition) throws SQLException {
+ public void loadConfigTypes(List<MConfig> linkConfig, List<MConfig> fromConfig,
+ List<MConfig> toConfig, PreparedStatement configFetchStmt, PreparedStatement inputFetchStmt,
+ int configPosition) throws SQLException {
// Get list of structures from database
- ResultSet rsetForm = formFetchStmt.executeQuery();
- while (rsetForm.next()) {
- long formId = rsetForm.getLong(1);
- Long formConnectorId = rsetForm.getLong(2);
- String operation = rsetForm.getString(3);
- String formName = rsetForm.getString(4);
- String formType = rsetForm.getString(5);
- int formIndex = rsetForm.getInt(6);
- List<MInput<?>> formInputs = new ArrayList<MInput<?>>();
+ ResultSet rsetConfig = configFetchStmt.executeQuery();
+ while (rsetConfig.next()) {
+ long configId = rsetConfig.getLong(1);
+ Long configConnectorId = rsetConfig.getLong(2);
+ String operation = rsetConfig.getString(3);
+ String configName = rsetConfig.getString(4);
+ String configType = rsetConfig.getString(5);
+ int configIndex = rsetConfig.getInt(6);
+ List<MInput<?>> configInputs = new ArrayList<MInput<?>>();
- MForm mDriverConfig = new MForm(formName, formInputs);
- mDriverConfig.setPersistenceId(formId);
+ MConfig config = new MConfig(configName, configInputs);
+ config.setPersistenceId(configId);
- inputFetchStmt.setLong(formPosition, formId);
+ inputFetchStmt.setLong(configPosition, configId);
ResultSet rsetInput = inputFetchStmt.executeQuery();
while (rsetInput.next()) {
long inputId = rsetInput.getLong(1);
String inputName = rsetInput.getString(2);
- long inputForm = rsetInput.getLong(3);
+ long inputConfig = rsetInput.getLong(3);
short inputIndex = rsetInput.getShort(4);
String inputType = rsetInput.getString(5);
boolean inputSensitivity = rsetInput.getBoolean(6);
@@ -2327,7 +2274,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
default:
throw new SqoopException(DerbyRepoError.DERBYREPO_0006,
"input-" + inputName + ":" + inputId + ":"
- + "form-" + inputForm + ":" + mit.name());
+ + "config-" + inputConfig + ":" + mit.name());
}
// Set persistent ID
@@ -2340,75 +2287,75 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
input.restoreFromUrlSafeValueString(value);
}
- if (mDriverConfig.getInputs().size() != inputIndex) {
+ if (config.getInputs().size() != inputIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0009,
- "form: " + mDriverConfig
+ "config: " + config
+ "; input: " + input
+ "; index: " + inputIndex
- + "; expected: " + mDriverConfig.getInputs().size()
+ + "; expected: " + config.getInputs().size()
);
}
- mDriverConfig.getInputs().add(input);
+ config.getInputs().add(input);
}
- if (mDriverConfig.getInputs().size() == 0) {
+ if (config.getInputs().size() == 0) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0008,
- "connector-" + formConnectorId
- + "; form: " + mDriverConfig
+ "connector-" + configConnectorId
+ + "; config: " + config
);
}
- MFormType mDriverConfigt = MFormType.valueOf(formType);
- switch (mDriverConfigt) {
- case CONNECTION:
- if (connectionForms.size() != formIndex) {
+ MConfigType mConfigType = MConfigType.valueOf(configType);
+ switch (mConfigType) {
+ case LINK:
+ if (linkConfig.size() != configIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0010,
- "connector-" + formConnectorId
- + "; form: " + mDriverConfig
- + "; index: " + formIndex
- + "; expected: " + connectionForms.size()
+ "connector-" + configConnectorId
+ + "; config: " + config
+ + "; index: " + configIndex
+ + "; expected: " + linkConfig.size()
);
}
- connectionForms.add(mDriverConfig);
+ linkConfig.add(config);
break;
case JOB:
Direction type = Direction.valueOf(operation);
- List<MForm> jobForms;
+ List<MConfig> jobConfigs;
switch(type) {
case FROM:
- jobForms = fromJobForms;
+ jobConfigs = fromConfig;
break;
case TO:
- jobForms = toJobForms;
+ jobConfigs = toConfig;
break;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
- if (jobForms.size() != formIndex) {
+ if (jobConfigs.size() != configIndex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0010,
- "connector-" + formConnectorId
- + "; form: " + mDriverConfig
- + "; index: " + formIndex
- + "; expected: " + jobForms.size()
+ "connector-" + configConnectorId
+ + "; config: " + config
+ + "; index: " + configIndex
+ + "; expected: " + jobConfigs.size()
);
}
- jobForms.add(mDriverConfig);
+ jobConfigs.add(config);
break;
default:
throw new SqoopException(DerbyRepoError.DERBYREPO_0007,
- "connector-" + formConnectorId + ":" + mDriverConfig);
+ "connector-" + configConnectorId + ":" + config);
}
}
}
private void createInputValues(String query,
long id,
- List<MForm> forms,
+ List<MConfig> configs,
Connection conn) throws SQLException {
PreparedStatement stmt = null;
int result;
@@ -2416,8 +2363,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
try {
stmt = conn.prepareStatement(query);
- for (MForm form : forms) {
- for (MInput input : form.getInputs()) {
+ for (MConfig config : configs) {
+ for (MInput input : config.getInputs()) {
// Skip empty values as we're not interested in storing those in db
if (input.isEmpty()) {
continue;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
index 58eed2d..fc3ec18 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
@@ -56,30 +56,30 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQC_VERSION = "SQC_VERSION";
- // SQ_FORM
+ // SQ_CONFIG
- public static final String TABLE_SQ_FORM_NAME = "SQ_FORM";
+ public static final String TABLE_SQ_CONFIG_NAME = "SQ_CONFIG";
- public static final String TABLE_SQ_FORM = SCHEMA_PREFIX
- + TABLE_SQ_FORM_NAME;
+ public static final String TABLE_SQ_CONFIG = SCHEMA_PREFIX
+ + TABLE_SQ_CONFIG_NAME;
- public static final String COLUMN_SQF_ID = "SQF_ID";
+ public static final String COLUMN_SQ_CFG_ID = "SQ_CFG_ID";
- public static final String COLUMN_SQF_CONNECTOR = "SQF_CONNECTOR";
+ public static final String COLUMN_SQ_CFG_OWNER = "SQ_CFG_OWNER";
- public static final String COLUMN_SQF_OPERATION = "SQF_OPERATION";
+ public static final String COLUMN_SQ_CFG_OPERATION = "SQ_CFG_OPERATION";
- public static final String COLUMN_SQF_DIRECTION = "SQF_DIRECTION";
+ public static final String COLUMN_SQ_CFG_DIRECTION = "SQ_CFG_DIRECTION";
- public static final String COLUMN_SQF_NAME = "SQF_NAME";
+ public static final String COLUMN_SQ_CFG_NAME = "SQ_CFG_NAME";
- public static final String COLUMN_SQF_TYPE = "SQF_TYPE";
+ public static final String COLUMN_SQ_CFG_TYPE = "SQ_CFG_TYPE";
- public static final String COLUMN_SQF_INDEX = "SQF_INDEX";
+ public static final String COLUMN_SQ_CFG_INDEX = "SQ_CFG_INDEX";
- public static final String CONSTRAINT_SQF_SQC_NAME = CONSTRAINT_PREFIX + "SQF_SQC";
+ public static final String CONSTRAINT_SQ_CFG_SQC_NAME = CONSTRAINT_PREFIX + "SQ_CFG_SQC";
- public static final String CONSTRAINT_SQF_SQC = SCHEMA_PREFIX + CONSTRAINT_SQF_SQC_NAME;
+ public static final String CONSTRAINT_SQ_CFG_SQC = SCHEMA_PREFIX + CONSTRAINT_SQ_CFG_SQC_NAME;
// SQ_INPUT
@@ -92,7 +92,7 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQI_NAME = "SQI_NAME";
- public static final String COLUMN_SQI_FORM = "SQI_FORM";
+ public static final String COLUMN_SQI_CONFIG = "SQI_CONFIG";
public static final String COLUMN_SQI_INDEX = "SQI_INDEX";
@@ -104,36 +104,34 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQI_ENUMVALS = "SQI_ENUMVALS";
- public static final String CONSTRAINT_SQI_SQF_NAME = CONSTRAINT_PREFIX + "SQI_SQF";
+ public static final String CONSTRAINT_SQI_SQ_CFG_NAME = CONSTRAINT_PREFIX + "SQI_SQ_CFG";
- public static final String CONSTRAINT_SQI_SQF = SCHEMA_PREFIX + CONSTRAINT_SQI_SQF_NAME;
+ public static final String CONSTRAINT_SQI_SQ_CFG = SCHEMA_PREFIX + CONSTRAINT_SQI_SQ_CFG_NAME;
- // SQ_CONNECTION
+ public static final String TABLE_SQ_LINK_NAME = "SQ_LINK";
- public static final String TABLE_SQ_CONNECTION_NAME = "SQ_CONNECTION";
+ public static final String TABLE_SQ_LINK = SCHEMA_PREFIX
+ + TABLE_SQ_LINK_NAME;
- public static final String TABLE_SQ_CONNECTION = SCHEMA_PREFIX
- + TABLE_SQ_CONNECTION_NAME;
+ public static final String COLUMN_SQ_LNK_ID = "SQ_LNK_ID";
- public static final String COLUMN_SQN_ID = "SQN_ID";
+ public static final String COLUMN_SQ_LNK_NAME = "SQ_LNK_NAME";
- public static final String COLUMN_SQN_NAME = "SQN_NAME";
+ public static final String COLUMN_SQ_LNK_CONNECTOR = "SQ_LNK_CONNECTOR";
- public static final String COLUMN_SQN_CONNECTOR = "SQN_CONNECTOR";
+ public static final String COLUMN_SQ_LNK_CREATION_USER = "SQ_LNK_CREATION_USER";
- public static final String COLUMN_SQN_CREATION_USER = "SQN_CREATION_USER";
+ public static final String COLUMN_SQ_LNK_CREATION_DATE = "SQ_LNK_CREATION_DATE";
- public static final String COLUMN_SQN_CREATION_DATE = "SQN_CREATION_DATE";
+ public static final String COLUMN_SQ_LNK_UPDATE_USER = "SQ_LNK_UPDATE_USER";
- public static final String COLUMN_SQN_UPDATE_USER = "SQN_UPDATE_USER";
+ public static final String COLUMN_SQ_LNK_UPDATE_DATE = "SQ_LNK_UPDATE_DATE";
- public static final String COLUMN_SQN_UPDATE_DATE = "SQN_UPDATE_DATE";
+ public static final String COLUMN_SQ_LNK_ENABLED = "SQ_LNK_ENABLED";
- public static final String COLUMN_SQN_ENABLED = "SQN_ENABLED";
+ public static final String CONSTRAINT_SQ_LNK_SQC_NAME = CONSTRAINT_PREFIX + "SQ_LNK_SQC";
- public static final String CONSTRAINT_SQN_SQC_NAME = CONSTRAINT_PREFIX + "SQN_SQC";
-
- public static final String CONSTRAINT_SQN_SQC = SCHEMA_PREFIX + CONSTRAINT_SQN_SQC_NAME;
+ public static final String CONSTRAINT_SQ_LNK_SQC = SCHEMA_PREFIX + CONSTRAINT_SQ_LNK_SQC_NAME;
// SQ_JOB
@@ -146,13 +144,13 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQB_NAME = "SQB_NAME";
- public static final String COLUMN_SQB_CONNECTION = "SQB_CONNECTION";
+ public static final String COLUMN_SQB_LINK = "SQB_LINK";
public static final String COLUMN_SQB_TYPE = "SQB_TYPE";
- public static final String COLUMN_SQB_FROM_CONNECTION = "SQB_FROM_CONNECTION";
+ public static final String COLUMN_SQB_FROM_LINK = "SQB_FROM_LINK";
- public static final String COLUMN_SQB_TO_CONNECTION = "SQB_TO_CONNECTION";
+ public static final String COLUMN_SQB_TO_LINK = "SQB_TO_LINK";
public static final String COLUMN_SQB_CREATION_USER = "SQB_CREATION_USER";
@@ -164,39 +162,37 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQB_ENABLED = "SQB_ENABLED";
- public static final String CONSTRAINT_SQB_SQN_NAME = CONSTRAINT_PREFIX + "SQB_SQN";
-
- public static final String CONSTRAINT_SQB_SQN = SCHEMA_PREFIX + CONSTRAINT_SQB_SQN_NAME;
+ public static final String CONSTRAINT_SQB_SQ_LNK_NAME = CONSTRAINT_PREFIX + "SQB_SQ_LNK";
- public static final String CONSTRAINT_SQB_SQN_FROM_NAME = CONSTRAINT_PREFIX + "SQB_SQN_FROM";
+ public static final String CONSTRAINT_SQB_SQ_LNK = SCHEMA_PREFIX + CONSTRAINT_SQB_SQ_LNK_NAME;
- public static final String CONSTRAINT_SQB_SQN_FROM = SCHEMA_PREFIX + CONSTRAINT_SQB_SQN_FROM_NAME;
+ public static final String CONSTRAINT_SQB_SQ_LNK_FROM_NAME = CONSTRAINT_PREFIX + "SQB_SQ_LNK_FROM";
- public static final String CONSTRAINT_SQB_SQN_TO_NAME = CONSTRAINT_PREFIX + "SQB_SQN_TO";
+ public static final String CONSTRAINT_SQB_SQ_LNK_FROM = SCHEMA_PREFIX + CONSTRAINT_SQB_SQ_LNK_FROM_NAME;
- public static final String CONSTRAINT_SQB_SQN_TO = SCHEMA_PREFIX + CONSTRAINT_SQB_SQN_TO_NAME;
+ public static final String CONSTRAINT_SQB_SQ_LNK_TO_NAME = CONSTRAINT_PREFIX + "SQB_SQ_LNK_TO";
- // SQ_CONNECTION_INPUT
+ public static final String CONSTRAINT_SQB_SQ_LNK_TO = SCHEMA_PREFIX + CONSTRAINT_SQB_SQ_LNK_TO_NAME;
- public static final String TABLE_SQ_CONNECTION_INPUT_NAME =
- "SQ_CONNECTION_INPUT";
+ public static final String TABLE_SQ_LINK_INPUT_NAME =
+ "SQ_LINK_INPUT";
- public static final String TABLE_SQ_CONNECTION_INPUT = SCHEMA_PREFIX
- + TABLE_SQ_CONNECTION_INPUT_NAME;
+ public static final String TABLE_SQ_LINK_INPUT = SCHEMA_PREFIX
+ + TABLE_SQ_LINK_INPUT_NAME;
- public static final String COLUMN_SQNI_CONNECTION = "SQNI_CONNECTION";
+ public static final String COLUMN_SQ_LNKI_LINK = "SQ_LNKI_LINK";
- public static final String COLUMN_SQNI_INPUT = "SQNI_INPUT";
+ public static final String COLUMN_SQ_LNKI_INPUT = "SQ_LNKI_INPUT";
- public static final String COLUMN_SQNI_VALUE = "SQNI_VALUE";
+ public static final String COLUMN_SQ_LNKI_VALUE = "SQ_LNKI_VALUE";
- public static final String CONSTRAINT_SQNI_SQN_NAME = CONSTRAINT_PREFIX + "SQNI_SQN";
+ public static final String CONSTRAINT_SQ_LNKI_SQ_LNK_NAME = CONSTRAINT_PREFIX + "SQ_LNKI_SQ_LNK";
- public static final String CONSTRAINT_SQNI_SQN = SCHEMA_PREFIX + CONSTRAINT_SQNI_SQN_NAME;
+ public static final String CONSTRAINT_SQ_LNKI_SQ_LNK = SCHEMA_PREFIX + CONSTRAINT_SQ_LNKI_SQ_LNK_NAME;
- public static final String CONSTRAINT_SQNI_SQI_NAME = CONSTRAINT_PREFIX + "SQNI_SQI";
+ public static final String CONSTRAINT_SQ_LNKI_SQI_NAME = CONSTRAINT_PREFIX + "SQ_LNKI_SQI";
- public static final String CONSTRAINT_SQNI_SQI = SCHEMA_PREFIX + CONSTRAINT_SQNI_SQI_NAME;
+ public static final String CONSTRAINT_SQ_LNKI_SQI = SCHEMA_PREFIX + CONSTRAINT_SQ_LNKI_SQI_NAME;
// SQ_JOB_INPUT
@@ -314,12 +310,12 @@ public final class DerbySchemaConstants {
static {
tablesV1 = new HashSet<String>();
tablesV1.add(TABLE_SQ_CONNECTOR_NAME);
- tablesV1.add(TABLE_SQ_CONNECTION_NAME);
- tablesV1.add(TABLE_SQ_CONNECTION_INPUT_NAME);
+ tablesV1.add(TABLE_SQ_LINK_NAME);
+ tablesV1.add(TABLE_SQ_LINK_INPUT_NAME);
tablesV1.add(TABLE_SQ_COUNTER_NAME);
tablesV1.add(TABLE_SQ_COUNTER_GROUP_NAME);
tablesV1.add(TABLE_SQ_COUNTER_SUBMISSION_NAME);
- tablesV1.add(TABLE_SQ_FORM_NAME);
+ tablesV1.add(TABLE_SQ_CONFIG_NAME);
tablesV1.add(TABLE_SQ_INPUT_NAME);
tablesV1.add(TABLE_SQ_JOB_NAME);
tablesV1.add(TABLE_SQ_JOB_INPUT_NAME);
[03/52] [abbrv] git commit: SQOOP-1374: From/To: Metadata upgrade
Posted by ab...@apache.org.
SQOOP-1374: From/To: Metadata upgrade
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/51a07bc3
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/51a07bc3
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/51a07bc3
Branch: refs/heads/SQOOP-1367
Commit: 51a07bc352dff37e9482744d272fc54112a1861c
Parents: cd882a9
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Mon Sep 1 15:15:53 2014 +0200
Committer: Abraham Elmahrek <ab...@elmahrek.com>
Committed: Thu Oct 9 17:58:17 2014 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/model/MConnection.java | 1 +
.../main/java/org/apache/sqoop/model/MJob.java | 1 +
.../GenericJdbcConnectorMetadataUpgrader.java | 8 +-
.../connector/jdbc/GenericJdbcValidator.java | 4 +-
.../sqoop/connector/hdfs/HdfsConnector.java | 2 +-
.../connector/hdfs/HdfsMetadataUpgrader.java | 83 ++
.../sqoop/connector/ConnectorManager.java | 53 +-
.../sqoop/connector/ConnectorManagerUtils.java | 70 ++
.../apache/sqoop/repository/JdbcRepository.java | 12 +-
.../org/apache/sqoop/repository/Repository.java | 54 +-
.../TestFrameworkMetadataUpgrader.java | 270 ++---
.../repository/derby/DerbyRepoConstants.java | 4 +-
.../derby/DerbyRepositoryHandler.java | 258 ++++-
.../repository/derby/DerbySchemaConstants.java | 14 +
.../repository/derby/DerbySchemaQuery.java | 218 +++-
.../sqoop/repository/derby/DerbyTestCase.java | 1084 +++++++++++-------
.../derby/TestConnectionHandling.java | 418 +++----
.../repository/derby/TestConnectorHandling.java | 132 +--
.../repository/derby/TestFrameworkHandling.java | 193 ++--
.../sqoop/repository/derby/TestInputTypes.java | 206 ++--
.../sqoop/repository/derby/TestInternals.java | 62 +-
.../sqoop/repository/derby/TestJobHandling.java | 484 ++++----
.../derby/TestSubmissionHandling.java | 420 +++----
.../sqoop/connector/spi/MetadataUpgrader.java | 3 +-
24 files changed, 2386 insertions(+), 1668 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/common/src/main/java/org/apache/sqoop/model/MConnection.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MConnection.java b/common/src/main/java/org/apache/sqoop/model/MConnection.java
index e5a4fb8..f84abbf 100644
--- a/common/src/main/java/org/apache/sqoop/model/MConnection.java
+++ b/common/src/main/java/org/apache/sqoop/model/MConnection.java
@@ -66,6 +66,7 @@ public class MConnection extends MAccountableEntity implements MClonable {
this.connectorId = other.connectorId;
this.connectorPart = connectorPart;
this.frameworkPart = frameworkPart;
+ this.setPersistenceId(other.getPersistenceId());
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/common/src/main/java/org/apache/sqoop/model/MJob.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MJob.java b/common/src/main/java/org/apache/sqoop/model/MJob.java
index 11839fc..182bbfb 100644
--- a/common/src/main/java/org/apache/sqoop/model/MJob.java
+++ b/common/src/main/java/org/apache/sqoop/model/MJob.java
@@ -106,6 +106,7 @@ public class MJob extends MAccountableEntity implements MClonable {
this.fromConnectorPart = fromPart;
this.toConnectorPart = toPart;
this.frameworkPart = frameworkPart;
+ this.setPersistenceId(other.getPersistenceId());
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java
index 2b12009..cbe72f6 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java
@@ -49,7 +49,6 @@ public class GenericJdbcConnectorMetadataUpgrader extends MetadataUpgrader {
@Override
public void upgrade(MJobForms original, MJobForms upgradeTarget) {
doUpgrade(original.getForms(), upgradeTarget.getForms());
-
}
@SuppressWarnings("unchecked")
@@ -65,12 +64,17 @@ public class GenericJdbcConnectorMetadataUpgrader extends MetadataUpgrader {
for (MForm form : target) {
List<MInput<?>> inputs = form.getInputs();
MForm originalForm = formMap.get(form.getName());
+ if (originalForm == null) {
+ LOG.warn("Form: '" + form.getName() + "' not present in old " +
+ "connector. So it and its inputs will not be transferred by the upgrader.");
+ continue;
+ }
for (MInput input : inputs) {
try {
MInput originalInput = originalForm.getInput(input.getName());
input.setValue(originalInput.getValue());
} catch (SqoopException ex) {
- LOG.warn("Input: " + input.getName() + " not present in old " +
+ LOG.warn("Input: '" + input.getName() + "' not present in old " +
"connector. So it will not be transferred by the upgrader.");
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
index eea86b2..0a60e90 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
@@ -79,7 +79,7 @@ public class GenericJdbcValidator extends Validator {
}
private Validation validateToJobConfiguration(ToJobConfiguration configuration) {
- Validation validation = new Validation(ToJobConfiguration.class);
+ Validation validation = new Validation(FromJobConfiguration.class);
if(configuration.toTable.tableName == null && configuration.toTable.sql == null) {
validation.addMessage(Status.UNACCEPTABLE, "toTable", "Either table name or SQL must be specified");
@@ -103,7 +103,7 @@ public class GenericJdbcValidator extends Validator {
}
private Validation validateFromJobConfiguration(FromJobConfiguration configuration) {
- Validation validation = new Validation(ToJobConfiguration.class);
+ Validation validation = new Validation(FromJobConfiguration.class);
if(configuration.fromTable.tableName == null && configuration.fromTable.sql == null) {
validation.addMessage(Status.UNACCEPTABLE, "fromTable", "Either table name or SQL must be specified");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
index 557091e..883636c 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
@@ -127,6 +127,6 @@ public class HdfsConnector extends SqoopConnector {
*/
@Override
public MetadataUpgrader getMetadataUpgrader() {
- return null;
+ return new HdfsMetadataUpgrader();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsMetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsMetadataUpgrader.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsMetadataUpgrader.java
new file mode 100644
index 0000000..3e51e38
--- /dev/null
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsMetadataUpgrader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sqoop.connector.hdfs;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.spi.MetadataUpgrader;
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MJobForms;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class HdfsMetadataUpgrader extends MetadataUpgrader {
+ private static final Logger LOG =
+ Logger.getLogger(HdfsMetadataUpgrader.class);
+
+ /*
+ * For now, there is no real upgrade. So copy all data over,
+ * set the validation messages and error messages to be the same as for the
+ * inputs in the original one.
+ */
+
+ @Override
+ public void upgrade(MConnectionForms original,
+ MConnectionForms upgradeTarget) {
+ doUpgrade(original.getForms(), upgradeTarget.getForms());
+ }
+
+ @Override
+ public void upgrade(MJobForms original, MJobForms upgradeTarget) {
+ doUpgrade(original.getForms(), upgradeTarget.getForms());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void doUpgrade(List<MForm> original, List<MForm> target) {
+ // Easier to find the form in the original forms list if we use a map.
+ // Since the constructor of MJobForms takes a list,
+ // index is not guaranteed to be the same, so we need to look for
+ // equivalence
+ Map<String, MForm> formMap = new HashMap<String, MForm>();
+ for (MForm form : original) {
+ formMap.put(form.getName(), form);
+ }
+ for (MForm form : target) {
+ List<MInput<?>> inputs = form.getInputs();
+ MForm originalForm = formMap.get(form.getName());
+ if (originalForm == null) {
+ LOG.warn("Form: '" + form.getName() + "' not present in old " +
+ "connector. So it and its inputs will not be transferred by the upgrader.");
+ continue;
+ }
+ for (MInput input : inputs) {
+ try {
+ MInput originalInput = originalForm.getInput(input.getName());
+ input.setValue(originalInput.getValue());
+ } catch (SqoopException ex) {
+ LOG.warn("Input: '" + input.getName() + "' not present in old " +
+ "connector. So it will not be transferred by the upgrader.");
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
index b92ff4d..db6f579 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
@@ -17,10 +17,7 @@
*/
package org.apache.sqoop.connector;
-import java.io.IOException;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -150,48 +147,22 @@ public class ConnectorManager implements Reconfigurable {
LOG.trace("Begin connector manager initialization");
}
- List<URL> connectorConfigs = new ArrayList<URL>();
+ List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs();
- try {
- Enumeration<URL> appPathConfigs =
- ConnectorManager.class.getClassLoader().getResources(
- ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
-
- while (appPathConfigs.hasMoreElements()) {
- connectorConfigs.add(appPathConfigs.nextElement());
- }
-
- ClassLoader ctxLoader = Thread.currentThread().getContextClassLoader();
-
- if (ctxLoader != null) {
- Enumeration<URL> ctxPathConfigs = ctxLoader.getResources(
- ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
-
- while (ctxPathConfigs.hasMoreElements()) {
- URL configUrl = ctxPathConfigs.nextElement();
- if (!connectorConfigs.contains(configUrl)) {
- connectorConfigs.add(configUrl);
- }
- }
- }
-
- LOG.info("Connector config urls: " + connectorConfigs);
+ LOG.info("Connector config urls: " + connectorConfigs);
- if (connectorConfigs.size() == 0) {
- throw new SqoopException(ConnectorError.CONN_0002);
- }
+ if (connectorConfigs.size() == 0) {
+ throw new SqoopException(ConnectorError.CONN_0002);
+ }
- for (URL url : connectorConfigs) {
- ConnectorHandler handler = new ConnectorHandler(url);
- ConnectorHandler handlerOld =
- handlerMap.put(handler.getUniqueName(), handler);
- if (handlerOld != null) {
- throw new SqoopException(ConnectorError.CONN_0006,
- handler + ", " + handlerOld);
- }
+ for (URL url : connectorConfigs) {
+ ConnectorHandler handler = new ConnectorHandler(url);
+ ConnectorHandler handlerOld =
+ handlerMap.put(handler.getUniqueName(), handler);
+ if (handlerOld != null) {
+ throw new SqoopException(ConnectorError.CONN_0006,
+ handler + ", " + handlerOld);
}
- } catch (IOException ex) {
- throw new SqoopException(ConnectorError.CONN_0001, ex);
}
registerConnectors(autoUpgrade);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
new file mode 100644
index 0000000..c7193ee
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector;
+
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.core.ConfigurationConstants;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+
+/**
+ * Utilities for ConnectorManager.
+ */
+public class ConnectorManagerUtils {
+
+ /**
+ * Get a list of URLs of connectors that are installed.
+ * Check
+ * @return List of URLs.
+ */
+ public static List<URL> getConnectorConfigs() {
+ List<URL> connectorConfigs = new ArrayList<URL>();
+
+ try {
+ // Check ConnectorManager classloader.
+ Enumeration<URL> appPathConfigs =
+ ConnectorManager.class.getClassLoader().getResources(
+ ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
+ while (appPathConfigs.hasMoreElements()) {
+ connectorConfigs.add(appPathConfigs.nextElement());
+ }
+
+ // Check thread context classloader.
+ ClassLoader ctxLoader = Thread.currentThread().getContextClassLoader();
+ if (ctxLoader != null) {
+ Enumeration<URL> ctxPathConfigs = ctxLoader.getResources(
+ ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
+
+ while (ctxPathConfigs.hasMoreElements()) {
+ URL configUrl = ctxPathConfigs.nextElement();
+ if (!connectorConfigs.contains(configUrl)) {
+ connectorConfigs.add(configUrl);
+ }
+ }
+ }
+ } catch (IOException ex) {
+ throw new SqoopException(ConnectorError.CONN_0001, ex);
+ }
+
+ return connectorConfigs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
index 9b64661..fa119a5 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
@@ -207,12 +207,12 @@ public class JdbcRepository extends Repository {
*/
@Override
public List<MConnector> findConnectors() {
- return (List<MConnector>) doWithConnection(new DoWithConnection() {
- @Override
- public Object doIt(Connection conn) {
- return handler.findConnectors(conn);
- }
- });
+ return (List<MConnector>) doWithConnection(new DoWithConnection() {
+ @Override
+ public Object doIt(Connection conn) {
+ return handler.findConnectors(conn);
+ }
+ });
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/core/src/main/java/org/apache/sqoop/repository/Repository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java
index e9c32e0..8e8dd80 100644
--- a/core/src/main/java/org/apache/sqoop/repository/Repository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java
@@ -445,24 +445,48 @@ public abstract class Repository {
for (MJob job : jobs) {
// Make a new copy of the forms from the connector,
// else the values will get set in the forms in the connector for
- // each connection.
- List<MForm> forms = newConnector.getJobForms(Direction.FROM).clone(false).getForms();
- MJobForms newJobForms = new MJobForms(forms);
- upgrader.upgrade(job.getConnectorPart(Direction.FROM), newJobForms);
- // @TODO(Abe): Check From and To
- MJob newJob = new MJob(job, newJobForms, newJobForms, job.getFrameworkPart());
+ // each job.
+ List<MForm> fromForms = newConnector.getJobForms(Direction.FROM).clone(false).getForms();
+ List<MForm> toForms = newConnector.getJobForms(Direction.TO).clone(false).getForms();
+
+ // New FROM direction forms, old TO direction forms.
+ if (job.getConnectorId(Direction.FROM) == newConnector.getPersistenceId()) {
+ MJobForms newFromJobForms = new MJobForms(fromForms);
+ MJobForms oldToJobForms = job.getConnectorPart(Direction.TO);
+ upgrader.upgrade(job.getConnectorPart(Direction.FROM), newFromJobForms);
+ MJob newJob = new MJob(job, newFromJobForms, oldToJobForms, job.getFrameworkPart());
+ updateJob(newJob, tx);
- // Transform form structures to objects for validations
- // @TODO(Abe): Check From and To
- Object newConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.FROM));
- FormUtils.fromForms(newJob.getConnectorPart(Direction.FROM).getForms(), newConfigurationObject);
+ // Transform form structures to objects for validations
+// Object newFromConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.FROM));
+// FormUtils.fromForms(newJob.getConnectorPart(Direction.FROM).getForms(), newFromConfigurationObject);
+// Validation fromValidation = validator.validateJob(newFromConfigurationObject);
+// if (fromValidation.getStatus().canProceed()) {
+// updateJob(newJob, tx);
+// } else {
+// logInvalidModelObject("job", newJob, fromValidation);
+// upgradeSuccessful = false;
+// }
+ }
- Validation validation = validator.validateJob(newConfigurationObject);
- if (validation.getStatus().canProceed()) {
+ // Old FROM direction forms, new TO direction forms.
+ if (job.getConnectorId(Direction.TO) == newConnector.getPersistenceId()) {
+ MJobForms oldFromJobForms = job.getConnectorPart(Direction.FROM);
+ MJobForms newToJobForms = new MJobForms(toForms);
+ upgrader.upgrade(job.getConnectorPart(Direction.TO), newToJobForms);
+ MJob newJob = new MJob(job, oldFromJobForms, newToJobForms, job.getFrameworkPart());
updateJob(newJob, tx);
- } else {
- logInvalidModelObject("job", newJob, validation);
- upgradeSuccessful = false;
+
+ // Transform form structures to objects for validations
+// Object newToConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.TO));
+// FormUtils.fromForms(newJob.getConnectorPart(Direction.TO).getForms(), newToConfigurationObject);
+// Validation toValidation = validator.validateJob(newToConfigurationObject);
+// if (toValidation.getStatus().canProceed()) {
+// updateJob(newJob, tx);
+// } else {
+// logInvalidModelObject("job", newJob, toValidation);
+// upgradeSuccessful = false;
+// }
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java
index e0c4561..81d197e 100644
--- a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java
+++ b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java
@@ -32,139 +32,139 @@ import static org.junit.Assert.assertNull;
*/
public class TestFrameworkMetadataUpgrader {
-// FrameworkMetadataUpgrader upgrader;
-//
-// @Before
-// public void initializeUpgrader() {
-// upgrader = new FrameworkMetadataUpgrader();
-// }
-//
-// /**
-// * We take the same forms on input and output and we
-// * expect that all values will be correctly transferred.
-// */
-// @Test
-// public void testConnectionUpgrade() {
-// MConnectionForms original = connection1();
-// MConnectionForms target = connection1();
-//
-// original.getStringInput("f1.s1").setValue("A");
-// original.getStringInput("f1.s2").setValue("B");
-// original.getIntegerInput("f1.i").setValue(3);
-//
-// upgrader.upgrade(original, target);
-//
-// assertEquals("A", target.getStringInput("f1.s1").getValue());
-// assertEquals("B", target.getStringInput("f1.s2").getValue());
-// assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
-// }
-//
-// /**
-// * We take the same forms on input and output and we
-// * expect that all values will be correctly transferred.
-// */
-// @Test
-// public void testJobUpgrade() {
-// MJobForms original = job1(MJob.Type.IMPORT);
-// MJobForms target = job1(MJob.Type.IMPORT);
-//
-// original.getStringInput("f1.s1").setValue("A");
-// original.getStringInput("f1.s2").setValue("B");
-// original.getIntegerInput("f1.i").setValue(3);
-//
-// upgrader.upgrade(original, target);
-//
-// assertEquals("A", target.getStringInput("f1.s1").getValue());
-// assertEquals("B", target.getStringInput("f1.s2").getValue());
-// assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
-// }
-//
-// /**
-// * Upgrade scenario when new input has been added to the target forms.
-// */
-// @Test
-// public void testNonExistingInput() {
-// MConnectionForms original = connection1();
-// MConnectionForms target = connection2();
-//
-// original.getStringInput("f1.s1").setValue("A");
-// original.getStringInput("f1.s2").setValue("B");
-// original.getIntegerInput("f1.i").setValue(3);
-//
-// upgrader.upgrade(original, target);
-//
-// assertEquals("A", target.getStringInput("f1.s1").getValue());
-// assertNull(target.getStringInput("f1.s2_").getValue());
-// assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
-// }
-//
-// /**
-// * Upgrade scenario when entire has been added in the target and
-// * therefore is missing in the original.
-// */
-// @Test
-// public void testNonExistingForm() {
-// MConnectionForms original = connection1();
-// MConnectionForms target = connection3();
-//
-// original.getStringInput("f1.s1").setValue("A");
-// original.getStringInput("f1.s2").setValue("B");
-// original.getIntegerInput("f1.i").setValue(3);
-//
-// upgrader.upgrade(original, target);
-//
-// assertNull(target.getStringInput("f2.s1").getValue());
-// assertNull(target.getStringInput("f2.s2").getValue());
-// assertNull(target.getIntegerInput("f2.i").getValue());
-// }
-//
-// MJobForms job1(MJob.Type type) {
-// return new MJobForms(type, forms1());
-// }
-//
-// MConnectionForms connection1() {
-// return new MConnectionForms(forms1());
-// }
-//
-// MConnectionForms connection2() {
-// return new MConnectionForms(forms2());
-// }
-//
-// MConnectionForms connection3() {
-// return new MConnectionForms(forms3());
-// }
-//
-// List<MForm> forms1() {
-// List<MForm> list = new LinkedList<MForm>();
-// list.add(new MForm("f1", inputs1("f1")));
-// return list;
-// }
-//
-// List<MInput<?>> inputs1(String formName) {
-// List<MInput<?>> list = new LinkedList<MInput<?>>();
-// list.add(new MStringInput(formName + ".s1", false, (short)30));
-// list.add(new MStringInput(formName + ".s2", false, (short)30));
-// list.add(new MIntegerInput(formName + ".i", false));
-// return list;
-// }
-//
-// List<MForm> forms2() {
-// List<MForm> list = new LinkedList<MForm>();
-// list.add(new MForm("f1", inputs2("f1")));
-// return list;
-// }
-//
-// List<MInput<?>> inputs2(String formName) {
-// List<MInput<?>> list = new LinkedList<MInput<?>>();
-// list.add(new MStringInput(formName + ".s1", false, (short)30));
-// list.add(new MStringInput(formName + ".s2_", false, (short)30));
-// list.add(new MIntegerInput(formName + ".i", false));
-// return list;
-// }
-//
-// List<MForm> forms3() {
-// List<MForm> list = new LinkedList<MForm>();
-// list.add(new MForm("f2", inputs1("f2")));
-// return list;
-// }
+ FrameworkMetadataUpgrader upgrader;
+
+ @Before
+ public void initializeUpgrader() {
+ upgrader = new FrameworkMetadataUpgrader();
+ }
+
+ /**
+ * We take the same forms on input and output and we
+ * expect that all values will be correctly transferred.
+ */
+ @Test
+ public void testConnectionUpgrade() {
+ MConnectionForms original = connection1();
+ MConnectionForms target = connection1();
+
+ original.getStringInput("f1.s1").setValue("A");
+ original.getStringInput("f1.s2").setValue("B");
+ original.getIntegerInput("f1.i").setValue(3);
+
+ upgrader.upgrade(original, target);
+
+ assertEquals("A", target.getStringInput("f1.s1").getValue());
+ assertEquals("B", target.getStringInput("f1.s2").getValue());
+ assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
+ }
+
+ /**
+ * We take the same forms on input and output and we
+ * expect that all values will be correctly transferred.
+ */
+ @Test
+ public void testJobUpgrade() {
+ MJobForms original = job1();
+ MJobForms target = job1();
+
+ original.getStringInput("f1.s1").setValue("A");
+ original.getStringInput("f1.s2").setValue("B");
+ original.getIntegerInput("f1.i").setValue(3);
+
+ upgrader.upgrade(original, target);
+
+ assertEquals("A", target.getStringInput("f1.s1").getValue());
+ assertEquals("B", target.getStringInput("f1.s2").getValue());
+ assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
+ }
+
+ /**
+ * Upgrade scenario when new input has been added to the target forms.
+ */
+ @Test
+ public void testNonExistingInput() {
+ MConnectionForms original = connection1();
+ MConnectionForms target = connection2();
+
+ original.getStringInput("f1.s1").setValue("A");
+ original.getStringInput("f1.s2").setValue("B");
+ original.getIntegerInput("f1.i").setValue(3);
+
+ upgrader.upgrade(original, target);
+
+ assertEquals("A", target.getStringInput("f1.s1").getValue());
+ assertNull(target.getStringInput("f1.s2_").getValue());
+ assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
+ }
+
+ /**
+ * Upgrade scenario when entire has been added in the target and
+ * therefore is missing in the original.
+ */
+ @Test
+ public void testNonExistingForm() {
+ MConnectionForms original = connection1();
+ MConnectionForms target = connection3();
+
+ original.getStringInput("f1.s1").setValue("A");
+ original.getStringInput("f1.s2").setValue("B");
+ original.getIntegerInput("f1.i").setValue(3);
+
+ upgrader.upgrade(original, target);
+
+ assertNull(target.getStringInput("f2.s1").getValue());
+ assertNull(target.getStringInput("f2.s2").getValue());
+ assertNull(target.getIntegerInput("f2.i").getValue());
+ }
+
+ MJobForms job1() {
+ return new MJobForms(forms1());
+ }
+
+ MConnectionForms connection1() {
+ return new MConnectionForms(forms1());
+ }
+
+ MConnectionForms connection2() {
+ return new MConnectionForms(forms2());
+ }
+
+ MConnectionForms connection3() {
+ return new MConnectionForms(forms3());
+ }
+
+ List<MForm> forms1() {
+ List<MForm> list = new LinkedList<MForm>();
+ list.add(new MForm("f1", inputs1("f1")));
+ return list;
+ }
+
+ List<MInput<?>> inputs1(String formName) {
+ List<MInput<?>> list = new LinkedList<MInput<?>>();
+ list.add(new MStringInput(formName + ".s1", false, (short)30));
+ list.add(new MStringInput(formName + ".s2", false, (short)30));
+ list.add(new MIntegerInput(formName + ".i", false));
+ return list;
+ }
+
+ List<MForm> forms2() {
+ List<MForm> list = new LinkedList<MForm>();
+ list.add(new MForm("f1", inputs2("f1")));
+ return list;
+ }
+
+ List<MInput<?>> inputs2(String formName) {
+ List<MInput<?>> list = new LinkedList<MInput<?>>();
+ list.add(new MStringInput(formName + ".s1", false, (short)30));
+ list.add(new MStringInput(formName + ".s2_", false, (short)30));
+ list.add(new MIntegerInput(formName + ".i", false));
+ return list;
+ }
+
+ List<MForm> forms3() {
+ List<MForm> list = new LinkedList<MForm>();
+ list.add(new MForm("f2", inputs1("f2")));
+ return list;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java
index 030dde7..fdcecf2 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java
@@ -40,8 +40,10 @@ public final class DerbyRepoConstants {
* 3 - Version 1.99.4
* SQ_SUBMISSION modified SQS_EXTERNAL_ID varchar(50)
* Increased size of SQ_CONNECTOR.SQC_VERSION to 64
+ * 4 - Version 1.99.4
+ * Changed to FROM/TO design.
*/
- public static final int VERSION = 3;
+ public static final int VERSION = 4;
private DerbyRepoConstants() {
// Disable explicit object creation
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index 88be9fb..68aea9c 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -19,6 +19,7 @@ package org.apache.sqoop.repository.derby;
import static org.apache.sqoop.repository.derby.DerbySchemaQuery.*;
+import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@@ -36,6 +37,8 @@ import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.DirectionError;
import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.ConnectorHandler;
+import org.apache.sqoop.connector.ConnectorManagerUtils;
import org.apache.sqoop.model.MBooleanInput;
import org.apache.sqoop.model.MConnection;
import org.apache.sqoop.model.MConnectionForms;
@@ -73,6 +76,14 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
private static final String EMBEDDED_DERBY_DRIVER_CLASSNAME =
"org.apache.derby.jdbc.EmbeddedDriver";
+ /**
+ * Unique name of HDFS Connector.
+ * HDFS Connector was originally part of the Sqoop framework, but now is its
+ * own connector. This constant is used to pre-register the HDFS Connector
+ * so that jobs that are being upgraded can reference the HDFS Connector.
+ */
+ private static final String CONNECTOR_HDFS = "hdfs-connector";
+
private JdbcRepositoryContext repoContext;
private DataSource dataSource;
private JdbcRepositoryTransactionFactory txFactory;
@@ -391,6 +402,25 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_MODIFY_COLUMN_SQS_EXTERNAL_ID_VARCHAR_50, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTOR_MODIFY_COLUMN_SQC_VERSION_VARCHAR_64, conn);
}
+ if(version <= 3) {
+ // Schema modifications
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_RENAME_COLUMN_SQF_OPERATION_TO_SQF_DIRECTION, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_CONNECTION_TO_SQB_FROM_CONNECTION, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_CONNECTION, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQN, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_FROM, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_TO, conn);
+
+ // Data modifications only for non-fresh install.
+ if (version > 0) {
+ // Register HDFS connector
+ updateJobData(conn, registerHdfsConnector(conn));
+ }
+
+ // Wait to remove SQB_TYPE (IMPORT/EXPORT) until we update data.
+ // Data updates depend on knowledge of the type of job.
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE, conn);
+ }
ResultSet rs = null;
PreparedStatement stmt = null;
@@ -414,6 +444,172 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
}
/**
+ * Upgrade job data from IMPORT/EXPORT to FROM/TO.
+ * Since the framework is no longer responsible for HDFS,
+ * the HDFS connector/connection must be added.
+ * Also, the framework forms are moved around such that
+ * they belong to the added HDFS connector. Any extra forms
+ * are removed.
+ * NOTE: Connector forms should have a direction (FROM/TO),
+ * but framework forms should not.
+ *
+ * Here's a brief list describing the data migration process.
+ * 1. Change SQ_FORM.SQF_DIRECTION from IMPORT to FROM.
+ * 2. Change SQ_FORM.SQF_DIRECTION from EXPORT to TO.
+ * 3. Change EXPORT to TO in newly existing SQF_DIRECTION.
+ * This should affect connectors only since Connector forms
+ * should have had a value for SQF_OPERATION.
+ * 4. Change IMPORT to FROM in newly existing SQF_DIRECTION.
+ * This should affect connectors only since Connector forms
+ * should have had a value for SQF_OPERATION.
+ * 5. Add HDFS connector for jobs to reference.
+ * 6. Set 'input' and 'output' forms connector.
+ * to HDFS connector.
+ * 7. Throttling form was originally the second form in
+ * the framework. It should now be the first form.
+ * 8. Remove the EXPORT throttling form and ensure all of
+ * its dependencies point to the IMPORT throttling form.
+ * Then make sure the throttling form does not have a direction.
+ * Framework forms should not have a direction.
+ * 9. Create an HDFS connection to reference and update
+ * jobs to reference that connection. IMPORT jobs
+ * should have TO HDFS connector, EXPORT jobs should have
+ * FROM HDFS connector.
+ * 10. Update 'table' form names to 'fromTable' and 'toTable'.
+ * Also update the relevant inputs as well.
+ * @param conn
+ */
+ private void updateJobData(Connection conn, long connectorId) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Updating existing data for generic connectors.");
+ }
+
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_OPERATION_TO_SQF_DIRECTION, conn,
+ Direction.FROM.toString(), "IMPORT");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_OPERATION_TO_SQF_DIRECTION, conn,
+ Direction.TO.toString(), "EXPORT");
+
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR_HDFS_FORM_DIRECTION, conn,
+ Direction.FROM.toString(),
+ "input");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR_HDFS_FORM_DIRECTION, conn,
+ Direction.TO.toString(),
+ "output");
+
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR, conn,
+ new Long(connectorId), "input", "output");
+
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_UPDATE_THROTTLING_FORM_INPUTS, conn,
+ "IMPORT", "EXPORT");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_REMOVE_EXTRA_FORM_INPUTS, conn,
+ "throttling", "EXPORT");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_REMOVE_EXTRA_FRAMEWORK_FORM, conn,
+ "throttling", "EXPORT");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_DIRECTION_TO_NULL, conn,
+ "throttling");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_FRAMEWORK_INDEX, conn,
+ new Long(0), "throttling");
+
+ MConnection hdfsConnection = createHdfsConnection(conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION_COPY_SQB_FROM_CONNECTION, conn,
+ "EXPORT");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_FROM_CONNECTION, conn,
+ new Long(hdfsConnection.getPersistenceId()), "EXPORT");
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION, conn,
+ new Long(hdfsConnection.getPersistenceId()), "IMPORT");
+
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_NAME, conn,
+ "fromTable", "table", Direction.FROM.toString());
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES, conn,
+ Direction.FROM.toString().toLowerCase(), "fromTable", Direction.FROM.toString());
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_NAME, conn,
+ "toTable", "table", Direction.TO.toString());
+ runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES, conn,
+ Direction.TO.toString().toLowerCase(), "toTable", Direction.TO.toString());
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Updated existing data for generic connectors.");
+ }
+ }
+
+ /**
+ * Pre-register HDFS Connector so that metadata upgrade will work.
+ */
+ protected long registerHdfsConnector(Connection conn) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Begin HDFS Connector pre-loading.");
+ }
+
+ List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs();
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Connector config urls: " + connectorConfigs);
+ }
+
+ ConnectorHandler handler = null;
+ for (URL url : connectorConfigs) {
+ handler = new ConnectorHandler(url);
+
+ if (handler.getMetadata().getPersistenceId() != -1) {
+ return handler.getMetadata().getPersistenceId();
+ }
+
+ if (handler.getUniqueName().equals(CONNECTOR_HDFS)) {
+ try {
+ PreparedStatement baseConnectorStmt = conn.prepareStatement(
+ STMT_INSERT_CONNECTOR_BASE,
+ Statement.RETURN_GENERATED_KEYS);
+ baseConnectorStmt.setString(1, handler.getMetadata().getUniqueName());
+ baseConnectorStmt.setString(2, handler.getMetadata().getClassName());
+ baseConnectorStmt.setString(3, "0");
+ if (baseConnectorStmt.executeUpdate() == 1) {
+ ResultSet rsetConnectorId = baseConnectorStmt.getGeneratedKeys();
+ if (rsetConnectorId.next()) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("HDFS Connector pre-loaded: " + rsetConnectorId.getLong(1));
+ }
+ return rsetConnectorId.getLong(1);
+ }
+ }
+ } catch (SQLException e) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0013);
+ }
+
+ break;
+ }
+ }
+
+ return -1L;
+ }
+
+ /**
+ * Create an HDFS connection.
+ * Intended to be used when moving HDFS connector out of framework
+ * to its own connector.
+ *
+ * NOTE: Upgrade path only!
+ */
+ private MConnection createHdfsConnection(Connection conn) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Creating HDFS connection.");
+ }
+
+ MConnector hdfsConnector = this.findConnector(CONNECTOR_HDFS, conn);
+ MFramework framework = findFramework(conn);
+ MConnection hdfsConnection = new MConnection(
+ hdfsConnector.getPersistenceId(),
+ hdfsConnector.getConnectionForms(),
+ framework.getConnectionForms());
+ this.createConnection(hdfsConnection, conn);
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Created HDFS connection.");
+ }
+
+ return hdfsConnection;
+ }
+
+ /**
* {@inheritDoc}
*/
@Override
@@ -536,7 +732,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
List<MForm> connectionForms = new ArrayList<MForm>();
List<MForm> jobForms = new ArrayList<MForm>();
- loadForms(connectionForms, jobForms, formFetchStmt, inputFetchStmt, 1);
+ loadFrameworkForms(connectionForms, jobForms, formFetchStmt, inputFetchStmt, 1);
// Return nothing If there aren't any framework metadata
if(connectionForms.isEmpty() && jobForms.isEmpty()) {
@@ -948,11 +1144,11 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
conn);
createInputValues(STMT_INSERT_JOB_INPUT,
jobId,
- job.getFrameworkPart().getForms(),
+ job.getConnectorPart(Direction.TO).getForms(),
conn);
createInputValues(STMT_INSERT_JOB_INPUT,
jobId,
- job.getConnectorPart(Direction.TO).getForms(),
+ job.getFrameworkPart().getForms(),
conn);
job.setPersistenceId(jobId);
@@ -993,9 +1189,13 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
job.getConnectorPart(Direction.FROM).getForms(),
conn);
createInputValues(STMT_INSERT_JOB_INPUT,
- job.getPersistenceId(),
- job.getFrameworkPart().getForms(),
- conn);
+ job.getPersistenceId(),
+ job.getConnectorPart(Direction.TO).getForms(),
+ conn);
+ createInputValues(STMT_INSERT_JOB_INPUT,
+ job.getPersistenceId(),
+ job.getFrameworkPart().getForms(),
+ conn);
} catch (SQLException ex) {
logException(ex, job);
@@ -1157,6 +1357,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
try {
stmt = conn.prepareStatement(STMT_SELECT_ALL_JOBS_FOR_CONNECTOR);
stmt.setLong(1, connectorId);
+ stmt.setLong(2, connectorId);
return loadJobs(stmt, conn);
} catch (SQLException ex) {
@@ -1664,7 +1865,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
formConnectorFetchStmt.setLong(1, connectorId);
inputFetchStmt.setLong(1, id);
- //inputFetchStmt.setLong(2, XXX); // Will be filled by loadForms
+ //inputFetchStmt.setLong(2, XXX); // Will be filled by loadFrameworkForms
inputFetchStmt.setLong(3, id);
List<MForm> connectorConnForms = new ArrayList<MForm>();
@@ -1674,9 +1875,9 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
List<MForm> toJobForms = new ArrayList<MForm>();
loadConnectorForms(connectorConnForms, fromJobForms, toJobForms,
- formConnectorFetchStmt, inputFetchStmt, 2);
- loadForms(frameworkConnForms, frameworkJobForms,
- formFrameworkFetchStmt, inputFetchStmt, 2);
+ formConnectorFetchStmt, inputFetchStmt, 2);
+ loadFrameworkForms(frameworkConnForms, frameworkJobForms,
+ formFrameworkFetchStmt, inputFetchStmt, 2);
MConnection connection = new MConnection(connectorId,
new MConnectionForms(connectorConnForms),
@@ -1736,7 +1937,7 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
toFormConnectorFetchStmt.setLong(1,toConnectorId);
inputFetchStmt.setLong(1, id);
- //inputFetchStmt.setLong(1, XXX); // Will be filled by loadForms
+ //inputFetchStmt.setLong(1, XXX); // Will be filled by loadFrameworkForms
inputFetchStmt.setLong(3, id);
List<MForm> toConnectorConnForms = new ArrayList<MForm>();
@@ -1765,8 +1966,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
toConnectorToJobForms,
toFormConnectorFetchStmt, inputFetchStmt, 2);
- loadForms(frameworkConnForms, frameworkJobForms,
- formFrameworkFetchStmt, inputFetchStmt, 2);
+ loadFrameworkForms(frameworkConnForms, frameworkJobForms,
+ formFrameworkFetchStmt, inputFetchStmt, 2);
MJob job = new MJob(
fromConnectorId, toConnectorId,
@@ -1902,11 +2103,22 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
*
* @param query Query that should be executed
*/
- private void runQuery(String query, Connection conn) {
- Statement stmt = null;
+ private void runQuery(String query, Connection conn, Object... args) {
+ PreparedStatement stmt = null;
try {
- stmt = conn.createStatement();
- if (stmt.execute(query)) {
+ stmt = conn.prepareStatement(query);
+
+ for (int i = 0; i < args.length; ++i) {
+ if (args[i] instanceof String) {
+ stmt.setString(i + 1, (String)args[i]);
+ } else if (args[i] instanceof Long) {
+ stmt.setLong(i + 1, (Long) args[i]);
+ } else {
+ stmt.setObject(i, args[i]);
+ }
+ }
+
+ if (stmt.execute()) {
ResultSet rset = stmt.getResultSet();
int count = 0;
while (rset.next()) {
@@ -1936,18 +2148,18 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler {
* @param inputFetchStmt Prepare statement for fetching inputs
* @throws SQLException In case of any failure on Derby side
*/
- public void loadForms(List<MForm> connectionForms,
- List<MForm> jobForms,
- PreparedStatement formFetchStmt,
- PreparedStatement inputFetchStmt,
- int formPosition) throws SQLException {
+ public void loadFrameworkForms(List<MForm> connectionForms,
+ List<MForm> jobForms,
+ PreparedStatement formFetchStmt,
+ PreparedStatement inputFetchStmt,
+ int formPosition) throws SQLException {
// Get list of structures from database
ResultSet rsetForm = formFetchStmt.executeQuery();
while (rsetForm.next()) {
long formId = rsetForm.getLong(1);
Long formConnectorId = rsetForm.getLong(2);
- String operation = rsetForm.getString(3);
+ String direction = rsetForm.getString(3);
String formName = rsetForm.getString(4);
String formType = rsetForm.getString(5);
int formIndex = rsetForm.getInt(6);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
index 1a77360..58eed2d 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
@@ -69,6 +69,8 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQF_OPERATION = "SQF_OPERATION";
+ public static final String COLUMN_SQF_DIRECTION = "SQF_DIRECTION";
+
public static final String COLUMN_SQF_NAME = "SQF_NAME";
public static final String COLUMN_SQF_TYPE = "SQF_TYPE";
@@ -144,6 +146,10 @@ public final class DerbySchemaConstants {
public static final String COLUMN_SQB_NAME = "SQB_NAME";
+ public static final String COLUMN_SQB_CONNECTION = "SQB_CONNECTION";
+
+ public static final String COLUMN_SQB_TYPE = "SQB_TYPE";
+
public static final String COLUMN_SQB_FROM_CONNECTION = "SQB_FROM_CONNECTION";
public static final String COLUMN_SQB_TO_CONNECTION = "SQB_TO_CONNECTION";
@@ -162,6 +168,14 @@ public final class DerbySchemaConstants {
public static final String CONSTRAINT_SQB_SQN = SCHEMA_PREFIX + CONSTRAINT_SQB_SQN_NAME;
+ public static final String CONSTRAINT_SQB_SQN_FROM_NAME = CONSTRAINT_PREFIX + "SQB_SQN_FROM";
+
+ public static final String CONSTRAINT_SQB_SQN_FROM = SCHEMA_PREFIX + CONSTRAINT_SQB_SQN_FROM_NAME;
+
+ public static final String CONSTRAINT_SQB_SQN_TO_NAME = CONSTRAINT_PREFIX + "SQB_SQN_TO";
+
+ public static final String CONSTRAINT_SQB_SQN_TO = SCHEMA_PREFIX + CONSTRAINT_SQB_SQN_TO_NAME;
+
// SQ_CONNECTION_INPUT
public static final String TABLE_SQ_CONNECTION_INPUT_NAME =
http://git-wip-us.apache.org/repos/asf/sqoop/blob/51a07bc3/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
index e5bb2e7..061551e 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaQuery.java
@@ -50,16 +50,16 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* <p>
* <strong>SQ_FORM</strong>: Form details.
* <pre>
- * +-----------------------------+
- * | SQ_FORM |
- * +-----------------------------+
- * | SQF_ID: BIGINT PK AUTO-GEN |
- * | SQF_CONNECTOR: BIGINT | FK SQ_CONNECTOR(SQC_ID),NULL for framework
- * | SQF_OPERATION: VARCHAR(32) | "IMPORT"|"EXPORT"|NULL
- * | SQF_NAME: VARCHAR(64) |
- * | SQF_TYPE: VARCHAR(32) | "CONNECTION"|"JOB"
- * | SQF_INDEX: SMALLINT |
- * +-----------------------------+
+ * +----------------------------------+
+ * | SQ_FORM |
+ * +----------------------------------+
+ * | SQF_ID: BIGINT PK AUTO-GEN |
+ * | SQF_CONNECTOR: BIGINT | FK SQ_CONNECTOR(SQC_ID),NULL for framework
+ * | SQF_DIRECTION: VARCHAR(32) | "FROM"|"TO"|NULL
+ * | SQF_NAME: VARCHAR(64) |
+ * | SQF_TYPE: VARCHAR(32) | "CONNECTION"|"JOB"
+ * | SQF_INDEX: SMALLINT |
+ * +----------------------------------+
* </pre>
* </p>
* <p>
@@ -104,8 +104,8 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* +--------------------------------+
* | SQB_ID: BIGINT PK AUTO-GEN |
* | SQB_NAME: VARCHAR(64) |
- * | SQB_TYPE: VARCHAR(64) |
- * | SQB_CONNECTION: BIGINT | FK SQ_CONNECTION(SQN_ID)
+ * | SQB_FROM_CONNECTION: BIGINT | FK SQ_CONNECTION(SQN_ID)
+ * | SQB_TO_CONNECTION: BIGINT | FK SQ_CONNECTION(SQN_ID)
* | SQB_CREATION_USER: VARCHAR(32) |
* | SQB_CREATION_DATE: TIMESTAMP |
* | SQB_UPDATE_USER: VARCHAR(32) |
@@ -286,13 +286,13 @@ public final class DerbySchemaQuery {
public static final String QUERY_CREATE_TABLE_SQ_JOB =
"CREATE TABLE " + TABLE_SQ_JOB + " ("
+ COLUMN_SQB_ID + " BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1) PRIMARY KEY, "
- + COLUMN_SQB_FROM_CONNECTION + " BIGINT, "
- + COLUMN_SQB_TO_CONNECTION + " BIGINT, "
+ + COLUMN_SQB_CONNECTION + " BIGINT, "
+ COLUMN_SQB_NAME + " VARCHAR(64), "
+ + COLUMN_SQB_TYPE + " VARCHAR(64),"
+ COLUMN_SQB_CREATION_DATE + " TIMESTAMP,"
+ COLUMN_SQB_UPDATE_DATE + " TIMESTAMP,"
+ "CONSTRAINT " + CONSTRAINT_SQB_SQN + " "
- + "FOREIGN KEY(" + COLUMN_SQB_FROM_CONNECTION + ") "
+ + "FOREIGN KEY(" + COLUMN_SQB_CONNECTION + ") "
+ "REFERENCES " + TABLE_SQ_CONNECTION + " (" + COLUMN_SQN_ID + ")"
+ ")";
@@ -459,7 +459,7 @@ public final class DerbySchemaQuery {
"SELECT "
+ COLUMN_SQF_ID + ", "
+ COLUMN_SQF_CONNECTOR + ", "
- + COLUMN_SQF_OPERATION + ", "
+ + COLUMN_SQF_DIRECTION + ", "
+ COLUMN_SQF_NAME + ", "
+ COLUMN_SQF_TYPE + ", "
+ COLUMN_SQF_INDEX
@@ -472,13 +472,13 @@ public final class DerbySchemaQuery {
"SELECT "
+ COLUMN_SQF_ID + ", "
+ COLUMN_SQF_CONNECTOR + ", "
- + COLUMN_SQF_OPERATION + ", "
+ + COLUMN_SQF_DIRECTION + ", "
+ COLUMN_SQF_NAME + ", "
+ COLUMN_SQF_TYPE + ", "
+ COLUMN_SQF_INDEX
+ " FROM " + TABLE_SQ_FORM
+ " WHERE " + COLUMN_SQF_CONNECTOR + " IS NULL "
- + " ORDER BY " + COLUMN_SQF_INDEX;
+ + " ORDER BY " + COLUMN_SQF_TYPE + ", " + COLUMN_SQF_DIRECTION + ", " + COLUMN_SQF_INDEX;
// DML: Fetch inputs for a given form
public static final String STMT_FETCH_INPUT =
@@ -530,10 +530,10 @@ public final class DerbySchemaQuery {
+ COLUMN_SQBI_VALUE
+ " FROM " + TABLE_SQ_INPUT
+ " LEFT OUTER JOIN " + TABLE_SQ_JOB_INPUT
- + " ON " + COLUMN_SQBI_INPUT + " = " + COLUMN_SQI_ID
- + " AND " + COLUMN_SQBI_JOB + " = ?"
- + " WHERE " + COLUMN_SQI_FORM + " = ?" +
- " AND (" + COLUMN_SQBI_JOB + " = ? OR " + COLUMN_SQBI_JOB + " IS NULL)"
+ + " ON " + COLUMN_SQBI_INPUT + " = " + COLUMN_SQI_ID
+ + " AND " + COLUMN_SQBI_JOB + " = ?"
+ + " WHERE " + COLUMN_SQI_FORM + " = ?"
+ + " AND (" + COLUMN_SQBI_JOB + " = ? OR " + COLUMN_SQBI_JOB + " IS NULL)"
+ " ORDER BY " + COLUMN_SQI_INDEX;
// DML: Insert connector base
@@ -548,7 +548,7 @@ public final class DerbySchemaQuery {
public static final String STMT_INSERT_FORM_BASE =
"INSERT INTO " + TABLE_SQ_FORM + " ("
+ COLUMN_SQF_CONNECTOR + ", "
- + COLUMN_SQF_OPERATION + ", "
+ + COLUMN_SQF_DIRECTION + ", "
+ COLUMN_SQF_NAME + ", "
+ COLUMN_SQF_TYPE + ", "
+ COLUMN_SQF_INDEX
@@ -770,50 +770,36 @@ public final class DerbySchemaQuery {
+ "job." + COLUMN_SQB_CREATION_DATE + ", "
+ "job." + COLUMN_SQB_UPDATE_USER + ", "
+ "job." + COLUMN_SQB_UPDATE_DATE
- + " FROM " + TABLE_SQ_JOB + " AS job"
+ + " FROM " + TABLE_SQ_JOB + " job"
+ " LEFT JOIN " + TABLE_SQ_CONNECTION
- + " as FROM_CONNECTOR ON " + COLUMN_SQB_FROM_CONNECTION + " = FROM_CONNECTOR." + COLUMN_SQN_ID
+ + " FROM_CONNECTOR ON " + COLUMN_SQB_FROM_CONNECTION + " = FROM_CONNECTOR." + COLUMN_SQN_ID
+ " LEFT JOIN " + TABLE_SQ_CONNECTION
- + " as TO_CONNECTOR ON " + COLUMN_SQB_TO_CONNECTION + " = TO_CONNECTOR." + COLUMN_SQN_ID
+ + " TO_CONNECTOR ON " + COLUMN_SQB_TO_CONNECTION + " = TO_CONNECTOR." + COLUMN_SQN_ID
+ " WHERE " + COLUMN_SQB_ID + " = ?";
// DML: Select all jobs
public static final String STMT_SELECT_JOB_ALL =
"SELECT "
- + "FROM_CONNECTOR." + COLUMN_SQN_CONNECTOR + ", "
- + "TO_CONNECTOR." + COLUMN_SQN_CONNECTOR + ", "
- + "job." + COLUMN_SQB_ID + ", "
- + "job." + COLUMN_SQB_NAME + ", "
- + "job." + COLUMN_SQB_FROM_CONNECTION + ", "
- + "job." + COLUMN_SQB_TO_CONNECTION + ", "
- + "job." + COLUMN_SQB_ENABLED + ", "
- + "job." + COLUMN_SQB_CREATION_USER + ", "
- + "job." + COLUMN_SQB_CREATION_DATE + ", "
- + "job." + COLUMN_SQB_UPDATE_USER + ", "
- + "job." + COLUMN_SQB_UPDATE_DATE
- + " FROM " + TABLE_SQ_JOB + " AS job"
- + " LEFT JOIN " + TABLE_SQ_CONNECTION
- + " as FROM_CONNECTOR ON " + COLUMN_SQB_FROM_CONNECTION + " = FROM_CONNECTOR." + COLUMN_SQN_ID
- + " LEFT JOIN " + TABLE_SQ_CONNECTION
- + " as TO_CONNECTOR ON " + COLUMN_SQB_TO_CONNECTION + " = TO_CONNECTOR." + COLUMN_SQN_ID;
+ + "FROM_CONNECTION." + COLUMN_SQN_CONNECTOR + ", "
+ + "TO_CONNECTION." + COLUMN_SQN_CONNECTOR + ", "
+ + "JOB." + COLUMN_SQB_ID + ", "
+ + "JOB." + COLUMN_SQB_NAME + ", "
+ + "JOB." + COLUMN_SQB_FROM_CONNECTION + ", "
+ + "JOB." + COLUMN_SQB_TO_CONNECTION + ", "
+ + "JOB." + COLUMN_SQB_ENABLED + ", "
+ + "JOB." + COLUMN_SQB_CREATION_USER + ", "
+ + "JOB." + COLUMN_SQB_CREATION_DATE + ", "
+ + "JOB." + COLUMN_SQB_UPDATE_USER + ", "
+ + "JOB." + COLUMN_SQB_UPDATE_DATE
+ + " FROM " + TABLE_SQ_JOB + " JOB"
+ + " LEFT JOIN " + TABLE_SQ_CONNECTION + " FROM_CONNECTION"
+ + " ON " + COLUMN_SQB_FROM_CONNECTION + " = FROM_CONNECTION." + COLUMN_SQN_ID
+ + " LEFT JOIN " + TABLE_SQ_CONNECTION + " TO_CONNECTION"
+ + " ON " + COLUMN_SQB_TO_CONNECTION + " = TO_CONNECTION." + COLUMN_SQN_ID;
// DML: Select all jobs for a Connector
- public static final String STMT_SELECT_ALL_JOBS_FOR_CONNECTOR =
- "SELECT "
- + COLUMN_SQN_CONNECTOR + ", "
- + COLUMN_SQB_ID + ", "
- + COLUMN_SQB_NAME + ", "
- + COLUMN_SQB_FROM_CONNECTION + ", "
- + COLUMN_SQB_TO_CONNECTION + ", "
- + COLUMN_SQB_ENABLED + ", "
- + COLUMN_SQB_CREATION_USER + ", "
- + COLUMN_SQB_CREATION_DATE + ", "
- + COLUMN_SQB_UPDATE_USER + ", "
- + COLUMN_SQB_UPDATE_DATE
- + " FROM " + TABLE_SQ_JOB
- + " LEFT JOIN " + TABLE_SQ_CONNECTION
- + " ON " + COLUMN_SQB_FROM_CONNECTION + " = " + COLUMN_SQN_ID
- + " AND " + COLUMN_SQN_CONNECTOR + " = ? ";
+ public static final String STMT_SELECT_ALL_JOBS_FOR_CONNECTOR = STMT_SELECT_JOB_ALL
+ + " WHERE FROM_CONNECTION." + COLUMN_SQN_CONNECTOR + " = ? OR TO_CONNECTION." + COLUMN_SQN_CONNECTOR + " = ?";
// DML: Insert new submission
public static final String STMT_INSERT_SUBMISSION =
@@ -964,6 +950,122 @@ public final class DerbySchemaQuery {
"ALTER TABLE " + TABLE_SQ_CONNECTOR + " ALTER COLUMN "
+ COLUMN_SQC_VERSION + " SET DATA TYPE VARCHAR(64)";
+ // Version 4 Upgrade
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_RENAME_COLUMN_SQB_CONNECTION_TO_SQB_FROM_CONNECTION =
+ "RENAME COLUMN " + TABLE_SQ_JOB + "." + COLUMN_SQB_CONNECTION
+ + " TO " + COLUMN_SQB_FROM_CONNECTION;
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_COLUMN_SQB_TO_CONNECTION =
+ "ALTER TABLE " + TABLE_SQ_JOB + " ADD COLUMN " + COLUMN_SQB_TO_CONNECTION
+ + " BIGINT";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_CONSTRAINT_SQB_SQN =
+ "ALTER TABLE " + TABLE_SQ_JOB + " DROP CONSTRAINT " + CONSTRAINT_SQB_SQN;
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_FROM =
+ "ALTER TABLE " + TABLE_SQ_JOB + " ADD CONSTRAINT " + CONSTRAINT_SQB_SQN_FROM
+ + " FOREIGN KEY (" + COLUMN_SQB_FROM_CONNECTION + ") REFERENCES "
+ + TABLE_SQ_CONNECTION + " (" + COLUMN_SQN_ID + ")";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_ADD_CONSTRAINT_SQB_SQN_TO =
+ "ALTER TABLE " + TABLE_SQ_JOB + " ADD CONSTRAINT " + CONSTRAINT_SQB_SQN_TO
+ + " FOREIGN KEY (" + COLUMN_SQB_TO_CONNECTION + ") REFERENCES "
+ + TABLE_SQ_CONNECTION + " (" + COLUMN_SQN_ID + ")";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_RENAME_COLUMN_SQF_OPERATION_TO_SQF_DIRECTION =
+ "RENAME COLUMN " + TABLE_SQ_FORM + "." + COLUMN_SQF_OPERATION
+ + " TO " + COLUMN_SQF_DIRECTION;
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_OPERATION_TO_SQF_DIRECTION =
+ "UPDATE " + TABLE_SQ_FORM + " SET " + COLUMN_SQF_DIRECTION
+ + "=? WHERE " + COLUMN_SQF_DIRECTION + "=?"
+ + " AND " + COLUMN_SQF_CONNECTOR + " IS NOT NULL";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR =
+ "UPDATE " + TABLE_SQ_FORM + " SET " + COLUMN_SQF_CONNECTOR + "= ?"
+ + " WHERE " + COLUMN_SQF_CONNECTOR + " IS NULL AND "
+ + COLUMN_SQF_NAME + " IN (?, ?)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_CONNECTOR_HDFS_FORM_DIRECTION =
+ "UPDATE " + TABLE_SQ_FORM + " SET " + COLUMN_SQF_DIRECTION + "= ?"
+ + " WHERE " + COLUMN_SQF_NAME + "= ?";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION_COPY_SQB_FROM_CONNECTION =
+ "UPDATE " + TABLE_SQ_JOB + " SET "
+ + COLUMN_SQB_TO_CONNECTION + "=" + COLUMN_SQB_FROM_CONNECTION
+ + " WHERE " + COLUMN_SQB_TYPE + "= ?";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_FROM_CONNECTION =
+ "UPDATE " + TABLE_SQ_JOB + " SET " + COLUMN_SQB_FROM_CONNECTION + "=?"
+ + " WHERE " + COLUMN_SQB_TYPE + "= ?";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_SQB_TO_CONNECTION =
+ "UPDATE " + TABLE_SQ_JOB + " SET " + COLUMN_SQB_TO_CONNECTION + "=?"
+ + " WHERE " + COLUMN_SQB_TYPE + "= ?";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_SQF_NAME =
+ "UPDATE " + TABLE_SQ_FORM + " SET "
+ + COLUMN_SQF_NAME + "= ?"
+ + " WHERE " + COLUMN_SQF_NAME + "= ?"
+ + " AND " + COLUMN_SQF_DIRECTION + "= ?";
+
+ /**
+ * Intended to rename forms based on direction.
+ * e.g. If SQ_FORM.SQF_NAME = 'table' and parameter 1 = 'from'
+ * then SQ_FORM.SQF_NAME = 'fromTable'.
+ */
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_TABLE_INPUT_NAMES =
+ "UPDATE " + TABLE_SQ_INPUT + " SET "
+ + COLUMN_SQI_NAME + "=("
+ + "? || UPPER(SUBSTR(" + COLUMN_SQI_NAME + ",1,1))"
+ + " || SUBSTR(" + COLUMN_SQI_NAME + ",2) )"
+ + " WHERE " + COLUMN_SQI_FORM + " IN ("
+ + " SELECT " + COLUMN_SQF_ID + " FROM " + TABLE_SQ_FORM + " WHERE " + COLUMN_SQF_NAME + "= ?"
+ + " AND " + COLUMN_SQF_DIRECTION + "= ?)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_DIRECTION_TO_NULL =
+ "UPDATE " + TABLE_SQ_FORM + " SET "
+ + COLUMN_SQF_DIRECTION + "= NULL"
+ + " WHERE " + COLUMN_SQF_NAME + "= ?";
+
+ public static final String QUERY_SELECT_THROTTLING_FORM_INPUT_IDS =
+ "SELECT SQI." + COLUMN_SQI_ID + " FROM " + TABLE_SQ_INPUT + " SQI"
+ + " INNER JOIN " + TABLE_SQ_FORM + " SQF ON SQI." + COLUMN_SQI_FORM + "=SQF." + COLUMN_SQF_ID
+ + " WHERE SQF." + COLUMN_SQF_NAME + "='throttling' AND SQF." + COLUMN_SQF_DIRECTION + "=?";
+
+ /**
+ * Intended to change SQ_JOB_INPUT.SQBI_INPUT from EXPORT
+ * throttling form, to IMPORT throttling form.
+ */
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_UPDATE_THROTTLING_FORM_INPUTS =
+ "UPDATE " + TABLE_SQ_JOB_INPUT + " SQBI SET"
+ + " SQBI." + COLUMN_SQBI_INPUT + "=(" + QUERY_SELECT_THROTTLING_FORM_INPUT_IDS
+ + " AND SQI." + COLUMN_SQI_NAME + "=("
+ + "SELECT SQI2." + COLUMN_SQI_NAME + " FROM " + TABLE_SQ_INPUT + " SQI2"
+ + " WHERE SQI2." + COLUMN_SQI_ID + "=SQBI." + COLUMN_SQBI_INPUT + " FETCH FIRST 1 ROWS ONLY"
+ + "))"
+ + "WHERE SQBI." + COLUMN_SQBI_INPUT + " IN (" + QUERY_SELECT_THROTTLING_FORM_INPUT_IDS + ")";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_REMOVE_EXTRA_FORM_INPUTS =
+ "DELETE FROM " + TABLE_SQ_INPUT + " SQI"
+ + " WHERE SQI." + COLUMN_SQI_FORM + " IN ("
+ + "SELECT SQF." + COLUMN_SQF_ID + " FROM " + TABLE_SQ_FORM + " SQF "
+ + " WHERE SQF." + COLUMN_SQF_NAME + "= ?"
+ + " AND SQF." + COLUMN_SQF_DIRECTION + "= ?)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_REMOVE_EXTRA_FRAMEWORK_FORM =
+ "DELETE FROM " + TABLE_SQ_FORM
+ + " WHERE " + COLUMN_SQF_NAME + "= ?"
+ + " AND " + COLUMN_SQF_DIRECTION + "= ?";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_FORM_UPDATE_FRAMEWORK_INDEX =
+ "UPDATE " + TABLE_SQ_FORM + " SET "
+ + COLUMN_SQF_INDEX + "= ?"
+ + " WHERE " + COLUMN_SQF_NAME + "= ?";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_REMOVE_COLUMN_SQB_TYPE =
+ "ALTER TABLE " + TABLE_SQ_JOB + " DROP COLUMN " + COLUMN_SQB_TYPE;
+
private DerbySchemaQuery() {
// Disable explicit object creation
}
[45/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java b/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java
new file mode 100644
index 0000000..4372171
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.ResourceBundle;
+
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriver;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MStringInput;
+import org.apache.sqoop.model.MToConfig;
+import org.apache.sqoop.utils.MapResourceBundle;
+
+/**
+ *
+ */
+public class ConfigTestUtil {
+ public static MConnector getConnector(String name) {
+ return getConnector(name, true, true);
+ }
+
+ public static MConnector getConnector(String name, boolean from, boolean to) {
+ MFromConfig fromConfig = null;
+ MToConfig toConfig = null;
+ if (from) {
+ fromConfig = getFromConfig();
+ }
+ if (to) {
+ toConfig = getToConfig();
+ }
+ return new MConnector(name, name + ".class", "1.0-test",
+ getLinkConfig(), fromConfig, toConfig);
+ }
+
+
+ public static MLink getLink(String name) {
+ return new MLink(1, getConnector(name).getLinkConfig());
+ }
+
+ public static MJob getJob(String name) {
+ return new MJob(1, 2, 1, 2, getConnector(name).getFromConfig(), getConnector(name)
+ .getToConfig(), getDriverConfig());
+ }
+
+ public static MDriverConfig getDriverConfig() {
+ List<MInput<?>> inputs;
+ MIntegerInput input;
+ MConfig config;
+ List<MConfig> driverConfigs = new ArrayList<MConfig>();
+ inputs = new ArrayList<MInput<?>>();
+
+ input = new MIntegerInput("numExtractors", false);
+ input.setPersistenceId(1);
+ inputs.add(input);
+
+ input = new MIntegerInput("numLoaders", false);
+ input.setPersistenceId(2);
+ inputs.add(input);
+
+ config = new MConfig("driver", inputs);
+ config.setPersistenceId(10);
+ driverConfigs.add(config);
+ return new MDriverConfig(driverConfigs);
+ }
+
+ public static MLinkConfig getLinkConfig() {
+ List<MInput<?>> inputs;
+ MStringInput input;
+ MConfig config;
+ List<MConfig> linkConfig = new ArrayList<MConfig>();
+ inputs = new ArrayList<MInput<?>>();
+
+ input = new MStringInput("url", false, (short) 10);
+ input.setPersistenceId(1);
+ inputs.add(input);
+
+ input = new MStringInput("username", false, (short) 10);
+ input.setPersistenceId(2);
+ input.setValue("test");
+ inputs.add(input);
+
+ input = new MStringInput("password", true, (short) 10);
+ input.setPersistenceId(3);
+ input.setValue("test");
+ inputs.add(input);
+
+ config = new MConfig("connection", inputs);
+ config.setPersistenceId(10);
+ linkConfig.add(config);
+
+ return new MLinkConfig(linkConfig);
+ }
+
+ static MFromConfig getFromConfig() {
+ List<MInput<?>> inputs;
+ MStringInput input;
+ MConfig config;
+ List<MConfig> jobConfigs = new ArrayList<MConfig>();
+
+ inputs = new ArrayList<MInput<?>>();
+
+ input = new MStringInput("A", false, (short) 10);
+ input.setPersistenceId(4);
+ inputs.add(input);
+
+ input = new MStringInput("B", false, (short) 10);
+ input.setPersistenceId(5);
+ inputs.add(input);
+
+ input = new MStringInput("C", false, (short) 10);
+ input.setPersistenceId(6);
+ inputs.add(input);
+
+ config = new MConfig("Z", inputs);
+ config.setPersistenceId(11);
+ jobConfigs.add(config);
+
+ inputs = new ArrayList<MInput<?>>();
+
+ input = new MStringInput("D", false, (short) 10);
+ input.setPersistenceId(7);
+ inputs.add(input);
+
+ input = new MStringInput("E", false, (short) 10);
+ input.setPersistenceId(8);
+ inputs.add(input);
+
+ input = new MStringInput("F", false, (short) 10);
+ input.setPersistenceId(9);
+ inputs.add(input);
+
+ config = new MConfig("from-table", inputs);
+ config.setPersistenceId(12);
+ jobConfigs.add(config);
+
+ return new MFromConfig(jobConfigs);
+ }
+
+ static MToConfig getToConfig() {
+ List<MInput<?>> inputs;
+ MStringInput input;
+ MConfig config;
+ List<MConfig> jobConfigs = new ArrayList<MConfig>();
+
+ inputs = new ArrayList<MInput<?>>();
+
+ input = new MStringInput("A", false, (short) 10);
+ input.setPersistenceId(4);
+ inputs.add(input);
+
+ input = new MStringInput("B", false, (short) 10);
+ input.setPersistenceId(5);
+ inputs.add(input);
+
+ input = new MStringInput("C", false, (short) 10);
+ input.setPersistenceId(6);
+ inputs.add(input);
+
+ config = new MConfig("Z", inputs);
+ config.setPersistenceId(11);
+ jobConfigs.add(config);
+
+ inputs = new ArrayList<MInput<?>>();
+
+ input = new MStringInput("D", false, (short) 10);
+ input.setPersistenceId(7);
+ inputs.add(input);
+
+ input = new MStringInput("E", false, (short) 10);
+ input.setPersistenceId(8);
+ inputs.add(input);
+
+ input = new MStringInput("F", false, (short) 10);
+ input.setPersistenceId(9);
+ inputs.add(input);
+
+ config = new MConfig("to-table", inputs);
+ config.setPersistenceId(12);
+ jobConfigs.add(config);
+
+ return new MToConfig(jobConfigs);
+ }
+
+ public static ResourceBundle getResourceBundle() {
+ Map<String, Object> map = new HashMap<String, Object>();
+ map.put("a", "a");
+ map.put("b", "b");
+
+ return new MapResourceBundle(map);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
index fe75ee0..9fd2fe3 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
@@ -17,10 +17,9 @@
*/
package org.apache.sqoop.json;
-import org.apache.sqoop.model.MConnector;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-import org.junit.Test;
+import static org.apache.sqoop.json.ConfigTestUtil.getConnector;
+import static org.apache.sqoop.json.ConfigTestUtil.getResourceBundle;
+import static org.junit.Assert.*;
import java.util.HashMap;
import java.util.LinkedList;
@@ -28,8 +27,10 @@ import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
-import static org.apache.sqoop.json.TestUtil.*;
-import static org.junit.Assert.*;
+import org.apache.sqoop.model.MConnector;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+import org.junit.Test;
/**
*
@@ -48,26 +49,26 @@ public class TestConnectorBean {
connectors.add(getConnector("mysql"));
// Create testing bundles
- Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
- bundles.put(1L, getResourceBundle());
- bundles.put(2L, getResourceBundle());
+ Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>();
+ configBundles.put(1L, getResourceBundle());
+ configBundles.put(2L, getResourceBundle());
// Serialize it to JSON object
- ConnectorBean bean = new ConnectorBean(connectors, bundles);
- JSONObject json = bean.extract(false);
+ ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles);
+ JSONObject connectorJSON = connectorBean.extract(false);
// "Move" it across network in text form
- String string = json.toJSONString();
+ String connectorJSONString = connectorJSON.toJSONString();
// Retrieved transferred object
- JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
- ConnectorBean retrievedBean = new ConnectorBean();
- retrievedBean.restore(retrievedJson);
+ JSONObject parsedConnector = (JSONObject) JSONValue.parse(connectorJSONString);
+ ConnectorBean parsedConnectorBean = new ConnectorBean();
+ parsedConnectorBean.restore(parsedConnector);
- assertEquals(connectors.size(), retrievedBean.getConnectors().size());
- assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
+ assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size());
+ assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0));
- ResourceBundle retrievedBundle = retrievedBean.getResourceBundles().get(1L);
+ ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L);
assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/TestDriverBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestDriverBean.java b/common/src/test/java/org/apache/sqoop/json/TestDriverBean.java
new file mode 100644
index 0000000..8c92aef
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/json/TestDriverBean.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json;
+
+import static org.apache.sqoop.json.ConfigTestUtil.getResourceBundle;
+import static org.junit.Assert.assertEquals;
+
+import java.util.ResourceBundle;
+
+import org.apache.sqoop.model.MDriver;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestDriverBean {
+
+ /**
+ * Test that by JSON serialization followed by deserialization we will get
+ * equal drive config object.
+ */
+ @Test
+ public void testSerialization() {
+ MDriver driver = new MDriver(ConfigTestUtil.getDriverConfig(), DriverBean.CURRENT_DRIVER_VERSION);
+
+ // Serialize it to JSON object
+ DriverBean bean = new DriverBean(driver, getResourceBundle());
+ JSONObject json = bean.extract(false);
+
+ // "Move" it across network in text form
+ String string = json.toJSONString();
+
+ // Retrieved transferred object
+ JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
+ DriverBean retrievedBean = new DriverBean();
+ retrievedBean.restore(retrievedJson);
+
+ assertEquals(driver, retrievedBean.getDriver());
+
+ ResourceBundle retrievedBundle = retrievedBean.getDriverConfigResourceBundle();
+ assertEquals("a", retrievedBundle.getString("a"));
+ assertEquals("b", retrievedBundle.getString("b"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java b/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
deleted file mode 100644
index fcce7b5..0000000
--- a/common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json;
-
-import org.apache.sqoop.model.MDriverConfig;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-import org.junit.Test;
-
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.json.TestUtil.*;
-import static org.junit.Assert.*;
-
-/**
- *
- */
-public class TestDriverConfigBean {
-
- /**
- * Test that by JSON serialization followed by deserialization we will get
- * equal framework object.
- */
- @Test
- public void testSerialization() {
- MDriverConfig driverConfig = getDriverConfig();
-
- // Serialize it to JSON object
- DriverConfigBean bean = new DriverConfigBean(driverConfig, getResourceBundle());
- JSONObject json = bean.extract(false);
-
- // "Move" it across network in text form
- String string = json.toJSONString();
-
- // Retrieved transferred object
- JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
- DriverConfigBean retrievedBean = new DriverConfigBean();
- retrievedBean.restore(retrievedJson);
-
- assertEquals(driverConfig, retrievedBean.getDriverConfig());
-
- ResourceBundle retrievedBundle = retrievedBean.getResourceBundle();
- assertEquals("a", retrievedBundle.getString("a"));
- assertEquals("b", retrievedBundle.getString("b"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestJobBean.java b/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
index 78a3420..1fc8dbd 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestJobBean.java
@@ -17,6 +17,11 @@
*/
package org.apache.sqoop.json;
+import static org.apache.sqoop.json.ConfigTestUtil.getJob;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Date;
+
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MStringInput;
@@ -25,11 +30,6 @@ import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
import org.junit.Test;
-import java.util.Date;
-
-import static org.apache.sqoop.json.TestUtil.getJob;
-import static org.junit.Assert.assertEquals;
-
/**
*
*/
@@ -46,25 +46,25 @@ public class TestJobBean {
job.setEnabled(false);
// Fill some data at the beginning
- MStringInput input = (MStringInput) job.getConnectorPart(Direction.FROM)
- .getForms().get(0).getInputs().get(0);
+ MStringInput input = (MStringInput) job.getJobConfig(Direction.FROM)
+ .getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there!");
- input = (MStringInput) job.getConnectorPart(Direction.TO)
- .getForms().get(0).getInputs().get(0);
+ input = (MStringInput) job.getJobConfig(Direction.TO)
+ .getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there again!");
// Serialize it to JSON object
- JobBean bean = new JobBean(job);
- JSONObject json = bean.extract(false);
+ JobBean jobBean = new JobBean(job);
+ JSONObject jobJson = jobBean.extract(false);
// "Move" it across network in text form
- String string = json.toJSONString();
+ String jobJsonString = jobJson.toJSONString();
// Retrieved transferred object
- JSONObject retrievedJson = (JSONObject)JSONValue.parseWithException(string);
- JobBean retrievedBean = new JobBean();
- retrievedBean.restore(retrievedJson);
- MJob target = retrievedBean.getJobs().get(0);
+ JSONObject parsedJobJson = (JSONObject)JSONValue.parseWithException(jobJsonString);
+ JobBean parsedJobBean = new JobBean();
+ parsedJobBean.restore(parsedJobJson);
+ MJob target = parsedJobBean.getJobs().get(0);
// Check id and name
assertEquals(666, target.getPersistenceId());
@@ -78,11 +78,11 @@ public class TestJobBean {
assertEquals(false, target.getEnabled());
// Test that value was correctly moved
- MStringInput targetInput = (MStringInput) target.getConnectorPart(Direction.FROM)
- .getForms().get(0).getInputs().get(0);
+ MStringInput targetInput = (MStringInput) target.getJobConfig(Direction.FROM)
+ .getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
- targetInput = (MStringInput) target.getConnectorPart(Direction.TO)
- .getForms().get(0).getInputs().get(0);
+ targetInput = (MStringInput) target.getJobConfig(Direction.TO)
+ .getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there again!", targetInput.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java b/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
index 9ca6b64..ac07137 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
@@ -17,19 +17,21 @@
*/
package org.apache.sqoop.json;
+import static org.apache.sqoop.json.ConfigTestUtil.getLink;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Date;
+
+import org.apache.sqoop.json.util.ConfigSerialization;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
-import org.json.simple.JSONArray;
import org.json.simple.JSONValue;
import org.junit.Test;
-import java.util.Date;
-
-import static org.junit.Assert.*;
-import static org.apache.sqoop.json.TestUtil.*;
-
/**
*
*/
@@ -48,31 +50,31 @@ public class TestLinkBean {
link.setEnabled(false);
// Fill some data at the beginning
- MStringInput input = (MStringInput) link.getConnectorPart().getForms()
+ MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs()
.get(0).getInputs().get(0);
input.setValue("Hi there!");
// Serialize it to JSON object
- LinkBean bean = new LinkBean(link);
- JSONObject json = bean.extract(false);
+ LinkBean linkBean = new LinkBean(link);
+ JSONObject json = linkBean.extract(false);
// Check for sensitivity
- JSONArray all = (JSONArray)json.get("all");
+ JSONArray all = (JSONArray)json.get(ConfigSerialization.ALL);
JSONObject allItem = (JSONObject)all.get(0);
- JSONArray connectors = (JSONArray)allItem.get("connector");
+ JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG);
JSONObject connector = (JSONObject)connectors.get(0);
- JSONArray inputs = (JSONArray)connector.get("inputs");
+ JSONArray inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS);
for (Object input1 : inputs) {
- assertTrue(((JSONObject)input1).containsKey("sensitive"));
+ assertTrue(((JSONObject)input1).containsKey(ConfigSerialization.CONFIG_INPUT_SENSITIVE));
}
// "Move" it across network in text form
- String string = json.toJSONString();
+ String linkJsonString = json.toJSONString();
// Retrieved transferred object
- JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
+ JSONObject parsedLinkJson = (JSONObject) JSONValue.parse(linkJsonString);
LinkBean retrievedBean = new LinkBean();
- retrievedBean.restore(retrievedJson);
+ retrievedBean.restore(parsedLinkJson);
MLink target = retrievedBean.getLinks().get(0);
// Check id and name
@@ -85,8 +87,8 @@ public class TestLinkBean {
assertEquals(false, target.getEnabled());
// Test that value was correctly moved
- MStringInput targetInput = (MStringInput) target.getConnectorPart()
- .getForms().get(0).getInputs().get(0);
+ MStringInput targetInput = (MStringInput) target.getConnectorLinkConfig()
+ .getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
}
@@ -104,7 +106,7 @@ public class TestLinkBean {
link.setEnabled(true);
// Fill some data at the beginning
- MStringInput input = (MStringInput) link.getConnectorPart().getForms()
+ MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs()
.get(0).getInputs().get(0);
input.setValue("Hi there!");
@@ -114,25 +116,25 @@ public class TestLinkBean {
JSONObject jsonFiltered = bean.extract(true);
// Sensitive values should exist
- JSONArray all = (JSONArray)json.get("all");
+ JSONArray all = (JSONArray)json.get(ConfigSerialization.ALL);
JSONObject allItem = (JSONObject)all.get(0);
- JSONArray connectors = (JSONArray)allItem.get("connector");
+ JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG);
JSONObject connector = (JSONObject)connectors.get(0);
- JSONArray inputs = (JSONArray)connector.get("inputs");
+ JSONArray inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS);
assertEquals(3, inputs.size());
// Inputs are ordered when creating link
JSONObject password = (JSONObject)inputs.get(2);
- assertTrue(password.containsKey("value"));
+ assertTrue(password.containsKey(ConfigSerialization.CONFIG_INPUT_VALUE));
// Sensitive values should not exist
- all = (JSONArray)jsonFiltered.get("all");
+ all = (JSONArray)jsonFiltered.get(ConfigSerialization.ALL);
allItem = (JSONObject)all.get(0);
- connectors = (JSONArray)allItem.get("connector");
+ connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG);
connector = (JSONObject)connectors.get(0);
- inputs = (JSONArray)connector.get("inputs");
+ inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS);
assertEquals(3, inputs.size());
// Inputs are ordered when creating link
password = (JSONObject)inputs.get(2);
- assertFalse(password.containsKey("value"));
+ assertFalse(password.containsKey(ConfigSerialization.CONFIG_INPUT_VALUE));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/TestUtil.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestUtil.java b/common/src/test/java/org/apache/sqoop/json/TestUtil.java
deleted file mode 100644
index 9875219..0000000
--- a/common/src/test/java/org/apache/sqoop/json/TestUtil.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json;
-
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
-import org.apache.sqoop.model.MStringInput;
-import org.apache.sqoop.utils.MapResourceBundle;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.ResourceBundle;
-
-/**
- *
- */
-public class TestUtil {
- public static MConnector getConnector(String name) {
- return getConnector(name, true, true);
- }
-
- public static MConnector getConnector(String name, boolean from, boolean to) {
- MJobForms fromJobForms = null;
- MJobForms toJobForms = null;
- if (from) {
- fromJobForms = getJobForms();
- }
- if (to) {
- toJobForms = getJobForms();
- }
- return new MConnector(name, name + ".class", "1.0-test",
- getConnectionForms(), fromJobForms, toJobForms);
- }
-
- public static MDriverConfig getDriverConfig() {
- return new MDriverConfig(getConnectionForms(), getJobForms(), "1");
- }
-
- public static MLink getLink(String name) {
- return new MLink(1, getConnector(name).getConnectionForms(), getDriverConfig()
- .getConnectionForms());
- }
-
- public static MJob getJob(String name) {
- return new MJob(1, 2, 1, 2, getConnector(name).getJobForms(Direction.FROM), getConnector(name)
- .getJobForms(Direction.TO), getDriverConfig().getJobForms());
- }
-
- public static MConnectionForms getConnectionForms() {
- List<MInput<?>> inputs;
- MStringInput input;
- MForm form;
- List<MForm> connectionForms = new ArrayList<MForm>();
- inputs = new ArrayList<MInput<?>>();
-
- input = new MStringInput("url", false, (short) 10);
- input.setPersistenceId(1);
- inputs.add(input);
-
- input = new MStringInput("username", false, (short) 10);
- input.setPersistenceId(2);
- input.setValue("test");
- inputs.add(input);
-
- input = new MStringInput("password", true, (short) 10);
- input.setPersistenceId(3);
- input.setValue("test");
- inputs.add(input);
-
- form = new MForm("connection", inputs);
- form.setPersistenceId(10);
- connectionForms.add(form);
-
- return new MConnectionForms(connectionForms);
- }
-
- public static MJobForms getJobForms() {
- List<MInput<?>> inputs;
- MStringInput input;
- MForm form;
- List<MForm> jobForms = new ArrayList<MForm>();
-
- inputs = new ArrayList<MInput<?>>();
-
- input = new MStringInput("A", false, (short) 10);
- input.setPersistenceId(4);
- inputs.add(input);
-
- input = new MStringInput("B", false, (short) 10);
- input.setPersistenceId(5);
- inputs.add(input);
-
- input = new MStringInput("C", false, (short) 10);
- input.setPersistenceId(6);
- inputs.add(input);
-
- form = new MForm("Z", inputs);
- form.setPersistenceId(11);
- jobForms.add(form);
-
- inputs = new ArrayList<MInput<?>>();
-
- input = new MStringInput("D", false, (short) 10);
- input.setPersistenceId(7);
- inputs.add(input);
-
- input = new MStringInput("E", false, (short) 10);
- input.setPersistenceId(8);
- inputs.add(input);
-
- input = new MStringInput("F", false, (short) 10);
- input.setPersistenceId(9);
- inputs.add(input);
-
- form = new MForm("connection", inputs);
- form.setPersistenceId(12);
- jobForms.add(form);
-
- return new MJobForms(jobForms);
- }
-
- public static ResourceBundle getResourceBundle() {
- Map<String, Object> map = new HashMap<String, Object>();
- map.put("a", "a");
- map.put("b", "b");
-
- return new MapResourceBundle(map);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java b/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
index f5f3389..fc5b21d 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
@@ -17,18 +17,20 @@
*/
package org.apache.sqoop.json;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-
/**
*
*/
@@ -44,7 +46,7 @@ public class TestValidationBean {
);
JSONObject json = bean.extract(false);
- // "Move" it across network in text form
+ // "Move" it across network in text config
String string = json.toJSONString();
// Retrieved transferred object
@@ -54,43 +56,43 @@ public class TestValidationBean {
assertNull(retrievedBean.getId());
- Validation.FormInput fa = new Validation.FormInput("f", "i");
- Validation.FormInput fb = new Validation.FormInput("f2", "i2");
+ ConfigValidator.ConfigInput fa = new ConfigValidator.ConfigInput("c", "i");
+ ConfigValidator.ConfigInput fb = new ConfigValidator.ConfigInput("c2", "i2");
- Validation fromConnector = retrievedBean.getConnectorValidation(Direction.FROM);
+ ConfigValidator fromConnector = retrievedBean.getConnectorValidation(Direction.FROM);
assertEquals(Status.FINE, fromConnector.getStatus());
assertEquals(2, fromConnector.getMessages().size());
assertTrue(fromConnector.getMessages().containsKey(fa));
- assertEquals(new Validation.Message(Status.FINE, "d"),
+ assertEquals(new ConfigValidator.Message(Status.FINE, "d"),
fromConnector.getMessages().get(fa));
- Validation toConnector = retrievedBean.getConnectorValidation(Direction.TO);
+ ConfigValidator toConnector = retrievedBean.getConnectorValidation(Direction.TO);
assertEquals(Status.FINE, toConnector.getStatus());
assertEquals(2, toConnector.getMessages().size());
assertTrue(toConnector.getMessages().containsKey(fa));
- assertEquals(new Validation.Message(Status.FINE, "d"),
+ assertEquals(new ConfigValidator.Message(Status.FINE, "d"),
toConnector.getMessages().get(fa));
- Validation framework = retrievedBean.getFrameworkValidation();
+ ConfigValidator framework = retrievedBean.getFrameworkValidation();
assertEquals(Status.UNACCEPTABLE, framework.getStatus());
assertEquals(2, framework.getMessages().size());
assertTrue(framework.getMessages().containsKey(fb));
- assertEquals(new Validation.Message(Status.UNACCEPTABLE, "c"),
+ assertEquals(new ConfigValidator.Message(Status.UNACCEPTABLE, "c"),
framework.getMessages().get(fb));
}
@Test
public void testJobValidationBeanId() {
// Serialize it to JSON object
- JobValidationBean bean = new JobValidationBean(
+ JobValidationBean jobValidatioBean = new JobValidationBean(
getValidation(Status.FINE),
getValidation(Status.FINE),
getValidation(Status.FINE)
);
- bean.setId((long) 10);
- JSONObject json = bean.extract(false);
+ jobValidatioBean.setId((long) 10);
+ JSONObject json = jobValidatioBean.extract(false);
- // "Move" it across network in text form
+ // "Move" it across network in text config
String string = json.toJSONString();
// Retrieved transferred object
@@ -105,12 +107,10 @@ public class TestValidationBean {
public void testLinkValidationBeanSerialization() {
// Serialize it to JSON object
LinkValidationBean bean = new LinkValidationBean(
- getValidation(Status.FINE),
- getValidation(Status.UNACCEPTABLE)
- );
+ getValidation(Status.FINE));
JSONObject json = bean.extract(false);
- // "Move" it across network in text form
+ // "Move" it across network in text config
String string = json.toJSONString();
// Retrieved transferred object
@@ -120,35 +120,25 @@ public class TestValidationBean {
assertNull(retrievedBean.getId());
- Validation.FormInput fa = new Validation.FormInput("f", "i");
- Validation.FormInput fb = new Validation.FormInput("f2", "i2");
-
- Validation connector = retrievedBean.getConnectorValidation();
+ ConfigValidator.ConfigInput ca = new ConfigValidator.ConfigInput("c", "i");
+ ConfigValidator connector = retrievedBean.getLinkConfigValidator();
assertEquals(Status.FINE, connector.getStatus());
assertEquals(2, connector.getMessages().size());
- assertTrue(connector.getMessages().containsKey(fa));
- assertEquals(new Validation.Message(Status.FINE, "d"),
- connector.getMessages().get(fa));
-
- Validation framework = retrievedBean.getFrameworkValidation();
- assertEquals(Status.UNACCEPTABLE, framework.getStatus());
- assertEquals(2, framework.getMessages().size());
- assertTrue(framework.getMessages().containsKey(fb));
- assertEquals(new Validation.Message(Status.UNACCEPTABLE, "c"),
- framework.getMessages().get(fb));
+ assertTrue(connector.getMessages().containsKey(ca));
+ assertEquals(new ConfigValidator.Message(Status.FINE, "d"),
+ connector.getMessages().get(ca));
}
@Test
public void testLinkValidationBeanId() {
// Serialize it to JSON object
LinkValidationBean bean = new LinkValidationBean(
- getValidation(Status.FINE),
getValidation(Status.FINE)
);
bean.setId((long) 10);
JSONObject json = bean.extract(false);
- // "Move" it across network in text form
+ // "Move" it across network in text config
String string = json.toJSONString();
// Retrieved transferred object
@@ -159,17 +149,12 @@ public class TestValidationBean {
assertEquals((Long)(long) 10, retrievedBean.getId());
}
- public Validation getValidation(Status status) {
- Map<Validation.FormInput, Validation.Message> messages =
- new HashMap<Validation.FormInput, Validation.Message>();
+ public ConfigValidator getValidation(Status status) {
+ Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
- messages.put(
- new Validation.FormInput("f", "i"),
- new Validation.Message(status, "d"));
- messages.put(
- new Validation.FormInput("f2", "i2"),
- new Validation.Message(status, "c"));
+ messages.put(new ConfigValidator.ConfigInput("c", "i"), new ConfigValidator.Message(status, "d"));
+ messages.put(new ConfigValidator.ConfigInput("c2", "i2"), new ConfigValidator.Message(status, "c"));
- return new Validation(status, messages);
+ return new ConfigValidator(status, messages);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java b/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
index 5c094fb..bdbad72 100644
--- a/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
+++ b/common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.json;
import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.ValidationResult;
+import org.apache.sqoop.validation.ConfigValidationResult;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
@@ -39,31 +39,31 @@ public class TestValidationResultBean {
@Test
public void testEmptyTransfer() {
- ValidationResult []empty = new ValidationResult[0];
+ ConfigValidationResult []empty = new ConfigValidationResult[0];
- ValidationResult []retrieved = transfer(empty);
+ ConfigValidationResult []retrieved = transfer(empty);
assertEquals(0, retrieved.length);
}
@Test
public void testOneMessage() {
- ValidationResult []empty = new ValidationResult[] {
+ ConfigValidationResult []empty = new ConfigValidationResult[] {
getResultA()
};
- ValidationResult []retrieved = transfer(empty);
+ ConfigValidationResult []retrieved = transfer(empty);
assertEquals(1, retrieved.length);
verifyResultA(retrieved[0]);
}
@Test
public void testTwoMessages() {
- ValidationResult []empty = new ValidationResult[] {
+ ConfigValidationResult []empty = new ConfigValidationResult[] {
getResultA(),
getResultA()
};
- ValidationResult []retrieved = transfer(empty);
+ ConfigValidationResult []retrieved = transfer(empty);
assertEquals(2, retrieved.length);
verifyResultA(retrieved[0]);
@@ -79,7 +79,7 @@ public class TestValidationResultBean {
assertNull(idNull);
}
- public void verifyResultA(ValidationResult result) {
+ public void verifyResultA(ConfigValidationResult result) {
assertNotNull(result);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
@@ -98,8 +98,8 @@ public class TestValidationResultBean {
assertEquals("B", messagesA.get(1).getMessage());
}
- public ValidationResult getResultA() {
- ValidationResult result = new ValidationResult();
+ public ConfigValidationResult getResultA() {
+ ConfigValidationResult result = new ConfigValidationResult();
List<Message> messages = new LinkedList<Message>();
messages.add(new Message(Status.ACCEPTABLE, "A"));
messages.add(new Message(Status.UNACCEPTABLE, "B"));
@@ -109,7 +109,7 @@ public class TestValidationResultBean {
private Long transfer(Long id) {
- ValidationResultBean bean = new ValidationResultBean(new ValidationResult[0]);
+ ValidationResultBean bean = new ValidationResultBean(new ConfigValidationResult[0]);
bean.setId(id);
JSONObject json = bean.extract(false);
@@ -122,7 +122,7 @@ public class TestValidationResultBean {
return retrievedBean.getId();
}
- private ValidationResult[] transfer(ValidationResult [] results) {
+ private ConfigValidationResult[] transfer(ConfigValidationResult [] results) {
ValidationResultBean bean = new ValidationResultBean(results);
JSONObject json = bean.extract(false);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java
new file mode 100644
index 0000000..4f0c84d
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.json.util;
+
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.model.MBooleanInput;
+import org.apache.sqoop.model.MEnumInput;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MMapInput;
+import org.apache.sqoop.model.MStringInput;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ *
+ */
+public class TestConfigSerialization {
+
+ @Test
+ public void testAllDataTypes() {
+ // Inserted values
+ Map<String, String> map = new HashMap<String, String>();
+ map.put("A", "B");
+
+ // Fill config with all values
+ MConfig config = getConfig();
+ config.getStringInput("String").setValue("A");
+ config.getMapInput("Map").setValue(map);
+ config.getIntegerInput("Integer").setValue(1);
+ config.getBooleanInput("Boolean").setValue(true);
+ config.getEnumInput("Enum").setValue("YES");
+
+ // Serialize that into JSON
+ JSONObject jsonObject = ConfigSerialization.extractConfig(config, false);
+ assertNotNull(jsonObject);
+
+ // Exchange the data on string level
+ String serializedJson = jsonObject.toJSONString();
+ JSONObject retrievedJson = (JSONObject) JSONValue.parse(serializedJson);
+
+ // And retrieve back from JSON representation
+ MConfig retrieved = ConfigSerialization.restoreConfig(retrievedJson);
+
+ // Verify all expected values
+ assertEquals("A", retrieved.getStringInput("String").getValue());
+ assertEquals(map, retrieved.getMapInput("Map").getValue());
+ assertEquals(1, (int)retrieved.getIntegerInput("Integer").getValue());
+ assertEquals(true, retrieved.getBooleanInput("Boolean").getValue());
+ assertEquals("YES", retrieved.getEnumInput("Enum").getValue());
+ }
+
+ @Test
+ public void testMapDataType() {
+ MConfig config = getMapConfig();
+
+ // Inserted values
+ Map<String, String> map = new HashMap<String, String>();
+ map.put("A", "B");
+ config.getMapInput("Map").setValue(map);
+
+ // Serialize
+ JSONObject jsonObject = ConfigSerialization.extractConfig(config, false);
+ String serializedJson = jsonObject.toJSONString();
+
+ // Deserialize
+ JSONObject retrievedJson = (JSONObject) JSONValue.parse(serializedJson);
+ MConfig retrieved = ConfigSerialization.restoreConfig(retrievedJson);
+ assertEquals(map, retrieved.getMapInput("Map").getValue());
+ }
+
+ @Test(expected=SqoopException.class)
+ public void testMapDataTypeException() {
+ MConfig config = getMapConfig();
+
+ // Inserted values
+ Map<String, String> map = new HashMap<String, String>();
+ map.put("A", "B");
+ config.getMapInput("Map").setValue(map);
+
+ // Serialize
+ JSONObject jsonObject = ConfigSerialization.extractConfig(config, false);
+ String serializedJson = jsonObject.toJSONString();
+
+ // Replace map value with a fake string to force exception
+ String badSerializedJson = serializedJson.replace("{\"A\":\"B\"}", "\"nonsensical string\"");
+ System.out.println(badSerializedJson);
+ JSONObject retrievedJson = (JSONObject) JSONValue.parse(badSerializedJson);
+ ConfigSerialization.restoreConfig(retrievedJson);
+ }
+
+ protected MConfig getMapConfig() {
+ List<MInput<?>> inputs;
+ MInput input;
+
+ inputs = new LinkedList<MInput<?>>();
+
+ input = new MMapInput("Map", false);
+ inputs.add(input);
+
+ return new MConfig("c", inputs);
+ }
+
+ /**
+ * Return config with all data types.
+ *
+ * @return
+ */
+ protected MConfig getConfig() {
+ List<MInput<?>> inputs;
+ MInput input;
+
+ inputs = new LinkedList<MInput<?>>();
+
+ input = new MStringInput("String", false, (short)30);
+ inputs.add(input);
+
+ input = new MMapInput("Map", false);
+ inputs.add(input);
+
+ input = new MIntegerInput("Integer", false);
+ inputs.add(input);
+
+ input = new MBooleanInput("Boolean", false);
+ inputs.add(input);
+
+ input = new MEnumInput("Enum", false, new String[] {"YES", "NO"});
+ inputs.add(input);
+
+ return new MConfig("c", inputs);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/json/util/TestFormSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/util/TestFormSerialization.java b/common/src/test/java/org/apache/sqoop/json/util/TestFormSerialization.java
deleted file mode 100644
index c4223ec..0000000
--- a/common/src/test/java/org/apache/sqoop/json/util/TestFormSerialization.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.json.util;
-
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MEnumInput;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MIntegerInput;
-import org.apache.sqoop.model.MMapInput;
-import org.apache.sqoop.model.MStringInput;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- *
- */
-public class TestFormSerialization {
-
- @Test
- public void testAllDataTypes() {
- // Inserted values
- Map<String, String> map = new HashMap<String, String>();
- map.put("A", "B");
-
- // Fill form with all values
- MForm form = getForm();
- form.getStringInput("String").setValue("A");
- form.getMapInput("Map").setValue(map);
- form.getIntegerInput("Integer").setValue(1);
- form.getBooleanInput("Boolean").setValue(true);
- form.getEnumInput("Enum").setValue("YES");
-
- // Serialize that into JSON
- JSONObject jsonObject = FormSerialization.extractForm(form, false);
- assertNotNull(jsonObject);
-
- // Exchange the data on string level
- String serializedJson = jsonObject.toJSONString();
- JSONObject retrievedJson = (JSONObject) JSONValue.parse(serializedJson);
-
- // And retrieve back from JSON representation
- MForm retrieved = FormSerialization.restoreForm(retrievedJson);
-
- // Verify all expected values
- assertEquals("A", retrieved.getStringInput("String").getValue());
- assertEquals(map, retrieved.getMapInput("Map").getValue());
- assertEquals(1, (int)retrieved.getIntegerInput("Integer").getValue());
- assertEquals(true, retrieved.getBooleanInput("Boolean").getValue());
- assertEquals("YES", retrieved.getEnumInput("Enum").getValue());
- }
-
- @Test
- public void testMapDataType() {
- MForm form = getMapForm();
-
- // Inserted values
- Map<String, String> map = new HashMap<String, String>();
- map.put("A", "B");
- form.getMapInput("Map").setValue(map);
-
- // Serialize
- JSONObject jsonObject = FormSerialization.extractForm(form, false);
- String serializedJson = jsonObject.toJSONString();
-
- // Deserialize
- JSONObject retrievedJson = (JSONObject) JSONValue.parse(serializedJson);
- MForm retrieved = FormSerialization.restoreForm(retrievedJson);
- assertEquals(map, retrieved.getMapInput("Map").getValue());
- }
-
- @Test(expected=SqoopException.class)
- public void testMapDataTypeException() {
- MForm form = getMapForm();
-
- // Inserted values
- Map<String, String> map = new HashMap<String, String>();
- map.put("A", "B");
- form.getMapInput("Map").setValue(map);
-
- // Serialize
- JSONObject jsonObject = FormSerialization.extractForm(form, false);
- String serializedJson = jsonObject.toJSONString();
-
- // Replace map value with a fake string to force exception
- String badSerializedJson = serializedJson.replace("{\"A\":\"B\"}", "\"nonsensical string\"");
- System.out.println(badSerializedJson);
- JSONObject retrievedJson = (JSONObject) JSONValue.parse(badSerializedJson);
- FormSerialization.restoreForm(retrievedJson);
- }
-
- protected MForm getMapForm() {
- List<MInput<?>> inputs;
- MInput input;
-
- inputs = new LinkedList<MInput<?>>();
-
- input = new MMapInput("Map", false);
- inputs.add(input);
-
- return new MForm("f", inputs);
- }
-
- /**
- * Return form with all data types.
- *
- * @return
- */
- protected MForm getForm() {
- List<MInput<?>> inputs;
- MInput input;
-
- inputs = new LinkedList<MInput<?>>();
-
- input = new MStringInput("String", false, (short)30);
- inputs.add(input);
-
- input = new MMapInput("Map", false);
- inputs.add(input);
-
- input = new MIntegerInput("Integer", false);
- inputs.add(input);
-
- input = new MBooleanInput("Boolean", false);
- inputs.add(input);
-
- input = new MEnumInput("Enum", false, new String[] {"YES", "NO"});
- inputs.add(input);
-
- return new MForm("f", inputs);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java b/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java
new file mode 100644
index 0000000..d5377f8
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.sqoop.common.SqoopException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test config utils
+ */
+public class TestConfigUtils extends TestCase {
+
+ public void testConfigs() {
+ TestConfiguration config = new TestConfiguration();
+ config.aConfig.a1 = "value";
+
+ List<MConfig> configsByInstance = ConfigUtils.toConfigs(config);
+ assertEquals(getConfigs(), configsByInstance);
+ assertEquals("value", configsByInstance.get(0).getInputs().get(0).getValue());
+
+ List<MConfig> configsByClass = ConfigUtils.toConfigs(TestConfiguration.class);
+ assertEquals(getConfigs(), configsByClass);
+
+ List<MConfig> configsByBoth = ConfigUtils.toConfigs(TestConfiguration.class, config);
+ assertEquals(getConfigs(), configsByBoth);
+ assertEquals("value", configsByBoth.get(0).getInputs().get(0).getValue());
+ }
+
+ public void testConfigsMissingAnnotation() {
+ try {
+ ConfigUtils.toConfigs(ConfigWithout.class);
+ } catch(SqoopException ex) {
+ assertEquals(ModelError.MODEL_003, ex.getErrorCode());
+ return;
+ }
+
+ fail("Correct exception wasn't thrown");
+ }
+
+ public void testNonUniqueFormNameAttributes() {
+ try {
+ ConfigUtils.toConfigs(ConfigurationWithNonUniqueFormNameAttribute.class);
+ } catch (SqoopException ex) {
+ assertEquals(ModelError.MODEL_012, ex.getErrorCode());
+ return;
+ }
+
+ fail("Correct exception wasn't thrown");
+ }
+
+ public void testInvalidFormNameAttribute() {
+ try {
+ ConfigUtils.toConfigs(ConfigurationWithInvalidFormNameAttribute.class);
+ } catch (SqoopException ex) {
+ assertEquals(ModelError.MODEL_013, ex.getErrorCode());
+ return;
+ }
+
+ fail("Correct exception wasn't thrown");
+ }
+
+ public void testInvalidFormNameAttributeLength() {
+ try {
+ ConfigUtils.toConfigs(ConfigurationWithInvalidFormNameAttributeLength.class);
+ } catch (SqoopException ex) {
+ assertEquals(ModelError.MODEL_014, ex.getErrorCode());
+ return;
+ }
+ fail("Correct exception wasn't thrown");
+ }
+
+ public void testFailureOnPrimitiveType() {
+ PrimitiveConfig config = new PrimitiveConfig();
+
+ try {
+ ConfigUtils.toConfigs(config);
+ fail("We were expecting exception for unsupported type.");
+ } catch(SqoopException ex) {
+ assertEquals(ModelError.MODEL_007, ex.getErrorCode());
+ }
+ }
+
+ public void testFillValues() {
+ List<MConfig> configs = getConfigs();
+
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("value");
+
+ TestConfiguration config = new TestConfiguration();
+
+ ConfigUtils.fromConfigs(configs, config);
+ assertEquals("value", config.aConfig.a1);
+ }
+
+ public void testFillValuesObjectReuse() {
+ List<MConfig> configs = getConfigs();
+
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("value");
+
+ TestConfiguration config = new TestConfiguration();
+ config.aConfig.a2 = "x";
+ config.bConfig.b1 = "y";
+
+ ConfigUtils.fromConfigs(configs, config);
+ assertEquals("value", config.aConfig.a1);
+ assertNull(config.aConfig.a2);
+ assertNull(config.bConfig.b2);
+ assertNull(config.bConfig.b2);
+ }
+
+ public void testJson() {
+ TestConfiguration config = new TestConfiguration();
+ config.aConfig.a1 = "A";
+ config.bConfig.b2 = "B";
+ config.cConfig.intValue = 4;
+ config.cConfig.map.put("C", "D");
+ config.cConfig.enumeration = Enumeration.X;
+
+ String json = ConfigUtils.toJson(config);
+
+ TestConfiguration targetConfig = new TestConfiguration();
+
+ // Old values from should be always removed
+ targetConfig.aConfig.a2 = "X";
+ targetConfig.bConfig.b1 = "Y";
+ // Nulls in configs shouldn't be an issue either
+ targetConfig.cConfig = null;
+
+ ConfigUtils.fillValues(json, targetConfig);
+
+ assertEquals("A", targetConfig.aConfig.a1);
+ assertNull(targetConfig.aConfig.a2);
+
+ assertNull(targetConfig.bConfig.b1);
+ assertEquals("B", targetConfig.bConfig.b2);
+
+ assertEquals((Integer)4, targetConfig.cConfig.intValue);
+ assertEquals(1, targetConfig.cConfig.map.size());
+ assertTrue(targetConfig.cConfig.map.containsKey("C"));
+ assertEquals("D", targetConfig.cConfig.map.get("C"));
+ assertEquals(Enumeration.X, targetConfig.cConfig.enumeration);
+ }
+
+ /**
+ * Config structure that corresponds to Config class declared below
+ * @return Config structure
+ */
+ protected List<MConfig> getConfigs() {
+ List<MConfig> ret = new LinkedList<MConfig>();
+
+ List<MInput<?>> inputs;
+
+ // Config A
+ inputs = new LinkedList<MInput<?>>();
+ inputs.add(new MStringInput("aConfig.a1", false, (short)30));
+ inputs.add(new MStringInput("aConfig.a2", true, (short)-1));
+ ret.add(new MConfig("aConfig", inputs));
+
+ // Config B
+ inputs = new LinkedList<MInput<?>>();
+ inputs.add(new MStringInput("bConfig.b1", false, (short)2));
+ inputs.add(new MStringInput("bConfig.b2", false, (short)3));
+ ret.add(new MConfig("bConfig", inputs));
+
+ // Config C
+ inputs = new LinkedList<MInput<?>>();
+ inputs.add(new MIntegerInput("cConfig.intValue", false));
+ inputs.add(new MMapInput("cConfig.map", false));
+ inputs.add(new MEnumInput("cConfig.enumeration", false, new String[]{"X", "Y"}));
+ ret.add(new MConfig("cConfig", inputs));
+
+ return ret;
+ }
+
+ @ConfigurationClass
+ public static class ConfigurationWithNonUniqueFormNameAttribute {
+ public ConfigurationWithNonUniqueFormNameAttribute() {
+ aForm = new InvalidConfig();
+ bForm = new InvalidConfig();
+ }
+
+ @Config(name = "sameName")
+ InvalidConfig aForm;
+ @Config(name = "sameName")
+ InvalidConfig bForm;
+ }
+
+ @ConfigurationClass
+ public static class ConfigurationWithInvalidFormNameAttribute {
+ public ConfigurationWithInvalidFormNameAttribute() {
+ invalidForm = new InvalidConfig();
+ }
+
+ @Config(name = "#_form")
+ InvalidConfig invalidForm;
+ }
+
+ @ConfigurationClass
+ public static class ConfigurationWithInvalidFormNameAttributeLength {
+ public ConfigurationWithInvalidFormNameAttributeLength() {
+ invalidLengthForm = new InvalidConfig();
+ }
+
+ @Config(name = "longest_form_more_than_30_characers")
+ InvalidConfig invalidLengthForm;
+ }
+
+ @ConfigurationClass
+ public static class TestConfiguration {
+
+ public TestConfiguration() {
+ aConfig = new AConfig();
+ bConfig = new BConfig();
+ cConfig = new CConfig();
+ }
+
+ @Config AConfig aConfig;
+ @Config BConfig bConfig;
+ @Config CConfig cConfig;
+ }
+
+ @ConfigurationClass
+ public static class PrimitiveConfig {
+ @Config DConfig dConfig;
+ }
+
+ @ConfigClass
+ public static class AConfig {
+ @Input(size = 30) String a1;
+ @Input(sensitive = true) String a2;
+ }
+
+ @ConfigClass
+ public static class BConfig {
+ @Input(size = 2) String b1;
+ @Input(size = 3) String b2;
+ }
+
+ @ConfigClass
+ public static class CConfig {
+ @Input Integer intValue;
+ @Input Map<String, String> map;
+ @Input Enumeration enumeration;
+
+ public CConfig() {
+ map = new HashMap<String, String>();
+ }
+ }
+
+ @ConfigClass
+ public static class InvalidConfig {
+
+ }
+
+ @ConfigClass
+ public static class DConfig {
+ @Input int value;
+ }
+
+ public static class ConfigWithout {
+ }
+
+ enum Enumeration {
+ X,
+ Y,
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java b/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
deleted file mode 100644
index d2a3476..0000000
--- a/common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-
-/**
- * Test form utils
- */
-public class TestFormUtils {
-
- @Test
- public void testToForms() {
- Config config = new Config();
- config.aForm.a1 = "value";
-
- List<MForm> formsByInstance = FormUtils.toForms(config);
- assertEquals(getForms(), formsByInstance);
- assertEquals("value", formsByInstance.get(0).getInputs().get(0).getValue());
-
- List<MForm> formsByClass = FormUtils.toForms(Config.class);
- assertEquals(getForms(), formsByClass);
-
- List<MForm> formsByBoth = FormUtils.toForms(Config.class, config);
- assertEquals(getForms(), formsByBoth);
- assertEquals("value", formsByBoth.get(0).getInputs().get(0).getValue());
- }
-
- @Test
- public void testToFormsMissingAnnotation() {
- try {
- FormUtils.toForms(ConfigWithout.class);
- } catch(SqoopException ex) {
- assertEquals(ModelError.MODEL_003, ex.getErrorCode());
- return;
- }
-
- fail("Correct exception wasn't thrown");
- }
-
- public void testNonUniqueFormNameAttributes() {
- try {
- FormUtils.toForms(ConfigurationWithNonUniqueFormNameAttribute.class);
- } catch (SqoopException ex) {
- assertEquals(ModelError.MODEL_012, ex.getErrorCode());
- return;
- }
-
- fail("Correct exception wasn't thrown");
- }
-
- public void testInvalidFormNameAttribute() {
- try {
- FormUtils.toForms(ConfigurationWithInvalidFormNameAttribute.class);
- } catch (SqoopException ex) {
- assertEquals(ModelError.MODEL_013, ex.getErrorCode());
- return;
- }
-
- fail("Correct exception wasn't thrown");
- }
-
- public void testInvalidFormNameAttributeLength() {
- try {
- FormUtils.toForms(ConfigurationWithInvalidFormNameAttributeLength.class);
- } catch (SqoopException ex) {
- assertEquals(ModelError.MODEL_014, ex.getErrorCode());
- return;
- }
- fail("Correct exception wasn't thrown");
- }
-
- @Test
- public void testFailureOnPrimitiveType() {
- PrimitiveConfig config = new PrimitiveConfig();
-
- try {
- FormUtils.toForms(config);
- fail("We were expecting exception for unsupported type.");
- } catch(SqoopException ex) {
- assertEquals(ModelError.MODEL_007, ex.getErrorCode());
- }
- }
-
- @Test
- public void testFillValues() {
- List<MForm> forms = getForms();
-
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("value");
-
- Config config = new Config();
-
- FormUtils.fromForms(forms, config);
- assertEquals("value", config.aForm.a1);
- }
-
- @Test
- public void testFillValuesObjectReuse() {
- List<MForm> forms = getForms();
-
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("value");
-
- Config config = new Config();
- config.aForm.a2 = "x";
- config.bForm.b1 = "y";
-
- FormUtils.fromForms(forms, config);
- assertEquals("value", config.aForm.a1);
- assertNull(config.aForm.a2);
- assertNull(config.bForm.b2);
- assertNull(config.bForm.b2);
- }
-
- @Test
- public void testJson() {
- Config config = new Config();
- config.aForm.a1 = "A";
- config.bForm.b2 = "B";
- config.cForm.intValue = 4;
- config.cForm.map.put("C", "D");
- config.cForm.enumeration = Enumeration.X;
-
- String json = FormUtils.toJson(config);
-
- Config targetConfig = new Config();
-
- // Old values from should be always removed
- targetConfig.aForm.a2 = "X";
- targetConfig.bForm.b1 = "Y";
- // Nulls in forms shouldn't be an issue either
- targetConfig.cForm = null;
-
- FormUtils.fillValues(json, targetConfig);
-
- assertEquals("A", targetConfig.aForm.a1);
- assertNull(targetConfig.aForm.a2);
-
- assertNull(targetConfig.bForm.b1);
- assertEquals("B", targetConfig.bForm.b2);
-
- assertEquals((Integer)4, targetConfig.cForm.intValue);
- assertEquals(1, targetConfig.cForm.map.size());
- assertTrue(targetConfig.cForm.map.containsKey("C"));
- assertEquals("D", targetConfig.cForm.map.get("C"));
- assertEquals(Enumeration.X, targetConfig.cForm.enumeration);
- }
-
- /**
- * Form structure that corresponds to Config class declared below
- * @return Form structure
- */
- protected List<MForm> getForms() {
- List<MForm> ret = new LinkedList<MForm>();
-
- List<MInput<?>> inputs;
-
- // Form A
- inputs = new LinkedList<MInput<?>>();
- inputs.add(new MStringInput("aForm.a1", false, (short)30));
- inputs.add(new MStringInput("aForm.a2", true, (short)-1));
- ret.add(new MForm("aForm", inputs));
-
- // Form B
- inputs = new LinkedList<MInput<?>>();
- inputs.add(new MStringInput("bForm.b1", false, (short)2));
- inputs.add(new MStringInput("bForm.b2", false, (short)3));
- ret.add(new MForm("bForm", inputs));
-
- // Form C
- inputs = new LinkedList<MInput<?>>();
- inputs.add(new MIntegerInput("cForm.intValue", false));
- inputs.add(new MMapInput("cForm.map", false));
- inputs.add(new MEnumInput("cForm.enumeration", false, new String[]{"X", "Y"}));
- ret.add(new MForm("cForm", inputs));
-
- return ret;
- }
-
- @ConfigurationClass
- public static class ConfigurationWithNonUniqueFormNameAttribute {
- public ConfigurationWithNonUniqueFormNameAttribute() {
- aForm = new InvalidForm();
- bForm = new InvalidForm();
- }
-
- @Form(name = "sameName")
- InvalidForm aForm;
- @Form(name = "sameName")
- InvalidForm bForm;
- }
-
- @ConfigurationClass
- public static class ConfigurationWithInvalidFormNameAttribute {
- public ConfigurationWithInvalidFormNameAttribute() {
- invalidForm = new InvalidForm();
- }
-
- @Form(name = "#_form")
- InvalidForm invalidForm;
- }
-
- @ConfigurationClass
- public static class ConfigurationWithInvalidFormNameAttributeLength {
- public ConfigurationWithInvalidFormNameAttributeLength() {
- invalidLengthForm = new InvalidForm();
- }
-
- @Form(name = "longest_form_more_than_30_characers")
- InvalidForm invalidLengthForm;
- }
-
- @ConfigurationClass
- public static class Config {
-
- public Config() {
- aForm = new AForm();
- bForm = new BForm();
- cForm = new CForm();
- }
-
- @Form AForm aForm;
- @Form BForm bForm;
- @Form CForm cForm;
- }
-
- @ConfigurationClass
- public static class PrimitiveConfig {
- @Form DForm dForm;
- }
-
- @FormClass
- public static class AForm {
- @Input(size = 30) String a1;
- @Input(sensitive = true) String a2;
- }
-
- @FormClass
- public static class BForm {
- @Input(size = 2) String b1;
- @Input(size = 3) String b2;
- }
-
- @FormClass
- public static class CForm {
- @Input Integer intValue;
- @Input Map<String, String> map;
- @Input Enumeration enumeration;
-
- public CForm() {
- map = new HashMap<String, String>();
- }
- }
-
- @FormClass
- public static class InvalidForm {
-
- }
- @FormClass
- public static class DForm {
- @Input int value;
- }
-
- public static class ConfigWithout {
- }
-
- enum Enumeration {
- X,
- Y,
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java b/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
index af0f450..ba53739 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
@@ -35,14 +35,13 @@ public class TestMAccountableEntity {
*/
@Test
public void testInitialization() {
- List<MForm> forms = new ArrayList<MForm>();
+ List<MConfig> configs = new ArrayList<MConfig>();
MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
- MForm form = new MForm("FORMNAME", list);
- forms.add(form);
- MAccountableEntity link = new MLink(123l, new MConnectionForms(
- forms), new MConnectionForms(forms));
+ MConfig config = new MConfig("CONFIGNAME", list);
+ configs.add(config);
+ MAccountableEntity link = new MLink(123l, new MLinkConfig(configs));
// Initially creation date and last update date is same
assertEquals(link.getCreationDate(), link.getLastUpdateDate());
Date testCreationDate = new Date();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfig.java b/common/src/test/java/org/apache/sqoop/model/TestMConfig.java
new file mode 100644
index 0000000..c5a07a0
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConfig.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestMConfig {
+
+ /**
+ * Test for initialization
+ */
+ @Test
+ public void testInitialization() {
+ MInput<String> input1 = new MStringInput("sqoopsqoop1", true, (short) 5);
+ MInput<String> input2 = new MStringInput("sqoopsqoop2", true, (short) 5);
+
+ List<MInput<?>> list = new ArrayList<MInput<?>>();
+ list.add(input1);
+ list.add(input2);
+ MConfig mConfig = new MConfig("config", list);
+
+ assertEquals("config", mConfig.getName());
+ assertEquals(2, mConfig.getInputs().size());
+ }
+
+ /**
+ * Test for equals method
+ */
+ @Test
+ public void testEquals() {
+ MInput<Integer> input1 = new MIntegerInput("sqoopsqoop1", false);
+ MInput<Integer> input2 = new MIntegerInput("sqoopsqoop2", false);
+ List<MInput<?>> list1 = new ArrayList<MInput<?>>();
+ list1.add(input1);
+ list1.add(input2);
+ MConfig mform1 = new MConfig("config", list1);
+
+ MInput<Integer> input3 = new MIntegerInput("sqoopsqoop1", false);
+ MInput<Integer> input4 = new MIntegerInput("sqoopsqoop2", false);
+ List<MInput<?>> list2 = new ArrayList<MInput<?>>();
+ list2.add(input3);
+ list2.add(input4);
+ MConfig mform2 = new MConfig("config", list2);
+ assertEquals(mform2, mform1);
+ }
+
+ @Test
+ public void testGetInputs() {
+ MIntegerInput intInput = new MIntegerInput("Config.A", false);
+ MMapInput mapInput = new MMapInput("Config.B", false);
+ MStringInput stringInput = new MStringInput("Config.C", false, (short)3);
+ MEnumInput enumInput = new MEnumInput("Config.D", false, new String[] {"I", "V"});
+
+ List<MInput<?>> inputs = new ArrayList<MInput<?>>();
+ inputs.add(intInput);
+ inputs.add(mapInput);
+ inputs.add(stringInput);
+ inputs.add(enumInput);
+
+ MConfig config = new MConfig("Config", inputs);
+ assertEquals(intInput, config.getIntegerInput("Config.A"));
+ assertEquals(mapInput, config.getMapInput("Config.B"));
+ assertEquals(stringInput, config.getStringInput("Config.C"));
+ assertEquals(enumInput, config.getEnumInput("Config.D"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java
new file mode 100644
index 0000000..9b60055
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.model;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestMConfigList {
+ @Test
+ public void testGetInputs() {
+ List<MConfig> configs = new LinkedList<MConfig>();
+
+ MIntegerInput intInput = new MIntegerInput("Config1.A", false);
+ MMapInput mapInput = new MMapInput("Config1.B", false);
+
+ List<MInput<?>> inputs = new ArrayList<MInput<?>>();
+ inputs.add(intInput);
+ inputs.add(mapInput);
+ configs.add(new MConfig("Config1", inputs));
+
+ MStringInput stringInput = new MStringInput("Config2.C", false, (short)3);
+ MEnumInput enumInput = new MEnumInput("Config2.D", false, new String[] {"I", "V"});
+
+ inputs = new ArrayList<MInput<?>>();
+ inputs.add(stringInput);
+ inputs.add(enumInput);
+ configs.add(new MConfig("Config2", inputs));
+
+ MConfigList config = new MConfigList(configs);
+ assertEquals(intInput, config.getIntegerInput("Config1.A"));
+ assertEquals(mapInput, config.getMapInput("Config1.B"));
+ assertEquals(stringInput, config.getStringInput("Config2.C"));
+ assertEquals(enumInput, config.getEnumInput("Config2.D"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java b/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java
deleted file mode 100644
index 243fff9..0000000
--- a/common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- * Test class for org.apache.sqoop.model.MConnectionForms
- */
-public class TestMConnectionForms {
-
- /**
- * Test for class initialization and values
- */
- @Test
- public void testInitialization() {
- List<MForm> forms = new ArrayList<MForm>();
- MConnectionForms connectionForms1 = new MConnectionForms(forms);
- List<MForm> testForms = new ArrayList<MForm>();
- assertEquals(testForms, connectionForms1.getForms());
- MConnectionForms connectionForms2 = new MConnectionForms(testForms);
- assertEquals(connectionForms2, connectionForms1);
- // Add a form to list for checking not equals
- MForm m = new MForm("test", null);
- testForms.add(m);
- assertFalse(connectionForms1.equals(connectionForms2));
- }
-}
[38/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
index 47350ea..595b1c8 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
@@ -17,9 +17,14 @@
*/
package org.apache.sqoop.repository.derby;
+import java.sql.Connection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
@@ -38,18 +43,19 @@ import static org.junit.Assert.*;
public class TestJobHandling extends DerbyTestCase {
DerbyRepositoryHandler handler;
+ Connection derbyConnection;
@Before
public void setUp() throws Exception {
super.setUp();
+ derbyConnection = getDerbyDatabaseConnection();
handler = new DerbyRepositoryHandler();
// We always needs schema for this test case
createSchema();
- // We always needs connector and framework structures in place
- loadConnectorAndDriverConfig();
+ loadConnectorLinkConfig();
// We always needs connection metadata in place
loadLinks();
@@ -59,58 +65,53 @@ public class TestJobHandling extends DerbyTestCase {
public void testFindJob() throws Exception {
// Let's try to find non existing job
try {
- handler.findJob(1, getDerbyDatabaseConnection());
+ handler.findJob(1, derbyConnection);
fail();
} catch(SqoopException ex) {
assertEquals(DerbyRepoError.DERBYREPO_0030, ex.getErrorCode());
}
- // Load prepared connections into database
loadJobs();
- MJob jobImport = handler.findJob(1, getDerbyDatabaseConnection());
- assertNotNull(jobImport);
- assertEquals(1, jobImport.getPersistenceId());
- assertEquals("JA", jobImport.getName());
-
- List<MForm> forms;
-
- // Check connector parts
- forms = jobImport.getConnectorPart(Direction.FROM).getForms();
- assertEquals(2, forms.size());
- assertEquals("Value5", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value5", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- forms = jobImport.getConnectorPart(Direction.TO).getForms();
- assertEquals(2, forms.size());
- assertEquals("Value9", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value9", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- // Check framework part
- forms = jobImport.getFrameworkPart().getForms();
- assertEquals(2, forms.size());
- assertEquals("Value17", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value19", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
+ MJob firstJob = handler.findJob(1, derbyConnection);
+ assertNotNull(firstJob);
+ assertEquals(1, firstJob.getPersistenceId());
+ assertEquals("JA", firstJob.getName());
+
+ List<MConfig> configs;
+
+ configs = firstJob.getJobConfig(Direction.FROM).getConfigs();
+ assertEquals(2, configs.size());
+ assertEquals("Value5", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ assertEquals("Value5", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(1).getInputs().get(1).getValue());
+
+ configs = firstJob.getJobConfig(Direction.TO).getConfigs();
+ assertEquals(2, configs.size());
+ assertEquals("Value9", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ assertEquals("Value9", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(1).getInputs().get(1).getValue());
+
+ configs = firstJob.getDriverConfig().getConfigs();
+ assertEquals(2, configs.size());
+ assertEquals("Value13", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ assertEquals("Value15", configs.get(1).getInputs().get(0).getValue());
+ assertNull(configs.get(1).getInputs().get(1).getValue());
}
@Test
public void testFindJobs() throws Exception {
List<MJob> list;
-
// Load empty list on empty repository
- list = handler.findJobs(getDerbyDatabaseConnection());
+ list = handler.findJobs(derbyConnection);
assertEquals(0, list.size());
-
loadJobs();
// Load all two connections on loaded repository
- list = handler.findJobs(getDerbyDatabaseConnection());
+ list = handler.findJobs(derbyConnection);
assertEquals(4, list.size());
assertEquals("JA", list.get(0).getName());
@@ -125,19 +126,19 @@ public class TestJobHandling extends DerbyTestCase {
@Test
public void testExistsJob() throws Exception {
// There shouldn't be anything on empty repository
- assertFalse(handler.existsJob(1, getDerbyDatabaseConnection()));
- assertFalse(handler.existsJob(2, getDerbyDatabaseConnection()));
- assertFalse(handler.existsJob(3, getDerbyDatabaseConnection()));
- assertFalse(handler.existsJob(4, getDerbyDatabaseConnection()));
- assertFalse(handler.existsJob(5, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsJob(1, derbyConnection));
+ assertFalse(handler.existsJob(2, derbyConnection));
+ assertFalse(handler.existsJob(3, derbyConnection));
+ assertFalse(handler.existsJob(4, derbyConnection));
+ assertFalse(handler.existsJob(5, derbyConnection));
loadJobs();
- assertTrue(handler.existsJob(1, getDerbyDatabaseConnection()));
- assertTrue(handler.existsJob(2, getDerbyDatabaseConnection()));
- assertTrue(handler.existsJob(3, getDerbyDatabaseConnection()));
- assertTrue(handler.existsJob(4, getDerbyDatabaseConnection()));
- assertFalse(handler.existsJob(5, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsJob(1, derbyConnection));
+ assertTrue(handler.existsJob(2, derbyConnection));
+ assertTrue(handler.existsJob(3, derbyConnection));
+ assertTrue(handler.existsJob(4, derbyConnection));
+ assertFalse(handler.existsJob(5, derbyConnection));
}
@Test
@@ -145,10 +146,10 @@ public class TestJobHandling extends DerbyTestCase {
loadJobs();
loadSubmissions();
- assertTrue(handler.inUseJob(1, getDerbyDatabaseConnection()));
- assertFalse(handler.inUseJob(2, getDerbyDatabaseConnection()));
- assertFalse(handler.inUseJob(3, getDerbyDatabaseConnection()));
- assertFalse(handler.inUseJob(4, getDerbyDatabaseConnection()));
+ assertTrue(handler.inUseJob(1, derbyConnection));
+ assertFalse(handler.inUseJob(2, derbyConnection));
+ assertFalse(handler.inUseJob(3, derbyConnection));
+ assertFalse(handler.inUseJob(4, derbyConnection));
}
@Test
@@ -158,34 +159,34 @@ public class TestJobHandling extends DerbyTestCase {
// Load some data
fillJob(job);
- handler.createJob(job, getDerbyDatabaseConnection());
+ handler.createJob(job, derbyConnection);
assertEquals(1, job.getPersistenceId());
assertCountForTable("SQOOP.SQ_JOB", 1);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 6);
- MJob retrieved = handler.findJob(1, getDerbyDatabaseConnection());
+ MJob retrieved = handler.findJob(1, derbyConnection);
assertEquals(1, retrieved.getPersistenceId());
- List<MForm> forms;
- forms = job.getConnectorPart(Direction.FROM).getForms();
- assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- forms = job.getConnectorPart(Direction.TO).getForms();
- assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
+ List<MConfig> configs;
+ configs = job.getJobConfig(Direction.FROM).getConfigs();
+ assertEquals("Value1", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ configs = job.getJobConfig(Direction.TO).getConfigs();
+ assertEquals("Value1", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
- forms = job.getFrameworkPart().getForms();
- assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
+ configs = job.getDriverConfig().getConfigs();
+ assertEquals("Value13", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ assertEquals("Value15", configs.get(1).getInputs().get(0).getValue());
+ assertNull(configs.get(1).getInputs().get(1).getValue());
// Let's create second job
job = getJob();
fillJob(job);
- handler.createJob(job, getDerbyDatabaseConnection());
+ handler.createJob(job, derbyConnection);
assertEquals(2, job.getPersistenceId());
assertCountForTable("SQOOP.SQ_JOB", 2);
@@ -199,48 +200,49 @@ public class TestJobHandling extends DerbyTestCase {
assertCountForTable("SQOOP.SQ_JOB", 4);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 24);
- MJob job = handler.findJob(1, getDerbyDatabaseConnection());
+ MJob job = handler.findJob(1, derbyConnection);
+
+ List<MConfig> configs;
- List<MForm> forms;
+ configs = job.getJobConfig(Direction.FROM).getConfigs();
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)configs.get(0).getInputs().get(1)).setValue(null);
- forms = job.getConnectorPart(Direction.FROM).getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
- forms = job.getConnectorPart(Direction.TO).getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
+ configs = job.getJobConfig(Direction.TO).getConfigs();
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)configs.get(0).getInputs().get(1)).setValue(null);
- forms = job.getFrameworkPart().getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+ configs = job.getDriverConfig().getConfigs();
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)configs.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+ ((MStringInput)configs.get(1).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)configs.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
job.setName("name");
- handler.updateJob(job, getDerbyDatabaseConnection());
+ handler.updateJob(job, derbyConnection);
assertEquals(1, job.getPersistenceId());
assertCountForTable("SQOOP.SQ_JOB", 4);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 26);
- MJob retrieved = handler.findJob(1, getDerbyDatabaseConnection());
+ MJob retrieved = handler.findJob(1, derbyConnection);
assertEquals("name", retrieved.getName());
- forms = job.getConnectorPart(Direction.FROM).getForms();
- assertEquals(2, forms.size());
- assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- forms = job.getConnectorPart(Direction.TO).getForms();
- assertEquals(2, forms.size());
- assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
-
- forms = retrieved.getFrameworkPart().getForms();
- assertEquals(2, forms.size());
- assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
- assertNotNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
+ configs = job.getJobConfig(Direction.FROM).getConfigs();
+ assertEquals(2, configs.size());
+ assertEquals("Updated", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ configs = job.getJobConfig(Direction.TO).getConfigs();
+ assertEquals(2, configs.size());
+ assertEquals("Updated", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+
+ configs = retrieved.getDriverConfig().getConfigs();
+ assertEquals(2, configs.size());
+ assertEquals("Updated", configs.get(0).getInputs().get(0).getValue());
+ assertNotNull(configs.get(0).getInputs().get(1).getValue());
+ assertEquals(((Map)configs.get(0).getInputs().get(1).getValue()).size(), 0);
}
@Test
@@ -248,16 +250,16 @@ public class TestJobHandling extends DerbyTestCase {
loadJobs();
// disable job 1
- handler.enableJob(1, false, getDerbyDatabaseConnection());
+ handler.enableJob(1, false, derbyConnection);
- MJob retrieved = handler.findJob(1, getDerbyDatabaseConnection());
+ MJob retrieved = handler.findJob(1, derbyConnection);
assertNotNull(retrieved);
assertEquals(false, retrieved.getEnabled());
// enable job 1
- handler.enableJob(1, true, getDerbyDatabaseConnection());
+ handler.enableJob(1, true, derbyConnection);
- retrieved = handler.findJob(1, getDerbyDatabaseConnection());
+ retrieved = handler.findJob(1, derbyConnection);
assertNotNull(retrieved);
assertEquals(true, retrieved.getEnabled());
}
@@ -266,28 +268,28 @@ public class TestJobHandling extends DerbyTestCase {
public void testDeleteJob() throws Exception {
loadJobs();
- handler.deleteJob(1, getDerbyDatabaseConnection());
+ handler.deleteJob(1, derbyConnection);
assertCountForTable("SQOOP.SQ_JOB", 3);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 18);
- handler.deleteJob(2, getDerbyDatabaseConnection());
+ handler.deleteJob(2, derbyConnection);
assertCountForTable("SQOOP.SQ_JOB", 2);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 12);
- handler.deleteJob(3, getDerbyDatabaseConnection());
+ handler.deleteJob(3, derbyConnection);
assertCountForTable("SQOOP.SQ_JOB", 1);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 6);
- handler.deleteJob(4, getDerbyDatabaseConnection());
+ handler.deleteJob(4, derbyConnection);
assertCountForTable("SQOOP.SQ_JOB", 0);
assertCountForTable("SQOOP.SQ_JOB_INPUT", 0);
}
public MJob getJob() {
return new MJob(1, 1, 1, 1,
- handler.findConnector("A", getDerbyDatabaseConnection()).getJobForms(Direction.FROM),
- handler.findConnector("A", getDerbyDatabaseConnection()).getJobForms(Direction.TO),
- handler.findDriverConfig(getDerbyDatabaseConnection()).getJobForms()
+ handler.findConnector("A", derbyConnection).getFromConfig(),
+ handler.findConnector("A", derbyConnection).getToConfig(),
+ handler.findDriver(derbyConnection).getDriverConfig()
);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
index 8dd1ce2..38e632a 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
@@ -17,9 +17,11 @@
*/
package org.apache.sqoop.repository.derby;
+import java.util.List;
+
import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
import org.junit.Before;
@@ -48,7 +50,7 @@ public class TestLinkHandling extends DerbyTestCase {
createSchema();
// We always needs connector and framework structures in place
- loadConnectorAndDriverConfig();
+ loadConnectorLinkConfig();
}
@Test
@@ -64,26 +66,19 @@ public class TestLinkHandling extends DerbyTestCase {
// Load prepared connections into database
loadLinks();
- MLink connA = handler.findLink(1, getDerbyDatabaseConnection());
- assertNotNull(connA);
- assertEquals(1, connA.getPersistenceId());
- assertEquals("CA", connA.getName());
-
- List<MForm> forms;
-
- // Check connector part
- forms = connA.getConnectorPart().getForms();
- assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value3", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- // Check framework part
- forms = connA.getFrameworkPart().getForms();
- assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
+ MLink linkA = handler.findLink(1, getDerbyDatabaseConnection());
+ assertNotNull(linkA);
+ assertEquals(1, linkA.getPersistenceId());
+ assertEquals("CA", linkA.getName());
+
+ List<MConfig> configs;
+
+ // Check connector link config
+ configs = linkA.getConnectorLinkConfig().getConfigs();
+ assertEquals("Value1", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ assertEquals("Value3", configs.get(1).getInputs().get(0).getValue());
+ assertNull(configs.get(1).getInputs().get(1).getValue());
}
@Test
@@ -128,24 +123,18 @@ public class TestLinkHandling extends DerbyTestCase {
handler.createLink(link, getDerbyDatabaseConnection());
assertEquals(1, link.getPersistenceId());
- assertCountForTable("SQOOP.SQ_CONNECTION", 1);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
+ assertCountForTable("SQOOP.SQ_LINK", 1);
+ assertCountForTable("SQOOP.SQ_LINK_INPUT", 2);
MLink retrieved = handler.findLink(1, getDerbyDatabaseConnection());
assertEquals(1, retrieved.getPersistenceId());
- List<MForm> forms;
- forms = link.getConnectorPart().getForms();
- assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value2", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- forms = link.getFrameworkPart().getForms();
- assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
+ List<MConfig> configs;
+ configs = link.getConnectorLinkConfig().getConfigs();
+ assertEquals("Value1", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ assertEquals("Value2", configs.get(1).getInputs().get(0).getValue());
+ assertNull(configs.get(1).getInputs().get(1).getValue());
// Let's create second link
link = getLink();
@@ -154,8 +143,8 @@ public class TestLinkHandling extends DerbyTestCase {
handler.createLink(link, getDerbyDatabaseConnection());
assertEquals(2, link.getPersistenceId());
- assertCountForTable("SQOOP.SQ_CONNECTION", 2);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 8);
+ assertCountForTable("SQOOP.SQ_LINK", 2);
+ assertCountForTable("SQOOP.SQ_LINK_INPUT", 4);
}
@Test
@@ -175,44 +164,30 @@ public class TestLinkHandling extends DerbyTestCase {
MLink link = handler.findLink(1, getDerbyDatabaseConnection());
- List<MForm> forms;
-
- forms = link.getConnectorPart().getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(1).getInputs().get(1)).setValue(null);
+ List<MConfig> configs;
- forms = link.getFrameworkPart().getForms();
- ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
- ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
- ((MMapInput)forms.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+ configs = link.getConnectorLinkConfig().getConfigs();
+ ((MStringInput)configs.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)configs.get(0).getInputs().get(1)).setValue(null);
+ ((MStringInput)configs.get(1).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)configs.get(1).getInputs().get(1)).setValue(null);
link.setName("name");
handler.updateLink(link, getDerbyDatabaseConnection());
assertEquals(1, link.getPersistenceId());
- assertCountForTable("SQOOP.SQ_CONNECTION", 2);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 10);
+ assertCountForTable("SQOOP.SQ_LINK", 2);
+ assertCountForTable("SQOOP.SQ_LINK_INPUT", 6);
MLink retrieved = handler.findLink(1, getDerbyDatabaseConnection());
assertEquals("name", link.getName());
- forms = retrieved.getConnectorPart().getForms();
- assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
- assertNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
- assertNull(forms.get(1).getInputs().get(1).getValue());
-
- forms = retrieved.getFrameworkPart().getForms();
- assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
- assertNotNull(forms.get(0).getInputs().get(1).getValue());
- assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
- assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
- assertNotNull(forms.get(1).getInputs().get(1).getValue());
- assertEquals(((Map)forms.get(1).getInputs().get(1).getValue()).size(), 0);
+ configs = retrieved.getConnectorLinkConfig().getConfigs();
+ assertEquals("Updated", configs.get(0).getInputs().get(0).getValue());
+ assertNull(configs.get(0).getInputs().get(1).getValue());
+ assertEquals("Updated", configs.get(1).getInputs().get(0).getValue());
+ assertNull(configs.get(1).getInputs().get(1).getValue());
}
@Test
@@ -239,18 +214,15 @@ public class TestLinkHandling extends DerbyTestCase {
loadLinks();
handler.deleteLink(1, getDerbyDatabaseConnection());
- assertCountForTable("SQOOP.SQ_CONNECTION", 1);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
+ assertCountForTable("SQOOP.SQ_LINK", 1);
+ assertCountForTable("SQOOP.SQ_LINK_INPUT", 4);
handler.deleteLink(2, getDerbyDatabaseConnection());
- assertCountForTable("SQOOP.SQ_CONNECTION", 0);
- assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 0);
+ assertCountForTable("SQOOP.SQ_LINK", 0);
+ assertCountForTable("SQOOP.SQ_LINK_INPUT", 0);
}
public MLink getLink() {
- return new MLink(1,
- handler.findConnector("A", getDerbyDatabaseConnection()).getConnectionForms(),
- handler.findDriverConfig(getDerbyDatabaseConnection()).getConnectionForms()
- );
+ return new MLink(1, handler.findConnector("A", getDerbyDatabaseConnection()).getLinkConfig());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
index cf0944d..8402d8c 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
@@ -48,7 +48,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
createSchema();
// We always needs connector and framework structures in place
- loadConnectorAndDriverConfig();
+ loadConnectorLinkConfig();
// We also always needs connection metadata in place
loadLinks();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java
index d1b6b9a..7109ae5 100644
--- a/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java
+++ b/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java
@@ -87,7 +87,7 @@ public class ConnectorRequestHandler implements RequestHandler {
connectors = new LinkedList<MConnector>();
bundles = new HashMap<Long, ResourceBundle>();
- connectors.add(ConnectorManager.getInstance().getConnectorMetadata(id));
+ connectors.add(ConnectorManager.getInstance().getConnectorConfig(id));
bundles.put(id, ConnectorManager.getInstance().getResourceBundle(id, locale));
AuditLoggerManager.getInstance()
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java
index c0d363e..aa773a9 100644
--- a/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java
+++ b/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java
@@ -20,7 +20,7 @@ package org.apache.sqoop.handler;
import org.apache.log4j.Logger;
import org.apache.sqoop.audit.AuditLoggerManager;
import org.apache.sqoop.driver.Driver;
-import org.apache.sqoop.json.DriverConfigBean;
+import org.apache.sqoop.json.DriverBean;
import org.apache.sqoop.json.JsonBean;
import org.apache.sqoop.server.RequestContext;
import org.apache.sqoop.server.RequestHandler;
@@ -40,11 +40,10 @@ public class DriverConfigRequestHandler implements RequestHandler {
@Override
public JsonBean handleEvent(RequestContext ctx) {
- AuditLoggerManager.getInstance()
- .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "get", "framework", "");
+ AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(),
+ ctx.getRequest().getRemoteAddr(), "get", "driverConfig", "");
- return new DriverConfigBean(Driver.getInstance().getDriverConfig(),
- Driver.getInstance().getBundle(ctx.getAcceptLanguageHeader()));
+ return new DriverBean(Driver.getInstance().getDriver(), Driver.getInstance()
+ .getBundle(ctx.getAcceptLanguageHeader()));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
index b61d3f5..462579c 100644
--- a/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
+++ b/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
@@ -17,6 +17,10 @@
*/
package org.apache.sqoop.handler;
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+
import org.apache.log4j.Logger;
import org.apache.sqoop.audit.AuditLoggerManager;
import org.apache.sqoop.common.Direction;
@@ -27,25 +31,24 @@ import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.JsonBean;
import org.apache.sqoop.json.ValidationResultBean;
-import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.json.util.ConfigSerialization;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.repository.Repository;
import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.server.RequestContext;
import org.apache.sqoop.server.RequestHandler;
import org.apache.sqoop.server.common.ServerError;
import org.apache.sqoop.utils.ClassUtils;
+import org.apache.sqoop.validation.ConfigValidationResult;
+import org.apache.sqoop.validation.ConfigValidationRunner;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.ValidationResult;
-import org.apache.sqoop.validation.ValidationRunner;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-
/**
* Job request handler is supporting following resources:
*
@@ -109,7 +112,7 @@ public class JobRequestHandler implements RequestHandler {
}
/**
- * Delete job from metadata repository.
+ * Delete job from repository.
*
* @param ctx Context object
* @return Empty bean
@@ -129,18 +132,14 @@ public class JobRequestHandler implements RequestHandler {
}
/**
- * Update or create job metadata in repository.
+ * Update or create job in repository.
*
* @param ctx Context object
* @return Validation bean object
*/
private JsonBean createUpdateJob(RequestContext ctx, boolean update) {
-// Check that given ID equals with sent ID, otherwise report an error UPDATE
-// String sxid = ctx.getLastURLElement();
-// long xid = Long.valueOf(sxid);
String username = ctx.getUserName();
-
JobBean bean = new JobBean();
try {
@@ -157,30 +156,29 @@ public class JobRequestHandler implements RequestHandler {
if(jobs.size() != 1) {
throw new SqoopException(ServerError.SERVER_0003,
- "Expected one job metadata but got " + jobs.size());
+ "Expected one job but got " + jobs.size());
}
// Job object
MJob job = jobs.get(0);
// Verify that user is not trying to spoof us
- MJobForms fromConnectorForms = ConnectorManager.getInstance()
- .getConnectorMetadata(job.getConnectorId(Direction.FROM))
- .getJobForms(Direction.FROM);
- MJobForms toConnectorForms = ConnectorManager.getInstance()
- .getConnectorMetadata(job.getConnectorId(Direction.TO))
- .getJobForms(Direction.TO);
- MJobForms frameworkForms = Driver.getInstance().getDriverConfig()
- .getJobForms();
-
- if(!fromConnectorForms.equals(job.getConnectorPart(Direction.FROM))
- || !frameworkForms.equals(job.getFrameworkPart())
- || !toConnectorForms.equals(job.getConnectorPart(Direction.TO))) {
+ MFromConfig fromConfig = ConnectorManager.getInstance()
+ .getConnectorConfig(job.getConnectorId(Direction.FROM))
+ .getFromConfig();
+ MToConfig toConfig = ConnectorManager.getInstance()
+ .getConnectorConfig(job.getConnectorId(Direction.TO))
+ .getToConfig();
+ MDriverConfig driverConfig = Driver.getInstance().getDriver().getDriverConfig();
+
+ if(!fromConfig.equals(job.getJobConfig(Direction.FROM))
+ || !driverConfig.equals(job.getDriverConfig())
+ || !toConfig.equals(job.getJobConfig(Direction.TO))) {
throw new SqoopException(ServerError.SERVER_0003,
- "Detected incorrect form structure");
+ "Detected incorrect config structure");
}
- // Responsible connector for this session
+ // Corresponding connectors for this
SqoopConnector fromConnector = ConnectorManager.getInstance().getConnector(job.getConnectorId(Direction.FROM));
SqoopConnector toConnector = ConnectorManager.getInstance().getConnector(job.getConnectorId(Direction.TO));
@@ -194,25 +192,27 @@ public class JobRequestHandler implements RequestHandler {
+ " does not support TO direction.");
}
- // We need translate forms to configuration objects
- Object fromConnectorConfig = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
- Object frameworkConfig = ClassUtils.instantiate(Driver.getInstance().getJobConfigurationClass());
- Object toConnectorConfig = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
+ // We need translate configs
+ Object fromConfigObject = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
+ Object toConfigObject = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
+
+ Object driverConfigObject = ClassUtils.instantiate(Driver.getInstance().getDriverConfigurationGroupClass());
+
+ ConfigUtils.fromConfigs(job.getJobConfig(Direction.FROM).getConfigs(), fromConfigObject);
+ ConfigUtils.fromConfigs(job.getJobConfig(Direction.TO).getConfigs(), toConfigObject);
+ ConfigUtils.fromConfigs(job.getDriverConfig().getConfigs(), driverConfigObject);
- FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromConnectorConfig);
- FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkConfig);
- FormUtils.fromForms(job.getConnectorPart(Direction.TO).getForms(), toConnectorConfig);
+ // Validate all configs
+ ConfigValidationRunner validationRunner = new ConfigValidationRunner();
+ ConfigValidationResult fromConfigvalidator = validationRunner.validate(fromConfigObject);
+ ConfigValidationResult toConfigValidator = validationRunner.validate(toConfigObject);
+ ConfigValidationResult driverConfigValidator = validationRunner.validate(driverConfigObject);
- // Validate all parts
- ValidationRunner validationRunner = new ValidationRunner();
- ValidationResult fromConnectorValidation = validationRunner.validate(fromConnectorConfig);
- ValidationResult frameworkValidation = validationRunner.validate(frameworkConfig);
- ValidationResult toConnectorValidation = validationRunner.validate(toConnectorConfig);
- Status finalStatus = Status.getWorstStatus(fromConnectorValidation.getStatus(), frameworkValidation.getStatus(), toConnectorValidation.getStatus());
+ Status finalStatus = Status.getWorstStatus(fromConfigvalidator.getStatus(), toConfigValidator.getStatus(), driverConfigValidator.getStatus());
// Return back validations in all cases
- ValidationResultBean outputBean = new ValidationResultBean(fromConnectorValidation, frameworkValidation, toConnectorValidation);
+ ValidationResultBean validationResultBean = new ValidationResultBean(fromConfigvalidator, toConfigValidator);
// If we're good enough let's perform the action
if(finalStatus.canProceed()) {
@@ -227,7 +227,7 @@ public class JobRequestHandler implements RequestHandler {
job.setCreationUser(username);
job.setLastUpdateUser(username);
RepositoryManager.getInstance().getRepository().createJob(job);
- outputBean.setId(job.getPersistenceId());
+ validationResultBean.setId(job.getPersistenceId());
AuditLoggerManager.getInstance()
.logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
@@ -236,7 +236,7 @@ public class JobRequestHandler implements RequestHandler {
}
- return outputBean;
+ return validationResultBean;
}
private JsonBean getJobs(RequestContext ctx) {
@@ -250,7 +250,7 @@ public class JobRequestHandler implements RequestHandler {
Locale locale = ctx.getAcceptLanguageHeader();
Repository repository = RepositoryManager.getInstance().getRepository();
- if (sjid.equals("all")) {
+ if (sjid.equals(ConfigSerialization.ALL)) {
List<MJob> jobs = repository.findJobs();
bean = new JobBean(jobs);
@@ -269,11 +269,8 @@ public class JobRequestHandler implements RequestHandler {
MJob job = repository.findJob(jid);
// @TODO(Abe): From/To
-
long connectorId = job.getConnectorId(Direction.FROM);
-
bean = new JobBean(job);
-
bean.addConnectorConfigBundle(connectorId,
ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
index 23fc9f1..80e65b8 100644
--- a/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
+++ b/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
@@ -17,18 +17,21 @@
*/
package org.apache.sqoop.handler;
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+
import org.apache.log4j.Logger;
import org.apache.sqoop.audit.AuditLoggerManager;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.ConnectorManager;
import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.driver.Driver;
-import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.JsonBean;
+import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.ValidationResultBean;
-import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.repository.Repository;
import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.server.RequestContext;
@@ -36,41 +39,37 @@ import org.apache.sqoop.server.RequestHandler;
import org.apache.sqoop.server.common.ServerError;
import org.apache.sqoop.utils.ClassUtils;
import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.ValidationResult;
-import org.apache.sqoop.validation.ValidationRunner;
+import org.apache.sqoop.validation.ConfigValidationResult;
+import org.apache.sqoop.validation.ConfigValidationRunner;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-
/**
* Connection request handler is supporting following resources:
*
- * GET /v1/connection/:xid
- * Return details about one particular connection with id :xid or about all of
+ * GET /v1/link/:xid
+ * Return details about one particular link with id :xid or about all of
* them if :xid equals to "all".
*
- * POST /v1/connection
- * Create new connection
+ * POST /v1/link
+ * Create new link
*
- * PUT /v1/connection/:xid
- * Update connection with id :xid.
+ * PUT /v1/link/:xid
+ * Update link with id :xid.
*
- * PUT /v1/connection/:xid/enable
- * Enable connection with id :xid
+ * PUT /v1/link/:xid/enable
+ * Enable link with id :xid
*
- * PUT /v1/connection/:xid/disable
- * Disable connection with id :xid
+ * PUT /v1/link/:xid/disable
+ * Disable link with id :xid
*
- * DELETE /v1/connection/:xid
- * Remove connection with id :xid
+ * DELETE /v1/link/:xid
+ * Remove link with id :xid
*
* Planned resources:
*
- * GET /v1/connection
- * Get brief list of all connections present in the system.
+ * GET /v1/link
+ * Get brief list of all links present in the system.
*
*/
public class LinkRequestHandler implements RequestHandler {
@@ -89,37 +88,37 @@ public class LinkRequestHandler implements RequestHandler {
public JsonBean handleEvent(RequestContext ctx) {
switch (ctx.getMethod()) {
case GET:
- return getConnections(ctx);
+ return getLink(ctx);
case POST:
- return createUpdateConnection(ctx, false);
+ return createUpdateLink(ctx, false);
case PUT:
if (ctx.getLastURLElement().equals(ENABLE)) {
- return enableConnection(ctx, true);
+ return enableLink(ctx, true);
} else if (ctx.getLastURLElement().equals(DISABLE)) {
- return enableConnection(ctx, false);
+ return enableLink(ctx, false);
} else {
- return createUpdateConnection(ctx, true);
+ return createUpdateLink(ctx, true);
}
case DELETE:
- return deleteConnection(ctx);
+ return deleteLink(ctx);
}
return null;
}
/**
- * Delete connection from metadata repository.
+ * Delete link from thes repository.
*
* @param ctx Context object
* @return Empty bean
*/
- private JsonBean deleteConnection(RequestContext ctx) {
+ private JsonBean deleteLink(RequestContext ctx) {
String sxid = ctx.getLastURLElement();
long xid = Long.valueOf(sxid);
AuditLoggerManager.getInstance()
.logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "delete", "connection", sxid);
+ "delete", "link", sxid);
Repository repository = RepositoryManager.getInstance().getRepository();
repository.deleteLink(xid);
@@ -128,20 +127,15 @@ public class LinkRequestHandler implements RequestHandler {
}
/**
- * Update or create connection metadata in repository.
+ * Update or create link in repository.
*
* @param ctx Context object
* @return Validation bean object
*/
- private JsonBean createUpdateConnection(RequestContext ctx, boolean update) {
-// Check that given ID equals with sent ID, otherwise report an error UPDATE
-// String sxid = ctx.getLastURLElement();
-// long xid = Long.valueOf(sxid);
+ private JsonBean createUpdateLink(RequestContext ctx, boolean update) {
String username = ctx.getUserName();
-
LinkBean bean = new LinkBean();
-
try {
JSONObject json =
(JSONObject) JSONValue.parse(ctx.getRequest().getReader());
@@ -152,93 +146,86 @@ public class LinkRequestHandler implements RequestHandler {
"Can't read request content", e);
}
- // Get connection object
- List<MLink> connections = bean.getLinks();
+ // Get link object
+ List<MLink> links = bean.getLinks();
- if(connections.size() != 1) {
+ if(links.size() != 1) {
throw new SqoopException(ServerError.SERVER_0003,
- "Expected one connection metadata but got " + connections.size());
+ "Expected one link but got " + links.size());
}
- MLink connection = connections.get(0);
+ MLink link = links.get(0);
// Verify that user is not trying to spoof us
- MConnectionForms connectorForms =
- ConnectorManager.getInstance().getConnectorMetadata(connection.getConnectorId())
- .getConnectionForms();
- MConnectionForms frameworkForms = Driver.getInstance().getDriverConfig()
- .getConnectionForms();
-
- if(!connectorForms.equals(connection.getConnectorPart())
- || !frameworkForms.equals(connection.getFrameworkPart())) {
+ MLinkConfig linkConfig =
+ ConnectorManager.getInstance().getConnectorConfig(link.getConnectorId())
+ .getLinkConfig();
+ if(!linkConfig.equals(link.getConnectorLinkConfig())) {
throw new SqoopException(ServerError.SERVER_0003,
- "Detected incorrect form structure");
+ "Detected incorrect config structure");
}
// Responsible connector for this session
- SqoopConnector connector = ConnectorManager.getInstance().getConnector(connection.getConnectorId());
+ SqoopConnector connector = ConnectorManager.getInstance().getConnector(link.getConnectorId());
- // We need translate forms to configuration objects
- Object connectorConfig = ClassUtils.instantiate(connector.getLinkConfigurationClass());
- Object frameworkConfig = ClassUtils.instantiate(Driver.getInstance().getLinkConfigurationClass());
+ // We need translate configs
+ Object connectorLinkConfig = ClassUtils.instantiate(connector.getLinkConfigurationClass());
- FormUtils.fromForms(connection.getConnectorPart().getForms(), connectorConfig);
- FormUtils.fromForms(connection.getFrameworkPart().getForms(), frameworkConfig);
+ ConfigUtils.fromConfigs(link.getConnectorLinkConfig().getConfigs(), connectorLinkConfig);
// Validate both parts
- ValidationRunner validationRunner = new ValidationRunner();
- ValidationResult connectorValidation = validationRunner.validate(connectorConfig);
- ValidationResult frameworkValidation = validationRunner.validate(frameworkConfig);
+ ConfigValidationRunner validationRunner = new ConfigValidationRunner();
+ ConfigValidationResult connectorLinkValidation = validationRunner.validate(connectorLinkConfig);
- Status finalStatus = Status.getWorstStatus(connectorValidation.getStatus(), frameworkValidation.getStatus());
+ Status finalStatus = Status.getWorstStatus(connectorLinkValidation.getStatus());
// Return back validations in all cases
- ValidationResultBean outputBean = new ValidationResultBean(connectorValidation, frameworkValidation);
+ ValidationResultBean outputBean = new ValidationResultBean(connectorLinkValidation);
// If we're good enough let's perform the action
if(finalStatus.canProceed()) {
if(update) {
AuditLoggerManager.getInstance()
.logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "update", "connection", String.valueOf(connection.getPersistenceId()));
+ "update", "link", String.valueOf(link.getPersistenceId()));
- connection.setLastUpdateUser(username);
- RepositoryManager.getInstance().getRepository().updateLink(connection);
+ link.setLastUpdateUser(username);
+ RepositoryManager.getInstance().getRepository().updateLink(link);
} else {
- connection.setCreationUser(username);
- connection.setLastUpdateUser(username);
- RepositoryManager.getInstance().getRepository().createLink(connection);
- outputBean.setId(connection.getPersistenceId());
+ link.setCreationUser(username);
+ link.setLastUpdateUser(username);
+ RepositoryManager.getInstance().getRepository().createLink(link);
+ outputBean.setId(link.getPersistenceId());
AuditLoggerManager.getInstance()
.logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "create", "connection", String.valueOf(connection.getPersistenceId()));
+ "create", "link", String.valueOf(link.getPersistenceId()));
}
}
return outputBean;
}
- private JsonBean getConnections(RequestContext ctx) {
+ private JsonBean getLink(RequestContext ctx) {
String sxid = ctx.getLastURLElement();
LinkBean bean;
AuditLoggerManager.getInstance()
.logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "get", "connection", sxid);
+ "get", "link", sxid);
Locale locale = ctx.getAcceptLanguageHeader();
Repository repository = RepositoryManager.getInstance().getRepository();
if (sxid.equals("all")) {
- List<MLink> connections = repository.findLinks();
- bean = new LinkBean(connections);
+ List<MLink> links = repository.findLinks();
+ bean = new LinkBean(links);
// Add associated resources into the bean
- for( MLink connection : connections) {
- long connectorId = connection.getConnectorId();
- if(!bean.hasConnectorBundle(connectorId)) {
+ for( MLink link : links) {
+ long connectorId = link.getConnectorId();
+ if(!bean.hasConnectorConfigBundle(connectorId)) {
bean.addConnectorConfigBundle(connectorId,
ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
}
@@ -246,21 +233,18 @@ public class LinkRequestHandler implements RequestHandler {
} else {
long xid = Long.valueOf(sxid);
- MLink connection = repository.findLink(xid);
- long connectorId = connection.getConnectorId();
+ MLink link = repository.findLink(xid);
+ long connectorId = link.getConnectorId();
- bean = new LinkBean(connection);
+ bean = new LinkBean(link);
bean.addConnectorConfigBundle(connectorId,
ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
}
-
- // Sent framework resource bundle in all cases
- bean.setDriverConfigBundle(Driver.getInstance().getBundle(locale));
return bean;
}
- private JsonBean enableConnection(RequestContext ctx, boolean enabled) {
+ private JsonBean enableLink(RequestContext ctx, boolean enabled) {
String[] elements = ctx.getUrlElements();
String sLinkId = elements[elements.length - 2];
long linkId = Long.valueOf(sLinkId);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
index 97d8e6f..8188831 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
@@ -24,8 +24,8 @@ import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.FormOptions;
-import org.apache.sqoop.shell.utils.JobDynamicFormOptions;
+import org.apache.sqoop.shell.utils.ConfigOptions;
+import org.apache.sqoop.shell.utils.JobDynamicConfigOptions;
import org.apache.sqoop.validation.Status;
import java.io.IOException;
@@ -33,7 +33,7 @@ import java.util.List;
import java.util.ResourceBundle;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
+import static org.apache.sqoop.shell.utils.ConfigFiller.*;
/**
*
@@ -63,11 +63,11 @@ public class CloneJobFunction extends SqoopFunction {
MJob job = client.getJob(jobId);
job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);
- ResourceBundle fromConnectorBundle = client.getConnectorConfigResourceBundle(
+ ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle(
job.getConnectorId(Direction.FROM));
- ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
- ResourceBundle toConnectorBundle = client.getConnectorConfigResourceBundle(
+ ResourceBundle toConnectorBundle = client.getConnectorConfigBundle(
job.getConnectorId(Direction.TO));
+ ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
Status status = Status.FINE;
@@ -84,7 +84,7 @@ public class CloneJobFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillJob(reader, job, fromConnectorBundle, driverConfigBundle, toConnectorBundle)) {
+ if(!fillJobWithBundle(reader, job, fromConnectorBundle, toConnectorBundle, driverConfigBundle)) {
return null;
}
@@ -92,9 +92,9 @@ public class CloneJobFunction extends SqoopFunction {
status = client.saveJob(job);
} while(!status.canProceed());
} else {
- JobDynamicFormOptions options = new JobDynamicFormOptions();
+ JobDynamicConfigOptions options = new JobDynamicConfigOptions();
options.prepareOptions(job);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
+ CommandLine line = ConfigOptions.parseOptions(options, 0, args, false);
if (fillJob(line, job)) {
status = client.saveJob(job);
if (!status.canProceed()) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
index d9babe0..c1a4f55 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
@@ -23,8 +23,8 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.LinkDynamicFormOptions;
-import org.apache.sqoop.shell.utils.FormOptions;
+import org.apache.sqoop.shell.utils.LinkDynamicConfigOptions;
+import org.apache.sqoop.shell.utils.ConfigOptions;
import org.apache.sqoop.validation.Status;
import java.io.IOException;
@@ -32,7 +32,7 @@ import java.util.List;
import java.util.ResourceBundle;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
+import static org.apache.sqoop.shell.utils.ConfigFiller.*;
/**
*
@@ -67,8 +67,7 @@ public class CloneLinkFunction extends SqoopFunction {
Status status = Status.FINE;
- ResourceBundle connectorConfigBundle = client.getConnectorConfigResourceBundle(connection.getConnectorId());
- ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+ ResourceBundle linkConfigBundle = client.getConnectorConfigBundle(connection.getConnectorId());
if (isInteractive) {
printlnResource(Constants.RES_PROMPT_UPDATE_LINK_CONFIG);
@@ -80,17 +79,17 @@ public class CloneLinkFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillLink(reader, connection, connectorConfigBundle, driverConfigBundle)) {
+ if(!fillLinkWithBundle(reader, connection, linkConfigBundle)) {
return null;
}
status = client.saveLink(connection);
} while(!status.canProceed());
} else {
- LinkDynamicFormOptions options = new LinkDynamicFormOptions();
+ LinkDynamicConfigOptions options = new LinkDynamicConfigOptions();
options.prepareOptions(connection);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
- if (fillConnection(line, connection)) {
+ CommandLine line = ConfigOptions.parseOptions(options, 0, args, false);
+ if (fillLink(line, connection)) {
status = client.saveLink(connection);
if (!status.canProceed()) {
printLinkValidationMessages(connection);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
index ccfed31..0d3d2b5 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
@@ -23,9 +23,9 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.FormDisplayer;
-import org.apache.sqoop.shell.utils.FormOptions;
-import org.apache.sqoop.shell.utils.JobDynamicFormOptions;
+import org.apache.sqoop.shell.utils.ConfigDisplayer;
+import org.apache.sqoop.shell.utils.ConfigOptions;
+import org.apache.sqoop.shell.utils.JobDynamicConfigOptions;
import org.apache.sqoop.validation.Status;
import java.io.IOException;
@@ -33,7 +33,7 @@ import java.util.List;
import java.util.ResourceBundle;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
+import static org.apache.sqoop.shell.utils.ConfigFiller.*;
/**
* Handles creation of new job objects.
@@ -67,16 +67,16 @@ public class CreateJobFunction extends SqoopFunction {
isInteractive);
}
- private Status createJob(Long fromConnectionId, Long toConnectionId, List<String> args, boolean isInteractive) throws IOException {
- printlnResource(Constants.RES_CREATE_CREATING_JOB, fromConnectionId, toConnectionId);
+ private Status createJob(Long fromLinkId, Long toLinkId, List<String> args, boolean isInteractive) throws IOException {
+ printlnResource(Constants.RES_CREATE_CREATING_JOB, fromLinkId, toLinkId);
ConsoleReader reader = new ConsoleReader();
- MJob job = client.createJob(fromConnectionId, toConnectionId);
+ MJob job = client.createJob(fromLinkId, toLinkId);
// @TODO(Abe): From/To.
- ResourceBundle fromConnectorBundle = client.getConnectorConfigResourceBundle(
+ ResourceBundle fromConfigBundle = client.getConnectorConfigBundle(
job.getConnectorId(Direction.FROM));
- ResourceBundle toConnectorBundle = client.getConnectorConfigResourceBundle(
+ ResourceBundle toConfigBundle = client.getConnectorConfigBundle(
job.getConnectorId(Direction.TO));
ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
@@ -92,7 +92,7 @@ public class CreateJobFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillJob(reader, job, fromConnectorBundle, driverConfigBundle, toConnectorBundle)) {
+ if(!fillJobWithBundle(reader, job, fromConfigBundle, toConfigBundle, driverConfigBundle)) {
return null;
}
@@ -100,9 +100,9 @@ public class CreateJobFunction extends SqoopFunction {
status = client.saveJob(job);
} while(!status.canProceed());
} else {
- JobDynamicFormOptions options = new JobDynamicFormOptions();
+ JobDynamicConfigOptions options = new JobDynamicConfigOptions();
options.prepareOptions(job);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
+ CommandLine line = ConfigOptions.parseOptions(options, 0, args, false);
if (fillJob(line, job)) {
status = client.saveJob(job);
if (!status.canProceed()) {
@@ -115,7 +115,7 @@ public class CreateJobFunction extends SqoopFunction {
}
}
- FormDisplayer.displayFormWarning(job);
+ ConfigDisplayer.displayConfigWarning(job);
printlnResource(Constants.RES_CREATE_JOB_SUCCESSFUL, status.name(), job.getPersistenceId());
return status;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
index 33d60c8..ce9988f 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
@@ -22,9 +22,9 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.LinkDynamicFormOptions;
-import org.apache.sqoop.shell.utils.FormDisplayer;
-import org.apache.sqoop.shell.utils.FormOptions;
+import org.apache.sqoop.shell.utils.LinkDynamicConfigOptions;
+import org.apache.sqoop.shell.utils.ConfigDisplayer;
+import org.apache.sqoop.shell.utils.ConfigOptions;
import org.apache.sqoop.validation.Status;
import java.io.IOException;
@@ -32,7 +32,7 @@ import java.util.List;
import java.util.ResourceBundle;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
+import static org.apache.sqoop.shell.utils.ConfigFiller.*;
/**
*
@@ -62,11 +62,9 @@ public class CreateLinkFunction extends SqoopFunction {
MLink link = client.createLink(connectorId);
- ResourceBundle connectorConfigBundle = client.getConnectorConfigResourceBundle(connectorId);
- ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+ ResourceBundle connectorConfigBundle = client.getConnectorConfigBundle(connectorId);
Status status = Status.FINE;
-
if (isInteractive) {
printlnResource(Constants.RES_PROMPT_FILL_LINK_CONFIG);
@@ -77,7 +75,7 @@ public class CreateLinkFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillLink(reader, link, connectorConfigBundle, driverConfigBundle)) {
+ if(!fillLinkWithBundle(reader, link, connectorConfigBundle)) {
return null;
}
@@ -85,10 +83,10 @@ public class CreateLinkFunction extends SqoopFunction {
status = client.saveLink(link);
} while(!status.canProceed());
} else {
- LinkDynamicFormOptions options = new LinkDynamicFormOptions();
+ LinkDynamicConfigOptions options = new LinkDynamicConfigOptions();
options.prepareOptions(link);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
- if (fillConnection(line, link)) {
+ CommandLine line = ConfigOptions.parseOptions(options, 0, args, false);
+ if (fillLink(line, link)) {
status = client.saveLink(link);
if (!status.canProceed()) {
printLinkValidationMessages(link);
@@ -100,7 +98,7 @@ public class CreateLinkFunction extends SqoopFunction {
}
}
- FormDisplayer.displayFormWarning(link);
+ ConfigDisplayer.displayConfigWarning(link);
printlnResource(Constants.RES_CREATE_LINK_SUCCESSFUL, status.name(), link.getPersistenceId());
return status;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java b/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java
index 28bc752..329b4d6 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowCommand.java
@@ -31,7 +31,7 @@ public class ShowCommand extends SqoopCommand {
.put(Constants.FN_SERVER, ShowServerFunction.class)
.put(Constants.FN_VERSION, ShowVersionFunction.class)
.put(Constants.FN_CONNECTOR, ShowConnectorFunction.class)
- .put(Constants.FN_DRIVER_CONFIG, ShowDriverConfigFunction.class)
+ .put(Constants.FN_DRIVER_CONFIG, ShowDriverFunction.class)
.put(Constants.FN_LINK, ShowLinkFunction.class)
.put(Constants.FN_JOB, ShowJobFunction.class)
.put(Constants.FN_SUBMISSION, ShowSubmissionFunction.class)
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
index 2ba75b4..09fb195 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
@@ -31,7 +31,7 @@ import org.apache.sqoop.shell.utils.TableDisplayer;
import org.apache.sqoop.validation.Status;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormDisplayer.*;
+import static org.apache.sqoop.shell.utils.ConfigDisplayer.*;
@SuppressWarnings("serial")
public class ShowConnectorFunction extends SqoopFunction {
@@ -115,7 +115,7 @@ public class ShowConnectorFunction extends SqoopFunction {
connector.getVersion(),
getSupportedDirections(connector)
);
- displayFormMetadataDetails(connector, client.getConnectorConfigResourceBundle(connector.getPersistenceId()));
+ displayConnectorConfigDetails(connector, client.getConnectorConfigBundle(connector.getPersistenceId()));
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/ShowDriverConfigFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowDriverConfigFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowDriverConfigFunction.java
deleted file mode 100644
index e6e1004..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowDriverConfigFunction.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.validation.Status;
-
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormDisplayer.*;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class ShowDriverConfigFunction extends SqoopFunction {
- public ShowDriverConfigFunction() {
- }
-
- @Override
- public boolean validateArgs(CommandLine line) {
- if (line.getArgs().length != 0) {
- printlnResource(Constants.RES_SHOW_DRIVER_CONFIG_USAGE);
- return false;
- }
- return true;
- }
-
- @Override
- public Object executeFunction(CommandLine line, boolean isInteractive) {
- showFramework();
- return Status.FINE;
- }
-
- private void showFramework() {
- MDriverConfig framework = client.getDriverConfig();
- ResourceBundle bundle = client.getDriverConfigBundle();
-
- printlnResource(Constants.RES_SHOW_PROMPT_DRIVER_CONFIG_OPTS, framework.getPersistenceId());
- displayFormMetadataDetails(framework, bundle);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/ShowDriverFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowDriverFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowDriverFunction.java
new file mode 100644
index 0000000..080792b
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowDriverFunction.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import static org.apache.sqoop.shell.ShellEnvironment.client;
+import static org.apache.sqoop.shell.ShellEnvironment.printlnResource;
+import static org.apache.sqoop.shell.utils.ConfigDisplayer.displayDriverConfigDetails;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.validation.Status;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class ShowDriverFunction extends SqoopFunction {
+ public ShowDriverFunction() {
+ }
+
+ @Override
+ public boolean validateArgs(CommandLine line) {
+ if (line.getArgs().length != 0) {
+ printlnResource(Constants.RES_SHOW_DRIVER_CONFIG_USAGE);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public Object executeFunction(CommandLine line, boolean isInteractive) {
+ showDriver();
+ return Status.FINE;
+ }
+
+ private void showDriver() {
+ printlnResource(Constants.RES_SHOW_PROMPT_DRIVER_CONFIG_OPTS, client.getDriver().getPersistenceId());
+ displayDriverConfigDetails(client.getDriverConfig(), client.getDriverConfigBundle());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
index e3f1f47..0640283 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
@@ -30,7 +30,7 @@ import java.util.LinkedList;
import java.util.List;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormDisplayer.*;
+import static org.apache.sqoop.shell.utils.ConfigDisplayer.*;
/**
*
@@ -124,12 +124,11 @@ public class ShowJobFunction extends SqoopFunction {
job.getLinkId(Direction.FROM),
job.getConnectorId(Direction.FROM));
- // Display connector part
- displayForms(job.getConnectorPart(Direction.FROM).getForms(),
- client.getConnectorConfigResourceBundle(job.getConnectorId(Direction.FROM)));
- displayForms(job.getFrameworkPart().getForms(),
+ displayConfig(job.getJobConfig(Direction.FROM).getConfigs(),
+ client.getConnectorConfigBundle(job.getConnectorId(Direction.FROM)));
+ displayConfig(job.getDriverConfig().getConfigs(),
client.getDriverConfigBundle());
- displayForms(job.getConnectorPart(Direction.TO).getForms(),
- client.getConnectorConfigResourceBundle(job.getConnectorId(Direction.TO)));
+ displayConfig(job.getJobConfig(Direction.TO).getConfigs(),
+ client.getConnectorConfigBundle(job.getConnectorId(Direction.TO)));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
index f500b9e..f4eae33 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
@@ -29,7 +29,7 @@ import java.util.LinkedList;
import java.util.List;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormDisplayer.*;
+import static org.apache.sqoop.shell.utils.ConfigDisplayer.*;
/**
*
@@ -103,26 +103,24 @@ public class ShowLinkFunction extends SqoopFunction {
displayLink(link);
}
- private void displayLink(MLink connection) {
+ private void displayLink(MLink link) {
DateFormat formatter = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
printlnResource(Constants.RES_SHOW_PROMPT_LINK_INFO,
- connection.getPersistenceId(),
- connection.getName(),
- connection.getEnabled(),
- connection.getCreationUser(),
- formatter.format(connection.getCreationDate()),
- connection.getLastUpdateUser(),
- formatter.format(connection.getLastUpdateDate())
+ link.getPersistenceId(),
+ link.getName(),
+ link.getEnabled(),
+ link.getCreationUser(),
+ formatter.format(link.getCreationDate()),
+ link.getLastUpdateUser(),
+ formatter.format(link.getLastUpdateDate())
);
- long connectorId = connection.getConnectorId();
+ long connectorId = link.getConnectorId();
printlnResource(Constants.RES_SHOW_PROMPT_LINK_CID_INFO, connectorId);
- // Display connector part
- displayForms(connection.getConnectorPart().getForms(),
- client.getConnectorConfigResourceBundle(connectorId));
- displayForms(connection.getFrameworkPart().getForms(),
- client.getDriverConfigBundle());
+ // Display link config
+ displayConfig(link.getConnectorLinkConfig().getConfigs(),
+ client.getConnectorConfigBundle(connectorId));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/SqoopFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/SqoopFunction.java b/shell/src/main/java/org/apache/sqoop/shell/SqoopFunction.java
index 5d69c2a..0845d8e 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/SqoopFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/SqoopFunction.java
@@ -25,7 +25,7 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.shell.core.ShellError;
-import org.apache.sqoop.shell.utils.FormOptions;
+import org.apache.sqoop.shell.utils.ConfigOptions;
import static org.apache.sqoop.shell.ShellEnvironment.*;
@@ -44,7 +44,7 @@ abstract public class SqoopFunction extends Options {
}
public Object execute(List<String> args) {
- CommandLine line = FormOptions.parseOptions(this, 1, args, true);
+ CommandLine line = ConfigOptions.parseOptions(this, 1, args, true);
try {
if (validateArgs(line)) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
index 96d1fd8..dd075d7 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
@@ -23,9 +23,9 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.FormDisplayer;
-import org.apache.sqoop.shell.utils.FormOptions;
-import org.apache.sqoop.shell.utils.JobDynamicFormOptions;
+import org.apache.sqoop.shell.utils.ConfigDisplayer;
+import org.apache.sqoop.shell.utils.ConfigOptions;
+import org.apache.sqoop.shell.utils.JobDynamicConfigOptions;
import org.apache.sqoop.validation.Status;
import java.io.IOException;
@@ -33,7 +33,7 @@ import java.util.List;
import java.util.ResourceBundle;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
+import static org.apache.sqoop.shell.utils.ConfigFiller.*;
/**
*
@@ -57,18 +57,19 @@ public class UpdateJobFunction extends SqoopFunction {
}
private Status updateJob(Long jobId, List<String> args, boolean isInteractive) throws IOException {
- printlnResource(Constants.RES_UPDATE_UPDATING_JOB, jobId);
+ printlnResource(Constants.RES_SQOOP_UPDATING_JOB, jobId);
ConsoleReader reader = new ConsoleReader();
MJob job = client.getJob(jobId);
- ResourceBundle fromConnectorBundle = client.getConnectorConfigResourceBundle(
+ ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle(
job.getConnectorId(Direction.FROM));
- ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
- ResourceBundle toConnectorBundle = client.getConnectorConfigResourceBundle(
+ ResourceBundle toConnectorBundle = client.getConnectorConfigBundle(
job.getConnectorId(Direction.TO));
+ ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+
Status status = Status.FINE;
if (isInteractive) {
@@ -81,7 +82,7 @@ public class UpdateJobFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillJob(reader, job, fromConnectorBundle, driverConfigBundle, toConnectorBundle)) {
+ if(!fillJobWithBundle(reader, job, fromConnectorBundle, toConnectorBundle, driverConfigBundle)) {
return status;
}
@@ -89,9 +90,9 @@ public class UpdateJobFunction extends SqoopFunction {
status = client.updateJob(job);
} while(!status.canProceed());
} else {
- JobDynamicFormOptions options = new JobDynamicFormOptions();
+ JobDynamicConfigOptions options = new JobDynamicConfigOptions();
options.prepareOptions(job);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
+ CommandLine line = ConfigOptions.parseOptions(options, 0, args, false);
if (fillJob(line, job)) {
status = client.updateJob(job);
if (!status.canProceed()) {
@@ -104,7 +105,7 @@ public class UpdateJobFunction extends SqoopFunction {
}
}
- FormDisplayer.displayFormWarning(job);
+ ConfigDisplayer.displayConfigWarning(job);
printlnResource(Constants.RES_UPDATE_JOB_SUCCESSFUL, status.name());
return status;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
index 30bb63e..d5ead38 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
@@ -22,9 +22,9 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.LinkDynamicFormOptions;
-import org.apache.sqoop.shell.utils.FormDisplayer;
-import org.apache.sqoop.shell.utils.FormOptions;
+import org.apache.sqoop.shell.utils.LinkDynamicConfigOptions;
+import org.apache.sqoop.shell.utils.ConfigDisplayer;
+import org.apache.sqoop.shell.utils.ConfigOptions;
import org.apache.sqoop.validation.Status;
import java.io.IOException;
@@ -32,7 +32,7 @@ import java.util.List;
import java.util.ResourceBundle;
import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
+import static org.apache.sqoop.shell.utils.ConfigFiller.*;
/**
*
@@ -56,14 +56,13 @@ public class UpdateLinkFunction extends SqoopFunction {
}
private Status updateLink(Long linkId, List<String> args, boolean isInteractive) throws IOException {
- printlnResource(Constants.RES_UPDATE_UPDATING_LINK, linkId);
+ printlnResource(Constants.RES_SQOOP_UPDATING_LINK, linkId);
ConsoleReader reader = new ConsoleReader();
MLink link = client.getLink(linkId);
- ResourceBundle connectorConfigBundle = client.getConnectorConfigResourceBundle(link.getConnectorId());
- ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+ ResourceBundle connectorLinkConfigBundle = client.getConnectorConfigBundle(link.getConnectorId());
Status status = Status.FINE;
@@ -77,7 +76,7 @@ public class UpdateLinkFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillLink(reader, link, connectorConfigBundle, driverConfigBundle)) {
+ if(!fillLinkWithBundle(reader, link, connectorLinkConfigBundle)) {
return null;
}
@@ -85,10 +84,10 @@ public class UpdateLinkFunction extends SqoopFunction {
status = client.updateLink(link);
} while(!status.canProceed());
} else {
- LinkDynamicFormOptions options = new LinkDynamicFormOptions();
+ LinkDynamicConfigOptions options = new LinkDynamicConfigOptions();
options.prepareOptions(link);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
- if (fillConnection(line, link)) {
+ CommandLine line = ConfigOptions.parseOptions(options, 0, args, false);
+ if (fillLink(line, link)) {
status = client.updateLink(link);
if (!status.canProceed()) {
printLinkValidationMessages(link);
@@ -99,7 +98,7 @@ public class UpdateLinkFunction extends SqoopFunction {
return null;
}
}
- FormDisplayer.displayFormWarning(link);
+ ConfigDisplayer.displayConfigWarning(link);
printlnResource(Constants.RES_UPDATE_LINK_SUCCESSFUL, status.name());
return status;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
index 4d3838c..a1bc5d5 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java
@@ -1,7 +1,7 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
+ * distributed with this work for additional inconfigation
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
@@ -316,11 +316,11 @@ public class Constants {
public static final String RES_SQOOP_PROMPT_SHELL_LOADEDRC =
"sqoop.prompt_shell_loadedrc";
- public static final String RES_UPDATE_UPDATING_LINK =
+ public static final String RES_SQOOP_UPDATING_LINK =
"update.link";
public static final String RES_UPDATE_LINK_SUCCESSFUL =
"update.link_successful";
- public static final String RES_UPDATE_UPDATING_JOB =
+ public static final String RES_SQOOP_UPDATING_JOB =
"update.job";
public static final String RES_UPDATE_JOB_SUCCESSFUL =
"update.job_successful";
@@ -352,35 +352,35 @@ public class Constants {
public static final String RES_TABLE_HEADER_ENABLED =
"table.header.enabled";
- public static final String RES_FORMDISPLAYER_CONNECTION =
- "formdisplayer.link";
- public static final String RES_FORMDISPLAYER_JOB =
- "formdisplayer.job";
- public static final String RES_FORMDISPLAYER_FORM =
- "formdisplayer.form";
- public static final String RES_FORMDISPLAYER_NAME =
- "formdisplayer.name";
- public static final String RES_FORMDISPLAYER_LABEL =
- "formdisplayer.label";
- public static final String RES_FORMDISPLAYER_HELP =
- "formdisplayer.help";
- public static final String RES_FORMDISPLAYER_INPUT =
- "formdisplayer.input";
- public static final String RES_FORMDISPLAYER_TYPE =
- "formdisplayer.type";
- public static final String RES_FORMDISPLAYER_SENSITIVE =
- "formdisplayer.sensitive";
- public static final String RES_FORMDISPLAYER_SIZE =
- "formdisplayer.size";
- public static final String RES_FORMDISPLAYER_POSSIBLE_VALUES =
- "formdisplayer.possible_values";
- public static final String RES_FORMDISPLAYER_UNSUPPORTED_DATATYPE =
- "formdisplayer.unsupported_datatype";
- public static final String RES_FORMDISPLAYER_INPUT_SENSITIVE =
- "formdisplayer.input_sensitive";
-
- public static final String RES_FORMDISPLAYER_FORM_WARNING =
- "formdisplayer.warning_message";
+ public static final String RES_CONFIG_DISPLAYER_LINK =
+ "config.displayer.link";
+ public static final String RES_CONFIG_DISPLAYER_JOB =
+ "config.displayer.job";
+ public static final String RES_CONFIG_DISPLAYER_CONFIG =
+ "config.displayer.config";
+ public static final String RES_CONFIG_DISPLAYER_NAME =
+ "config.displayer.name";
+ public static final String RES_CONFIG_DISPLAYER_LABEL =
+ "config.displayer.label";
+ public static final String RES_CONFIG_DISPLAYER_HELP =
+ "config.displayer.help";
+ public static final String RES_CONFIG_DISPLAYER_INPUT =
+ "config.displayer.input";
+ public static final String RES_CONFIG_DISPLAYER_TYPE =
+ "config.displayer.type";
+ public static final String RES_CONFIG_DISPLAYER_SENSITIVE =
+ "config.displayer.sensitive";
+ public static final String RES_CONFIG_DISPLAYER_SIZE =
+ "config.displayer.size";
+ public static final String RES_CONFIG_DISPLAYER_POSSIBLE_VALUES =
+ "config.displayer.possible_values";
+ public static final String RES_CONFIG_DISPLAYER_UNSUPPORTED_DATATYPE =
+ "config.displayer.unsupported_datatype";
+ public static final String RES_CONFIG_DISPLAYER_INPUT_SENSITIVE =
+ "config.displayer.input_sensitive";
+
+ public static final String RES_CONFIG_DISPLAYER_FORM_WARNING =
+ "config.displayer.warning_message";
public static final String RES_SUBMISSION_SUBMISSION_DETAIL =
"submission.submission_detail";
[10/52] [abbrv] SQOOP-1497: Sqoop2: Entity Nomenclature Revisited
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
new file mode 100644
index 0000000..7378eda
--- /dev/null
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.repository.derby;
+
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MMapInput;
+import org.apache.sqoop.model.MStringInput;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Test link methods on Derby repository.
+ */
+public class TestLinkHandling extends DerbyTestCase {
+
+ DerbyRepositoryHandler handler;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ handler = new DerbyRepositoryHandler();
+
+ // We always needs schema for this test case
+ createSchema();
+
+ // We always needs connector and framework structures in place
+ loadConnectorAndDriverConfig();
+ }
+
+ public void testFindLink() throws Exception {
+ // Let's try to find non existing link
+ try {
+ handler.findLink(1, getDerbyDatabaseConnection());
+ fail();
+ } catch(SqoopException ex) {
+ assertEquals(DerbyRepoError.DERBYREPO_0024, ex.getErrorCode());
+ }
+
+ // Load prepared connections into database
+ loadLinks();
+
+ MLink connA = handler.findLink(1, getDerbyDatabaseConnection());
+ assertNotNull(connA);
+ assertEquals(1, connA.getPersistenceId());
+ assertEquals("CA", connA.getName());
+
+ List<MForm> forms;
+
+ // Check connector part
+ forms = connA.getConnectorPart().getForms();
+ assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value3", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ // Check framework part
+ forms = connA.getFrameworkPart().getForms();
+ assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+ }
+
+ public void testFindLinks() throws Exception {
+ List<MLink> list;
+
+ // Load empty list on empty repository
+ list = handler.findLinks(getDerbyDatabaseConnection());
+ assertEquals(0, list.size());
+
+ loadLinks();
+
+ // Load all two connections on loaded repository
+ list = handler.findLinks(getDerbyDatabaseConnection());
+ assertEquals(2, list.size());
+
+ assertEquals("CA", list.get(0).getName());
+ assertEquals("CB", list.get(1).getName());
+ }
+
+ public void testExistsLink() throws Exception {
+ // There shouldn't be anything on empty repository
+ assertFalse(handler.existsLink(1, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsLink(2, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsLink(3, getDerbyDatabaseConnection()));
+
+ loadLinks();
+
+ assertTrue(handler.existsLink(1, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsLink(2, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsLink(3, getDerbyDatabaseConnection()));
+ }
+
+ public void testCreateLink() throws Exception {
+ MLink link = getLink();
+
+ // Load some data
+ fillLink(link);
+
+ handler.createLink(link, getDerbyDatabaseConnection());
+
+ assertEquals(1, link.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 1);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
+
+ MLink retrieved = handler.findLink(1, getDerbyDatabaseConnection());
+ assertEquals(1, retrieved.getPersistenceId());
+
+ List<MForm> forms;
+ forms = link.getConnectorPart().getForms();
+ assertEquals("Value1", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value2", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ forms = link.getFrameworkPart().getForms();
+ assertEquals("Value13", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Value15", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ // Let's create second link
+ link = getLink();
+ fillLink(link);
+
+ handler.createLink(link, getDerbyDatabaseConnection());
+
+ assertEquals(2, link.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 2);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 8);
+ }
+
+ public void testInUseLink() throws Exception {
+ loadLinks();
+
+ assertFalse(handler.inUseLink(1, getDerbyDatabaseConnection()));
+
+ loadJobs();
+
+ assertTrue(handler.inUseLink(1, getDerbyDatabaseConnection()));
+ }
+
+ public void testUpdateLink() throws Exception {
+ loadLinks();
+
+ MLink link = handler.findLink(1, getDerbyDatabaseConnection());
+
+ List<MForm> forms;
+
+ forms = link.getConnectorPart().getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(0).getInputs().get(1)).setValue(null);
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(1).getInputs().get(1)).setValue(null);
+
+ forms = link.getFrameworkPart().getForms();
+ ((MStringInput)forms.get(0).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(0).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+ ((MStringInput)forms.get(1).getInputs().get(0)).setValue("Updated");
+ ((MMapInput)forms.get(1).getInputs().get(1)).setValue(new HashMap<String, String>()); // inject new map value
+
+ link.setName("name");
+
+ handler.updateLink(link, getDerbyDatabaseConnection());
+
+ assertEquals(1, link.getPersistenceId());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 2);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 10);
+
+ MLink retrieved = handler.findLink(1, getDerbyDatabaseConnection());
+ assertEquals("name", link.getName());
+
+ forms = retrieved.getConnectorPart().getForms();
+ assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
+ assertNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
+ assertNull(forms.get(1).getInputs().get(1).getValue());
+
+ forms = retrieved.getFrameworkPart().getForms();
+ assertEquals("Updated", forms.get(0).getInputs().get(0).getValue());
+ assertNotNull(forms.get(0).getInputs().get(1).getValue());
+ assertEquals(((Map)forms.get(0).getInputs().get(1).getValue()).size(), 0);
+ assertEquals("Updated", forms.get(1).getInputs().get(0).getValue());
+ assertNotNull(forms.get(1).getInputs().get(1).getValue());
+ assertEquals(((Map)forms.get(1).getInputs().get(1).getValue()).size(), 0);
+ }
+
+ public void testEnableAndDisableLink() throws Exception {
+ loadLinks();
+
+ // disable link 1
+ handler.enableLink(1, false, getDerbyDatabaseConnection());
+
+ MLink retrieved = handler.findLink(1, getDerbyDatabaseConnection());
+ assertNotNull(retrieved);
+ assertEquals(false, retrieved.getEnabled());
+
+ // enable link 1
+ handler.enableLink(1, true, getDerbyDatabaseConnection());
+
+ retrieved = handler.findLink(1, getDerbyDatabaseConnection());
+ assertNotNull(retrieved);
+ assertEquals(true, retrieved.getEnabled());
+ }
+
+ public void testDeleteLink() throws Exception {
+ loadLinks();
+
+ handler.deleteLink(1, getDerbyDatabaseConnection());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 1);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 4);
+
+ handler.deleteLink(2, getDerbyDatabaseConnection());
+ assertCountForTable("SQOOP.SQ_CONNECTION", 0);
+ assertCountForTable("SQOOP.SQ_CONNECTION_INPUT", 0);
+ }
+
+ public MLink getLink() {
+ return new MLink(1,
+ handler.findConnector("A", getDerbyDatabaseConnection()).getConnectionForms(),
+ handler.findDriverConfig(getDerbyDatabaseConnection()).getConnectionForms()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
index 8fce0dd..38a995f 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
@@ -44,10 +44,10 @@ public class TestSubmissionHandling extends DerbyTestCase {
createSchema();
// We always needs connector and framework structures in place
- loadConnectorAndFramework();
+ loadConnectorAndDriverConfig();
// We also always needs connection metadata in place
- loadConnections();
+ loadLinks();
// And finally we always needs job metadata in place
loadJobs();
@@ -56,34 +56,34 @@ public class TestSubmissionHandling extends DerbyTestCase {
public void testFindSubmissionsUnfinished() throws Exception {
List<MSubmission> submissions;
- submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ submissions = handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(0, submissions.size());
loadSubmissions();
- submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ submissions = handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(2, submissions.size());
}
public void testExistsSubmission() throws Exception {
// There shouldn't be anything on empty repository
- assertFalse(handler.existsSubmission(1, getDerbyConnection()));
- assertFalse(handler.existsSubmission(2, getDerbyConnection()));
- assertFalse(handler.existsSubmission(3, getDerbyConnection()));
- assertFalse(handler.existsSubmission(4, getDerbyConnection()));
- assertFalse(handler.existsSubmission(5, getDerbyConnection()));
- assertFalse(handler.existsSubmission(6, getDerbyConnection()));
+ assertFalse(handler.existsSubmission(1, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsSubmission(2, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsSubmission(3, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsSubmission(4, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsSubmission(5, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsSubmission(6, getDerbyDatabaseConnection()));
loadSubmissions();
- assertTrue(handler.existsSubmission(1, getDerbyConnection()));
- assertTrue(handler.existsSubmission(2, getDerbyConnection()));
- assertTrue(handler.existsSubmission(3, getDerbyConnection()));
- assertTrue(handler.existsSubmission(4, getDerbyConnection()));
- assertTrue(handler.existsSubmission(5, getDerbyConnection()));
- assertFalse(handler.existsSubmission(6, getDerbyConnection()));
+ assertTrue(handler.existsSubmission(1, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsSubmission(2, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsSubmission(3, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsSubmission(4, getDerbyDatabaseConnection()));
+ assertTrue(handler.existsSubmission(5, getDerbyDatabaseConnection()));
+ assertFalse(handler.existsSubmission(6, getDerbyDatabaseConnection()));
}
public void testCreateSubmission() throws Exception {
@@ -111,13 +111,13 @@ public class TestSubmissionHandling extends DerbyTestCase {
submission.setExceptionStackTrace("Yeah it happens");
submission.setCounters(counters);
- handler.createSubmission(submission, getDerbyConnection());
+ handler.createSubmission(submission, getDerbyDatabaseConnection());
assertEquals(1, submission.getPersistenceId());
assertCountForTable("SQOOP.SQ_SUBMISSION", 1);
List<MSubmission> submissions =
- handler.findSubmissionsUnfinished(getDerbyConnection());
+ handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(1, submissions.size());
@@ -162,7 +162,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
// Let's create second (simpler) connection
submission =
new MSubmission(1, new Date(), SubmissionStatus.SUCCEEDED, "job-x");
- handler.createSubmission(submission, getDerbyConnection());
+ handler.createSubmission(submission, getDerbyDatabaseConnection());
assertEquals(2, submission.getPersistenceId());
assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
@@ -172,16 +172,16 @@ public class TestSubmissionHandling extends DerbyTestCase {
loadSubmissions();
List<MSubmission> submissions =
- handler.findSubmissionsUnfinished(getDerbyConnection());
+ handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(2, submissions.size());
MSubmission submission = submissions.get(0);
submission.setStatus(SubmissionStatus.SUCCEEDED);
- handler.updateSubmission(submission, getDerbyConnection());
+ handler.updateSubmission(submission, getDerbyDatabaseConnection());
- submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ submissions = handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(1, submissions.size());
}
@@ -190,7 +190,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
loadSubmissions();
List<MSubmission> submissions;
- submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ submissions = handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(2, submissions.size());
assertCountForTable("SQOOP.SQ_SUBMISSION", 5);
@@ -198,23 +198,23 @@ public class TestSubmissionHandling extends DerbyTestCase {
Calendar calendar = Calendar.getInstance();
// 2012-01-03 05:05:05
calendar.set(2012, Calendar.JANUARY, 3, 5, 5, 5);
- handler.purgeSubmissions(calendar.getTime(), getDerbyConnection());
+ handler.purgeSubmissions(calendar.getTime(), getDerbyDatabaseConnection());
- submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ submissions = handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(1, submissions.size());
assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
- handler.purgeSubmissions(new Date(), getDerbyConnection());
+ handler.purgeSubmissions(new Date(), getDerbyDatabaseConnection());
- submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ submissions = handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(0, submissions.size());
assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
- handler.purgeSubmissions(new Date(), getDerbyConnection());
+ handler.purgeSubmissions(new Date(), getDerbyDatabaseConnection());
- submissions = handler.findSubmissionsUnfinished(getDerbyConnection());
+ submissions = handler.findSubmissionsUnfinished(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(0, submissions.size());
assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
@@ -230,16 +230,16 @@ public class TestSubmissionHandling extends DerbyTestCase {
loadSubmissions();
assertCountForTable("SQOOP.SQ_SUBMISSION", 5);
- handler.deleteJob(1, getDerbyConnection());
+ handler.deleteJob(1, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_SUBMISSION", 3);
- handler.deleteJob(2, getDerbyConnection());
+ handler.deleteJob(2, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_SUBMISSION", 2);
- handler.deleteJob(3, getDerbyConnection());
+ handler.deleteJob(3, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_SUBMISSION", 1);
- handler.deleteJob(4, getDerbyConnection());
+ handler.deleteJob(4, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/handler/ConnectionRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/ConnectionRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/ConnectionRequestHandler.java
deleted file mode 100644
index 89ad3ec..0000000
--- a/server/src/main/java/org/apache/sqoop/handler/ConnectionRequestHandler.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.handler;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.audit.AuditLoggerManager;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.framework.FrameworkManager;
-import org.apache.sqoop.json.ConnectionBean;
-import org.apache.sqoop.json.JsonBean;
-import org.apache.sqoop.json.ValidationResultBean;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.repository.Repository;
-import org.apache.sqoop.repository.RepositoryManager;
-import org.apache.sqoop.server.RequestContext;
-import org.apache.sqoop.server.RequestHandler;
-import org.apache.sqoop.server.common.ServerError;
-import org.apache.sqoop.utils.ClassUtils;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.ValidationResult;
-import org.apache.sqoop.validation.ValidationRunner;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-
-/**
- * Connection request handler is supporting following resources:
- *
- * GET /v1/connection/:xid
- * Return details about one particular connection with id :xid or about all of
- * them if :xid equals to "all".
- *
- * POST /v1/connection
- * Create new connection
- *
- * PUT /v1/connection/:xid
- * Update connection with id :xid.
- *
- * PUT /v1/connection/:xid/enable
- * Enable connection with id :xid
- *
- * PUT /v1/connection/:xid/disable
- * Disable connection with id :xid
- *
- * DELETE /v1/connection/:xid
- * Remove connection with id :xid
- *
- * Planned resources:
- *
- * GET /v1/connection
- * Get brief list of all connections present in the system.
- *
- */
-public class ConnectionRequestHandler implements RequestHandler {
-
- private static final Logger LOG =
- Logger.getLogger(ConnectionRequestHandler.class);
-
- private static final String ENABLE = "enable";
- private static final String DISABLE = "disable";
-
- public ConnectionRequestHandler() {
- LOG.info("ConnectionRequestHandler initialized");
- }
-
- @Override
- public JsonBean handleEvent(RequestContext ctx) {
- switch (ctx.getMethod()) {
- case GET:
- return getConnections(ctx);
- case POST:
- return createUpdateConnection(ctx, false);
- case PUT:
- if (ctx.getLastURLElement().equals(ENABLE)) {
- return enableConnection(ctx, true);
- } else if (ctx.getLastURLElement().equals(DISABLE)) {
- return enableConnection(ctx, false);
- } else {
- return createUpdateConnection(ctx, true);
- }
- case DELETE:
- return deleteConnection(ctx);
- }
-
- return null;
- }
-
- /**
- * Delete connection from metadata repository.
- *
- * @param ctx Context object
- * @return Empty bean
- */
- private JsonBean deleteConnection(RequestContext ctx) {
- String sxid = ctx.getLastURLElement();
- long xid = Long.valueOf(sxid);
-
- AuditLoggerManager.getInstance()
- .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "delete", "connection", sxid);
-
- Repository repository = RepositoryManager.getInstance().getRepository();
- repository.deleteConnection(xid);
-
- return JsonBean.EMPTY_BEAN;
- }
-
- /**
- * Update or create connection metadata in repository.
- *
- * @param ctx Context object
- * @return Validation bean object
- */
- private JsonBean createUpdateConnection(RequestContext ctx, boolean update) {
-// Check that given ID equals with sent ID, otherwise report an error UPDATE
-// String sxid = ctx.getLastURLElement();
-// long xid = Long.valueOf(sxid);
-
- String username = ctx.getUserName();
-
- ConnectionBean bean = new ConnectionBean();
-
- try {
- JSONObject json =
- (JSONObject) JSONValue.parse(ctx.getRequest().getReader());
-
- bean.restore(json);
- } catch (IOException e) {
- throw new SqoopException(ServerError.SERVER_0003,
- "Can't read request content", e);
- }
-
- // Get connection object
- List<MConnection> connections = bean.getConnections();
-
- if(connections.size() != 1) {
- throw new SqoopException(ServerError.SERVER_0003,
- "Expected one connection metadata but got " + connections.size());
- }
-
- MConnection connection = connections.get(0);
-
- // Verify that user is not trying to spoof us
- MConnectionForms connectorForms =
- ConnectorManager.getInstance().getConnectorMetadata(connection.getConnectorId())
- .getConnectionForms();
- MConnectionForms frameworkForms = FrameworkManager.getInstance().getFramework()
- .getConnectionForms();
-
- if(!connectorForms.equals(connection.getConnectorPart())
- || !frameworkForms.equals(connection.getFrameworkPart())) {
- throw new SqoopException(ServerError.SERVER_0003,
- "Detected incorrect form structure");
- }
-
- // Responsible connector for this session
- SqoopConnector connector = ConnectorManager.getInstance().getConnector(connection.getConnectorId());
-
- // We need translate forms to configuration objects
- Object connectorConfig = ClassUtils.instantiate(connector.getConnectionConfigurationClass());
- Object frameworkConfig = ClassUtils.instantiate(FrameworkManager.getInstance().getConnectionConfigurationClass());
-
- FormUtils.fromForms(connection.getConnectorPart().getForms(), connectorConfig);
- FormUtils.fromForms(connection.getFrameworkPart().getForms(), frameworkConfig);
-
- // Validate both parts
- ValidationRunner validationRunner = new ValidationRunner();
- ValidationResult connectorValidation = validationRunner.validate(connectorConfig);
- ValidationResult frameworkValidation = validationRunner.validate(frameworkConfig);
-
- Status finalStatus = Status.getWorstStatus(connectorValidation.getStatus(), frameworkValidation.getStatus());
-
- // Return back validations in all cases
- ValidationResultBean outputBean = new ValidationResultBean(connectorValidation, frameworkValidation);
-
- // If we're good enough let's perform the action
- if(finalStatus.canProceed()) {
- if(update) {
- AuditLoggerManager.getInstance()
- .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "update", "connection", String.valueOf(connection.getPersistenceId()));
-
- connection.setLastUpdateUser(username);
- RepositoryManager.getInstance().getRepository().updateConnection(connection);
- } else {
- connection.setCreationUser(username);
- connection.setLastUpdateUser(username);
- RepositoryManager.getInstance().getRepository().createConnection(connection);
- outputBean.setId(connection.getPersistenceId());
-
- AuditLoggerManager.getInstance()
- .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "create", "connection", String.valueOf(connection.getPersistenceId()));
- }
- }
-
- return outputBean;
- }
-
- private JsonBean getConnections(RequestContext ctx) {
- String sxid = ctx.getLastURLElement();
- ConnectionBean bean;
-
- AuditLoggerManager.getInstance()
- .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "get", "connection", sxid);
-
- Locale locale = ctx.getAcceptLanguageHeader();
- Repository repository = RepositoryManager.getInstance().getRepository();
-
- if (sxid.equals("all")) {
-
- List<MConnection> connections = repository.findConnections();
- bean = new ConnectionBean(connections);
-
- // Add associated resources into the bean
- for( MConnection connection : connections) {
- long connectorId = connection.getConnectorId();
- if(!bean.hasConnectorBundle(connectorId)) {
- bean.addConnectorBundle(connectorId,
- ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
- }
- }
- } else {
- long xid = Long.valueOf(sxid);
-
- MConnection connection = repository.findConnection(xid);
- long connectorId = connection.getConnectorId();
-
- bean = new ConnectionBean(connection);
-
- bean.addConnectorBundle(connectorId,
- ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
- }
-
- // Sent framework resource bundle in all cases
- bean.setFrameworkBundle(FrameworkManager.getInstance().getBundle(locale));
-
- return bean;
- }
-
- private JsonBean enableConnection(RequestContext ctx, boolean enabled) {
- String[] elements = ctx.getUrlElements();
- String sxid = elements[elements.length - 2];
- long xid = Long.valueOf(sxid);
-
- Repository repository = RepositoryManager.getInstance().getRepository();
- repository.enableConnection(xid, enabled);
-
- return JsonBean.EMPTY_BEAN;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java
new file mode 100644
index 0000000..c0d363e
--- /dev/null
+++ b/server/src/main/java/org/apache/sqoop/handler/DriverConfigRequestHandler.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.handler;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.audit.AuditLoggerManager;
+import org.apache.sqoop.driver.Driver;
+import org.apache.sqoop.json.DriverConfigBean;
+import org.apache.sqoop.json.JsonBean;
+import org.apache.sqoop.server.RequestContext;
+import org.apache.sqoop.server.RequestHandler;
+
+/**
+ * Driver Config request handler is supporting following resources:
+ *
+ */
+public class DriverConfigRequestHandler implements RequestHandler {
+
+ private static final Logger LOG =
+ Logger.getLogger(DriverConfigRequestHandler.class);
+
+ public DriverConfigRequestHandler() {
+ LOG.info("DriverConfigRequestHandler initialized");
+ }
+
+ @Override
+ public JsonBean handleEvent(RequestContext ctx) {
+ AuditLoggerManager.getInstance()
+ .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
+ "get", "framework", "");
+
+ return new DriverConfigBean(Driver.getInstance().getDriverConfig(),
+ Driver.getInstance().getBundle(ctx.getAcceptLanguageHeader()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/handler/FrameworkRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/FrameworkRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/FrameworkRequestHandler.java
deleted file mode 100644
index 616e974..0000000
--- a/server/src/main/java/org/apache/sqoop/handler/FrameworkRequestHandler.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.handler;
-
-import org.apache.log4j.Logger;
-
-import org.apache.sqoop.audit.AuditLoggerManager;
-import org.apache.sqoop.framework.FrameworkManager;
-import org.apache.sqoop.json.FrameworkBean;
-import org.apache.sqoop.json.JsonBean;
-import org.apache.sqoop.server.RequestContext;
-import org.apache.sqoop.server.RequestHandler;
-
-/**
- * Framework request handler is supporting following resources:
- *
- * GET /v1/framework
- * Get framework metadata.
- */
-public class FrameworkRequestHandler implements RequestHandler {
-
- private static final Logger LOG =
- Logger.getLogger(FrameworkRequestHandler.class);
-
- public FrameworkRequestHandler() {
- LOG.info("FrameworkRequestHandler initialized");
- }
-
- @Override
- public JsonBean handleEvent(RequestContext ctx) {
- AuditLoggerManager.getInstance()
- .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
- "get", "framework", "");
-
- return new FrameworkBean(FrameworkManager.getInstance().getFramework(),
- FrameworkManager.getInstance().getBundle(ctx.getAcceptLanguageHeader()));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
index e2fc86c..b61d3f5 100644
--- a/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
+++ b/server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
@@ -23,7 +23,7 @@ import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.ConnectorManager;
import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.framework.FrameworkManager;
+import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.JsonBean;
import org.apache.sqoop.json.ValidationResultBean;
@@ -170,7 +170,7 @@ public class JobRequestHandler implements RequestHandler {
MJobForms toConnectorForms = ConnectorManager.getInstance()
.getConnectorMetadata(job.getConnectorId(Direction.TO))
.getJobForms(Direction.TO);
- MJobForms frameworkForms = FrameworkManager.getInstance().getFramework()
+ MJobForms frameworkForms = Driver.getInstance().getDriverConfig()
.getJobForms();
if(!fromConnectorForms.equals(job.getConnectorPart(Direction.FROM))
@@ -196,7 +196,7 @@ public class JobRequestHandler implements RequestHandler {
// We need translate forms to configuration objects
Object fromConnectorConfig = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
- Object frameworkConfig = ClassUtils.instantiate(FrameworkManager.getInstance().getJobConfigurationClass());
+ Object frameworkConfig = ClassUtils.instantiate(Driver.getInstance().getJobConfigurationClass());
Object toConnectorConfig = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromConnectorConfig);
@@ -259,8 +259,8 @@ public class JobRequestHandler implements RequestHandler {
// @TODO(Abe): From/To.
for( MJob job : jobs) {
long connectorId = job.getConnectorId(Direction.FROM);
- if(!bean.hasConnectorBundle(connectorId)) {
- bean.addConnectorBundle(connectorId,
+ if(!bean.hasConnectorConfigBundle(connectorId)) {
+ bean.addConnectorConfigBundle(connectorId,
ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
}
}
@@ -274,12 +274,12 @@ public class JobRequestHandler implements RequestHandler {
bean = new JobBean(job);
- bean.addConnectorBundle(connectorId,
+ bean.addConnectorConfigBundle(connectorId,
ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
}
// Sent framework resource bundle in all cases
- bean.setFrameworkBundle(FrameworkManager.getInstance().getBundle(locale));
+ bean.setDriverConfigBundle(Driver.getInstance().getBundle(locale));
return bean;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
new file mode 100644
index 0000000..23fc9f1
--- /dev/null
+++ b/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.handler;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.audit.AuditLoggerManager;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.ConnectorManager;
+import org.apache.sqoop.connector.spi.SqoopConnector;
+import org.apache.sqoop.driver.Driver;
+import org.apache.sqoop.json.LinkBean;
+import org.apache.sqoop.json.JsonBean;
+import org.apache.sqoop.json.ValidationResultBean;
+import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.repository.Repository;
+import org.apache.sqoop.repository.RepositoryManager;
+import org.apache.sqoop.server.RequestContext;
+import org.apache.sqoop.server.RequestHandler;
+import org.apache.sqoop.server.common.ServerError;
+import org.apache.sqoop.utils.ClassUtils;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.ValidationResult;
+import org.apache.sqoop.validation.ValidationRunner;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * Connection request handler is supporting following resources:
+ *
+ * GET /v1/connection/:xid
+ * Return details about one particular connection with id :xid or about all of
+ * them if :xid equals to "all".
+ *
+ * POST /v1/connection
+ * Create new connection
+ *
+ * PUT /v1/connection/:xid
+ * Update connection with id :xid.
+ *
+ * PUT /v1/connection/:xid/enable
+ * Enable connection with id :xid
+ *
+ * PUT /v1/connection/:xid/disable
+ * Disable connection with id :xid
+ *
+ * DELETE /v1/connection/:xid
+ * Remove connection with id :xid
+ *
+ * Planned resources:
+ *
+ * GET /v1/connection
+ * Get brief list of all connections present in the system.
+ *
+ */
+public class LinkRequestHandler implements RequestHandler {
+
+ private static final Logger LOG =
+ Logger.getLogger(LinkRequestHandler.class);
+
+ private static final String ENABLE = "enable";
+ private static final String DISABLE = "disable";
+
+ public LinkRequestHandler() {
+ LOG.info("LinkRequestHandler initialized");
+ }
+
+ @Override
+ public JsonBean handleEvent(RequestContext ctx) {
+ switch (ctx.getMethod()) {
+ case GET:
+ return getConnections(ctx);
+ case POST:
+ return createUpdateConnection(ctx, false);
+ case PUT:
+ if (ctx.getLastURLElement().equals(ENABLE)) {
+ return enableConnection(ctx, true);
+ } else if (ctx.getLastURLElement().equals(DISABLE)) {
+ return enableConnection(ctx, false);
+ } else {
+ return createUpdateConnection(ctx, true);
+ }
+ case DELETE:
+ return deleteConnection(ctx);
+ }
+
+ return null;
+ }
+
+ /**
+ * Delete connection from metadata repository.
+ *
+ * @param ctx Context object
+ * @return Empty bean
+ */
+ private JsonBean deleteConnection(RequestContext ctx) {
+ String sxid = ctx.getLastURLElement();
+ long xid = Long.valueOf(sxid);
+
+ AuditLoggerManager.getInstance()
+ .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
+ "delete", "connection", sxid);
+
+ Repository repository = RepositoryManager.getInstance().getRepository();
+ repository.deleteLink(xid);
+
+ return JsonBean.EMPTY_BEAN;
+ }
+
+ /**
+ * Update or create connection metadata in repository.
+ *
+ * @param ctx Context object
+ * @return Validation bean object
+ */
+ private JsonBean createUpdateConnection(RequestContext ctx, boolean update) {
+// Check that given ID equals with sent ID, otherwise report an error UPDATE
+// String sxid = ctx.getLastURLElement();
+// long xid = Long.valueOf(sxid);
+
+ String username = ctx.getUserName();
+
+ LinkBean bean = new LinkBean();
+
+ try {
+ JSONObject json =
+ (JSONObject) JSONValue.parse(ctx.getRequest().getReader());
+
+ bean.restore(json);
+ } catch (IOException e) {
+ throw new SqoopException(ServerError.SERVER_0003,
+ "Can't read request content", e);
+ }
+
+ // Get connection object
+ List<MLink> connections = bean.getLinks();
+
+ if(connections.size() != 1) {
+ throw new SqoopException(ServerError.SERVER_0003,
+ "Expected one connection metadata but got " + connections.size());
+ }
+
+ MLink connection = connections.get(0);
+
+ // Verify that user is not trying to spoof us
+ MConnectionForms connectorForms =
+ ConnectorManager.getInstance().getConnectorMetadata(connection.getConnectorId())
+ .getConnectionForms();
+ MConnectionForms frameworkForms = Driver.getInstance().getDriverConfig()
+ .getConnectionForms();
+
+ if(!connectorForms.equals(connection.getConnectorPart())
+ || !frameworkForms.equals(connection.getFrameworkPart())) {
+ throw new SqoopException(ServerError.SERVER_0003,
+ "Detected incorrect form structure");
+ }
+
+ // Responsible connector for this session
+ SqoopConnector connector = ConnectorManager.getInstance().getConnector(connection.getConnectorId());
+
+ // We need translate forms to configuration objects
+ Object connectorConfig = ClassUtils.instantiate(connector.getLinkConfigurationClass());
+ Object frameworkConfig = ClassUtils.instantiate(Driver.getInstance().getLinkConfigurationClass());
+
+ FormUtils.fromForms(connection.getConnectorPart().getForms(), connectorConfig);
+ FormUtils.fromForms(connection.getFrameworkPart().getForms(), frameworkConfig);
+
+ // Validate both parts
+ ValidationRunner validationRunner = new ValidationRunner();
+ ValidationResult connectorValidation = validationRunner.validate(connectorConfig);
+ ValidationResult frameworkValidation = validationRunner.validate(frameworkConfig);
+
+ Status finalStatus = Status.getWorstStatus(connectorValidation.getStatus(), frameworkValidation.getStatus());
+
+ // Return back validations in all cases
+ ValidationResultBean outputBean = new ValidationResultBean(connectorValidation, frameworkValidation);
+
+ // If we're good enough let's perform the action
+ if(finalStatus.canProceed()) {
+ if(update) {
+ AuditLoggerManager.getInstance()
+ .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
+ "update", "connection", String.valueOf(connection.getPersistenceId()));
+
+ connection.setLastUpdateUser(username);
+ RepositoryManager.getInstance().getRepository().updateLink(connection);
+ } else {
+ connection.setCreationUser(username);
+ connection.setLastUpdateUser(username);
+ RepositoryManager.getInstance().getRepository().createLink(connection);
+ outputBean.setId(connection.getPersistenceId());
+
+ AuditLoggerManager.getInstance()
+ .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
+ "create", "connection", String.valueOf(connection.getPersistenceId()));
+ }
+ }
+
+ return outputBean;
+ }
+
+ private JsonBean getConnections(RequestContext ctx) {
+ String sxid = ctx.getLastURLElement();
+ LinkBean bean;
+
+ AuditLoggerManager.getInstance()
+ .logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(),
+ "get", "connection", sxid);
+
+ Locale locale = ctx.getAcceptLanguageHeader();
+ Repository repository = RepositoryManager.getInstance().getRepository();
+
+ if (sxid.equals("all")) {
+
+ List<MLink> connections = repository.findLinks();
+ bean = new LinkBean(connections);
+
+ // Add associated resources into the bean
+ for( MLink connection : connections) {
+ long connectorId = connection.getConnectorId();
+ if(!bean.hasConnectorBundle(connectorId)) {
+ bean.addConnectorConfigBundle(connectorId,
+ ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
+ }
+ }
+ } else {
+ long xid = Long.valueOf(sxid);
+
+ MLink connection = repository.findLink(xid);
+ long connectorId = connection.getConnectorId();
+
+ bean = new LinkBean(connection);
+
+ bean.addConnectorConfigBundle(connectorId,
+ ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
+ }
+
+ // Sent framework resource bundle in all cases
+ bean.setDriverConfigBundle(Driver.getInstance().getBundle(locale));
+ return bean;
+ }
+
+ private JsonBean enableConnection(RequestContext ctx, boolean enabled) {
+ String[] elements = ctx.getUrlElements();
+ String sLinkId = elements[elements.length - 2];
+ long linkId = Long.valueOf(sLinkId);
+ Repository repository = RepositoryManager.getInstance().getRepository();
+ repository.enableLink(linkId, enabled);
+ return JsonBean.EMPTY_BEAN;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/handler/SubmissionRequestHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/handler/SubmissionRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/SubmissionRequestHandler.java
index 6e21dd9..8555b0c 100644
--- a/server/src/main/java/org/apache/sqoop/handler/SubmissionRequestHandler.java
+++ b/server/src/main/java/org/apache/sqoop/handler/SubmissionRequestHandler.java
@@ -22,8 +22,7 @@ import java.util.List;
import org.apache.log4j.Logger;
import org.apache.sqoop.audit.AuditLoggerManager;
import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.framework.FrameworkManager;
-import org.apache.sqoop.framework.JobManager;
+import org.apache.sqoop.driver.JobManager;
import org.apache.sqoop.json.JsonBean;
import org.apache.sqoop.json.SubmissionBean;
import org.apache.sqoop.model.MSubmission;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/server/v1/ConnectionServlet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/v1/ConnectionServlet.java b/server/src/main/java/org/apache/sqoop/server/v1/ConnectionServlet.java
deleted file mode 100644
index e8af4d7..0000000
--- a/server/src/main/java/org/apache/sqoop/server/v1/ConnectionServlet.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.server.v1;
-
-import org.apache.sqoop.handler.ConnectionRequestHandler;
-import org.apache.sqoop.json.JsonBean;
-import org.apache.sqoop.server.RequestContext;
-import org.apache.sqoop.server.RequestHandler;
-import org.apache.sqoop.server.SqoopProtocolServlet;
-
-/**
- *
- */
-public class ConnectionServlet extends SqoopProtocolServlet {
-
- private RequestHandler requestHandler;
-
- public ConnectionServlet() {
- requestHandler = new ConnectionRequestHandler();
- }
-
- @Override
- protected JsonBean handleGetRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
- }
-
- @Override
- protected JsonBean handlePostRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
- }
-
- @Override
- protected JsonBean handlePutRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
- }
-
- @Override
- protected JsonBean handleDeleteRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/server/v1/DriverConfigServlet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/v1/DriverConfigServlet.java b/server/src/main/java/org/apache/sqoop/server/v1/DriverConfigServlet.java
new file mode 100644
index 0000000..c2b1f9f
--- /dev/null
+++ b/server/src/main/java/org/apache/sqoop/server/v1/DriverConfigServlet.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.server.v1;
+
+import org.apache.sqoop.handler.DriverConfigRequestHandler;
+import org.apache.sqoop.json.JsonBean;
+import org.apache.sqoop.server.RequestContext;
+import org.apache.sqoop.server.RequestHandler;
+import org.apache.sqoop.server.SqoopProtocolServlet;
+
+/**
+ * Get driver config
+ */
+@SuppressWarnings("serial")
+public class DriverConfigServlet extends SqoopProtocolServlet {
+ private RequestHandler driverConfigRequestHandler;
+
+ public DriverConfigServlet() {
+ driverConfigRequestHandler = new DriverConfigRequestHandler();
+ }
+
+ @Override
+ protected JsonBean handleGetRequest(RequestContext ctx) throws Exception {
+ return driverConfigRequestHandler.handleEvent(ctx);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/server/v1/FrameworkServlet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/v1/FrameworkServlet.java b/server/src/main/java/org/apache/sqoop/server/v1/FrameworkServlet.java
deleted file mode 100644
index 9c99731..0000000
--- a/server/src/main/java/org/apache/sqoop/server/v1/FrameworkServlet.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.server.v1;
-
-import org.apache.sqoop.handler.FrameworkRequestHandler;
-import org.apache.sqoop.json.JsonBean;
-import org.apache.sqoop.server.RequestContext;
-import org.apache.sqoop.server.RequestHandler;
-import org.apache.sqoop.server.SqoopProtocolServlet;
-
-/**
- * Get framework metadata
- */
-public class FrameworkServlet extends SqoopProtocolServlet {
- private RequestHandler frameworkRequestHandler;
-
- public FrameworkServlet() {
- frameworkRequestHandler = new FrameworkRequestHandler();
- }
-
- @Override
- protected JsonBean handleGetRequest(RequestContext ctx) throws Exception {
- return frameworkRequestHandler.handleEvent(ctx);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/server/v1/JobServlet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/v1/JobServlet.java b/server/src/main/java/org/apache/sqoop/server/v1/JobServlet.java
index 34a0ffb..d295237 100644
--- a/server/src/main/java/org/apache/sqoop/server/v1/JobServlet.java
+++ b/server/src/main/java/org/apache/sqoop/server/v1/JobServlet.java
@@ -26,31 +26,32 @@ import org.apache.sqoop.server.SqoopProtocolServlet;
/**
*
*/
+@SuppressWarnings("serial")
public class JobServlet extends SqoopProtocolServlet {
- private RequestHandler requestHandler;
+ private RequestHandler jobRequestHandler;
public JobServlet() {
- requestHandler = new JobRequestHandler();
+ jobRequestHandler = new JobRequestHandler();
}
@Override
protected JsonBean handleGetRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
+ return jobRequestHandler.handleEvent(ctx);
}
@Override
protected JsonBean handlePostRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
+ return jobRequestHandler.handleEvent(ctx);
}
@Override
protected JsonBean handlePutRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
+ return jobRequestHandler.handleEvent(ctx);
}
@Override
protected JsonBean handleDeleteRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
+ return jobRequestHandler.handleEvent(ctx);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/server/v1/LinkServlet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/v1/LinkServlet.java b/server/src/main/java/org/apache/sqoop/server/v1/LinkServlet.java
new file mode 100644
index 0000000..20e5009
--- /dev/null
+++ b/server/src/main/java/org/apache/sqoop/server/v1/LinkServlet.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.server.v1;
+
+import org.apache.sqoop.handler.LinkRequestHandler;
+import org.apache.sqoop.json.JsonBean;
+import org.apache.sqoop.server.RequestContext;
+import org.apache.sqoop.server.RequestHandler;
+import org.apache.sqoop.server.SqoopProtocolServlet;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class LinkServlet extends SqoopProtocolServlet {
+
+ private RequestHandler linkRequestHandler;
+
+ public LinkServlet() {
+ linkRequestHandler = new LinkRequestHandler();
+ }
+
+ @Override
+ protected JsonBean handleGetRequest(RequestContext ctx) throws Exception {
+ return linkRequestHandler.handleEvent(ctx);
+ }
+
+ @Override
+ protected JsonBean handlePostRequest(RequestContext ctx) throws Exception {
+ return linkRequestHandler.handleEvent(ctx);
+ }
+
+ @Override
+ protected JsonBean handlePutRequest(RequestContext ctx) throws Exception {
+ return linkRequestHandler.handleEvent(ctx);
+ }
+
+ @Override
+ protected JsonBean handleDeleteRequest(RequestContext ctx) throws Exception {
+ return linkRequestHandler.handleEvent(ctx);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/java/org/apache/sqoop/server/v1/SubmissionServlet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/v1/SubmissionServlet.java b/server/src/main/java/org/apache/sqoop/server/v1/SubmissionServlet.java
index 7252e11..5c1d883 100644
--- a/server/src/main/java/org/apache/sqoop/server/v1/SubmissionServlet.java
+++ b/server/src/main/java/org/apache/sqoop/server/v1/SubmissionServlet.java
@@ -26,26 +26,27 @@ import org.apache.sqoop.server.SqoopProtocolServlet;
/**
*
*/
+@SuppressWarnings("serial")
public class SubmissionServlet extends SqoopProtocolServlet {
- private RequestHandler requestHandler;
+ private RequestHandler submissionRequestHandler;
public SubmissionServlet() {
- requestHandler = new SubmissionRequestHandler();
+ submissionRequestHandler = new SubmissionRequestHandler();
}
@Override
protected JsonBean handleGetRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
+ return submissionRequestHandler.handleEvent(ctx);
}
@Override
protected JsonBean handlePostRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
+ return submissionRequestHandler.handleEvent(ctx);
}
@Override
protected JsonBean handleDeleteRequest(RequestContext ctx) throws Exception {
- return requestHandler.handleEvent(ctx);
+ return submissionRequestHandler.handleEvent(ctx);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/server/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml
index f053062..d31120a 100644
--- a/server/src/main/webapp/WEB-INF/web.xml
+++ b/server/src/main/webapp/WEB-INF/web.xml
@@ -51,28 +51,29 @@ limitations under the License.
<url-pattern>/v1/connector/*</url-pattern>
</servlet-mapping>
- <!-- Framework servlet -->
+ <!-- Driver Config servlet -->
<servlet>
- <servlet-name>v1.FrameworkServlet</servlet-name>
- <servlet-class>org.apache.sqoop.server.v1.FrameworkServlet</servlet-class>
+ <servlet-name>v1.DriverConfigServlet</servlet-name>
+ <servlet-class>org.apache.sqoop.server.v1.DriverConfigServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
- <servlet-name>v1.FrameworkServlet</servlet-name>
- <url-pattern>/v1/framework/*</url-pattern>
+ <servlet-name>v1.DriverConfigServlet</servlet-name>
+ <url-pattern>/v1/config/driver/*</url-pattern>
</servlet-mapping>
- <!-- Connection servlet -->
+
+ <!-- Link servlet -->
<servlet>
- <servlet-name>v1.ConnectionServlet</servlet-name>
- <servlet-class>org.apache.sqoop.server.v1.ConnectionServlet</servlet-class>
+ <servlet-name>v1.LinkServlet</servlet-name>
+ <servlet-class>org.apache.sqoop.server.v1.LinkServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
- <servlet-name>v1.ConnectionServlet</servlet-name>
- <url-pattern>/v1/connection/*</url-pattern>
+ <servlet-name>v1.LinkServlet</servlet-name>
+ <url-pattern>/v1/link/*</url-pattern>
</servlet-mapping>
<!-- Job servlet -->
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/CloneCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CloneCommand.java b/shell/src/main/java/org/apache/sqoop/shell/CloneCommand.java
index 4cdf0e4..f71f82c 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/CloneCommand.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/CloneCommand.java
@@ -31,7 +31,7 @@ public class CloneCommand extends SqoopCommand {
Constants.CMD_CLONE,
Constants.CMD_CLONE_SC,
ImmutableMap.of(
- Constants.FN_CONNECTION, CloneConnectionFunction.class,
+ Constants.FN_LINK, CloneLinkFunction.class,
Constants.FN_JOB, CloneJobFunction.class
)
);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/CloneConnectionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CloneConnectionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CloneConnectionFunction.java
deleted file mode 100644
index d912c1c..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/CloneConnectionFunction.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import jline.ConsoleReader;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MPersistableEntity;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.ConnectionDynamicFormOptions;
-import org.apache.sqoop.shell.utils.FormOptions;
-import org.apache.sqoop.validation.Status;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class CloneConnectionFunction extends SqoopFunction {
- @SuppressWarnings("static-access")
- public CloneConnectionFunction() {
- this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_PROMPT_CONN_ID))
- .withLongOpt(Constants.OPT_XID)
- .hasArg()
- .isRequired()
- .create(Constants.OPT_XID_CHAR)
- );
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Object executeFunction(CommandLine line, boolean isInteractive) throws IOException {
- return cloneConnection(getLong(line, Constants.OPT_XID), line.getArgList(), isInteractive);
- }
-
- private Status cloneConnection(Long connectionId, List<String> args, boolean isInteractive) throws IOException {
- printlnResource(Constants.RES_CLONE_CLONING_CONN, connectionId);
-
- ConsoleReader reader = new ConsoleReader();
-
- MConnection connection = client.getConnection(connectionId);
- // Remove persistent id as we're making a clone
- connection.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);
-
- Status status = Status.FINE;
-
- ResourceBundle connectorBundle = client.getResourceBundle(connection.getConnectorId());
- ResourceBundle frameworkBundle = client.getFrameworkResourceBundle();
-
- if (isInteractive) {
- printlnResource(Constants.RES_PROMPT_UPDATE_CONN_METADATA);
-
- do {
- // Print error introduction if needed
- if( !status.canProceed() ) {
- errorIntroduction();
- }
-
- // Fill in data from user
- if(!fillConnection(reader, connection, connectorBundle, frameworkBundle)) {
- return null;
- }
-
- status = client.createConnection(connection);
- } while(!status.canProceed());
- } else {
- ConnectionDynamicFormOptions options = new ConnectionDynamicFormOptions();
- options.prepareOptions(connection);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
- if (fillConnection(line, connection)) {
- status = client.createConnection(connection);
- if (!status.canProceed()) {
- printConnectionValidationMessages(connection);
- return null;
- }
- } else {
- printConnectionValidationMessages(connection);
- return null;
- }
- }
-
- printlnResource(Constants.RES_CLONE_CONN_SUCCESSFUL, status.name(), connection.getPersistenceId());
-
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
index ed461ab..97d8e6f 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
@@ -63,10 +63,10 @@ public class CloneJobFunction extends SqoopFunction {
MJob job = client.getJob(jobId);
job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);
- ResourceBundle fromConnectorBundle = client.getResourceBundle(
+ ResourceBundle fromConnectorBundle = client.getConnectorConfigResourceBundle(
job.getConnectorId(Direction.FROM));
- ResourceBundle frameworkBundle = client.getFrameworkResourceBundle();
- ResourceBundle toConnectorBundle = client.getResourceBundle(
+ ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+ ResourceBundle toConnectorBundle = client.getConnectorConfigResourceBundle(
job.getConnectorId(Direction.TO));
Status status = Status.FINE;
@@ -75,7 +75,7 @@ public class CloneJobFunction extends SqoopFunction {
job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);
if (isInteractive) {
- printlnResource(Constants.RES_PROMPT_UPDATE_JOB_METADATA);
+ printlnResource(Constants.RES_PROMPT_UPDATE_JOB_CONFIG);
do {
// Print error introduction if needed
@@ -84,19 +84,19 @@ public class CloneJobFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillJob(reader, job, fromConnectorBundle, frameworkBundle, toConnectorBundle)) {
+ if(!fillJob(reader, job, fromConnectorBundle, driverConfigBundle, toConnectorBundle)) {
return null;
}
// Try to create
- status = client.createJob(job);
+ status = client.saveJob(job);
} while(!status.canProceed());
} else {
JobDynamicFormOptions options = new JobDynamicFormOptions();
options.prepareOptions(job);
CommandLine line = FormOptions.parseOptions(options, 0, args, false);
if (fillJob(line, job)) {
- status = client.createJob(job);
+ status = client.saveJob(job);
if (!status.canProceed()) {
printJobValidationMessages(job);
return null;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
new file mode 100644
index 0000000..d9babe0
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import jline.ConsoleReader;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MPersistableEntity;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.shell.utils.LinkDynamicFormOptions;
+import org.apache.sqoop.shell.utils.FormOptions;
+import org.apache.sqoop.validation.Status;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ResourceBundle;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+import static org.apache.sqoop.shell.utils.FormFiller.*;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class CloneLinkFunction extends SqoopFunction {
+ @SuppressWarnings("static-access")
+ public CloneLinkFunction() {
+ this.addOption(OptionBuilder
+ .withDescription(resourceString(Constants.RES_PROMPT_LINK_ID))
+ .withLongOpt(Constants.OPT_LID)
+ .hasArg()
+ .isRequired()
+ .create(Constants.OPT_LID_CHAR)
+ );
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object executeFunction(CommandLine line, boolean isInteractive) throws IOException {
+ return cloneLink(getLong(line, Constants.OPT_LID), line.getArgList(), isInteractive);
+ }
+
+ private Status cloneLink(Long connectionId, List<String> args, boolean isInteractive) throws IOException {
+ printlnResource(Constants.RES_CLONE_CLONING_LINK, connectionId);
+
+ ConsoleReader reader = new ConsoleReader();
+
+ MLink connection = client.getLink(connectionId);
+ // Remove persistent id as we're making a clone
+ connection.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);
+
+ Status status = Status.FINE;
+
+ ResourceBundle connectorConfigBundle = client.getConnectorConfigResourceBundle(connection.getConnectorId());
+ ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+
+ if (isInteractive) {
+ printlnResource(Constants.RES_PROMPT_UPDATE_LINK_CONFIG);
+
+ do {
+ // Print error introduction if needed
+ if( !status.canProceed() ) {
+ errorIntroduction();
+ }
+
+ // Fill in data from user
+ if(!fillLink(reader, connection, connectorConfigBundle, driverConfigBundle)) {
+ return null;
+ }
+
+ status = client.saveLink(connection);
+ } while(!status.canProceed());
+ } else {
+ LinkDynamicFormOptions options = new LinkDynamicFormOptions();
+ options.prepareOptions(connection);
+ CommandLine line = FormOptions.parseOptions(options, 0, args, false);
+ if (fillConnection(line, connection)) {
+ status = client.saveLink(connection);
+ if (!status.canProceed()) {
+ printLinkValidationMessages(connection);
+ return null;
+ }
+ } else {
+ printLinkValidationMessages(connection);
+ return null;
+ }
+ }
+
+ printlnResource(Constants.RES_CLONE_LINK_SUCCESSFUL, status.name(), connection.getPersistenceId());
+
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/CreateCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CreateCommand.java b/shell/src/main/java/org/apache/sqoop/shell/CreateCommand.java
index fce7c86..1ef2418 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/CreateCommand.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/CreateCommand.java
@@ -31,7 +31,7 @@ public class CreateCommand extends SqoopCommand {
Constants.CMD_CREATE,
Constants.CMD_CREATE_SC,
ImmutableMap.of(
- Constants.FN_CONNECTION, CreateConnectionFunction.class,
+ Constants.FN_LINK, CreateLinkFunction.class,
Constants.FN_JOB, CreateJobFunction.class
)
);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/CreateConnectionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CreateConnectionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CreateConnectionFunction.java
deleted file mode 100644
index 92a8aa5..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/CreateConnectionFunction.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell;
-
-import jline.ConsoleReader;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.ConnectionDynamicFormOptions;
-import org.apache.sqoop.shell.utils.FormDisplayer;
-import org.apache.sqoop.shell.utils.FormOptions;
-import org.apache.sqoop.validation.Status;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-import static org.apache.sqoop.shell.utils.FormFiller.*;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class CreateConnectionFunction extends SqoopFunction {
- @SuppressWarnings("static-access")
- public CreateConnectionFunction() {
- this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_CONNECTOR_ID))
- .withLongOpt(Constants.OPT_CID)
- .isRequired()
- .hasArg()
- .create(Constants.OPT_CID_CHAR));
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Object executeFunction(CommandLine line, boolean isInteractive) throws IOException {
- return createConnection(getLong(line, Constants.OPT_CID), line.getArgList(), isInteractive);
- }
-
- private Status createConnection(long connectorId, List<String> args, boolean isInteractive) throws IOException {
- printlnResource(Constants.RES_CREATE_CREATING_CONN, connectorId);
-
- ConsoleReader reader = new ConsoleReader();
-
- MConnection connection = client.newConnection(connectorId);
-
- ResourceBundle connectorBundle = client.getResourceBundle(connectorId);
- ResourceBundle frameworkBundle = client.getFrameworkResourceBundle();
-
- Status status = Status.FINE;
-
- if (isInteractive) {
- printlnResource(Constants.RES_PROMPT_FILL_CONN_METADATA);
-
- do {
- // Print error introduction if needed
- if( !status.canProceed() ) {
- errorIntroduction();
- }
-
- // Fill in data from user
- if(!fillConnection(reader, connection, connectorBundle, frameworkBundle)) {
- return null;
- }
-
- // Try to create
- status = client.createConnection(connection);
- } while(!status.canProceed());
- } else {
- ConnectionDynamicFormOptions options = new ConnectionDynamicFormOptions();
- options.prepareOptions(connection);
- CommandLine line = FormOptions.parseOptions(options, 0, args, false);
- if (fillConnection(line, connection)) {
- status = client.createConnection(connection);
- if (!status.canProceed()) {
- printConnectionValidationMessages(connection);
- return null;
- }
- } else {
- printConnectionValidationMessages(connection);
- return null;
- }
- }
-
- FormDisplayer.displayFormWarning(connection);
- printlnResource(Constants.RES_CREATE_CONN_SUCCESSFUL, status.name(), connection.getPersistenceId());
-
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
index 88afde3..ccfed31 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
@@ -43,18 +43,18 @@ public class CreateJobFunction extends SqoopFunction {
@SuppressWarnings("static-access")
public CreateJobFunction() {
this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_PROMPT_CONN_ID))
+ .withDescription(resourceString(Constants.RES_PROMPT_LINK_ID))
.withLongOpt(Constants.OPT_FROM)
.isRequired()
.hasArg()
- .create(Constants.OPT_FXID_CHAR)
+ .create(Constants.OPT_FROM_CHAR)
);
this.addOption(OptionBuilder
- .withDescription(resourceString(Constants.RES_PROMPT_CONN_ID))
+ .withDescription(resourceString(Constants.RES_PROMPT_LINK_ID))
.withLongOpt(Constants.OPT_TO)
.isRequired()
.hasArg()
- .create(Constants.OPT_TXID_CHAR)
+ .create(Constants.OPT_TO_CHAR)
);
}
@@ -71,19 +71,19 @@ public class CreateJobFunction extends SqoopFunction {
printlnResource(Constants.RES_CREATE_CREATING_JOB, fromConnectionId, toConnectionId);
ConsoleReader reader = new ConsoleReader();
- MJob job = client.newJob(fromConnectionId, toConnectionId);
+ MJob job = client.createJob(fromConnectionId, toConnectionId);
// @TODO(Abe): From/To.
- ResourceBundle fromConnectorBundle = client.getResourceBundle(
+ ResourceBundle fromConnectorBundle = client.getConnectorConfigResourceBundle(
job.getConnectorId(Direction.FROM));
- ResourceBundle toConnectorBundle = client.getResourceBundle(
+ ResourceBundle toConnectorBundle = client.getConnectorConfigResourceBundle(
job.getConnectorId(Direction.TO));
- ResourceBundle frameworkBundle = client.getFrameworkResourceBundle();
+ ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
Status status = Status.FINE;
if (isInteractive) {
- printlnResource(Constants.RES_PROMPT_FILL_JOB_METADATA);
+ printlnResource(Constants.RES_PROMPT_FILL_JOB_CONFIG);
do {
// Print error introduction if needed
@@ -92,19 +92,19 @@ public class CreateJobFunction extends SqoopFunction {
}
// Fill in data from user
- if(!fillJob(reader, job, fromConnectorBundle, frameworkBundle, toConnectorBundle)) {
+ if(!fillJob(reader, job, fromConnectorBundle, driverConfigBundle, toConnectorBundle)) {
return null;
}
// Try to create
- status = client.createJob(job);
+ status = client.saveJob(job);
} while(!status.canProceed());
} else {
JobDynamicFormOptions options = new JobDynamicFormOptions();
options.prepareOptions(job);
CommandLine line = FormOptions.parseOptions(options, 0, args, false);
if (fillJob(line, job)) {
- status = client.createJob(job);
+ status = client.saveJob(job);
if (!status.canProceed()) {
printJobValidationMessages(job);
return null;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
new file mode 100644
index 0000000..33d60c8
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell;
+
+import jline.ConsoleReader;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.shell.utils.LinkDynamicFormOptions;
+import org.apache.sqoop.shell.utils.FormDisplayer;
+import org.apache.sqoop.shell.utils.FormOptions;
+import org.apache.sqoop.validation.Status;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ResourceBundle;
+
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+import static org.apache.sqoop.shell.utils.FormFiller.*;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class CreateLinkFunction extends SqoopFunction {
+ @SuppressWarnings("static-access")
+ public CreateLinkFunction() {
+ this.addOption(OptionBuilder
+ .withDescription(resourceString(Constants.RES_CONNECTOR_ID))
+ .withLongOpt(Constants.OPT_CID)
+ .isRequired()
+ .hasArg()
+ .create(Constants.OPT_CID_CHAR));
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object executeFunction(CommandLine line, boolean isInteractive) throws IOException {
+ return createLink(getLong(line, Constants.OPT_CID), line.getArgList(), isInteractive);
+ }
+
+ private Status createLink(long connectorId, List<String> args, boolean isInteractive) throws IOException {
+ printlnResource(Constants.RES_CREATE_CREATING_LINK, connectorId);
+
+ ConsoleReader reader = new ConsoleReader();
+
+ MLink link = client.createLink(connectorId);
+
+ ResourceBundle connectorConfigBundle = client.getConnectorConfigResourceBundle(connectorId);
+ ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
+
+ Status status = Status.FINE;
+
+ if (isInteractive) {
+ printlnResource(Constants.RES_PROMPT_FILL_LINK_CONFIG);
+
+ do {
+ // Print error introduction if needed
+ if( !status.canProceed() ) {
+ errorIntroduction();
+ }
+
+ // Fill in data from user
+ if(!fillLink(reader, link, connectorConfigBundle, driverConfigBundle)) {
+ return null;
+ }
+
+ // Try to create
+ status = client.saveLink(link);
+ } while(!status.canProceed());
+ } else {
+ LinkDynamicFormOptions options = new LinkDynamicFormOptions();
+ options.prepareOptions(link);
+ CommandLine line = FormOptions.parseOptions(options, 0, args, false);
+ if (fillConnection(line, link)) {
+ status = client.saveLink(link);
+ if (!status.canProceed()) {
+ printLinkValidationMessages(link);
+ return null;
+ }
+ } else {
+ printLinkValidationMessages(link);
+ return null;
+ }
+ }
+
+ FormDisplayer.displayFormWarning(link);
+ printlnResource(Constants.RES_CREATE_LINK_SUCCESSFUL, status.name(), link.getPersistenceId());
+
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/049994a0/shell/src/main/java/org/apache/sqoop/shell/DeleteCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/DeleteCommand.java b/shell/src/main/java/org/apache/sqoop/shell/DeleteCommand.java
index 107e5e0..26e2bf6 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/DeleteCommand.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/DeleteCommand.java
@@ -31,7 +31,7 @@ public class DeleteCommand extends SqoopCommand {
Constants.CMD_DELETE,
Constants.CMD_DELETE_SC,
ImmutableMap.of(
- Constants.FN_CONNECTION, DeleteConnectionFunction.class,
+ Constants.FN_LINK, DeleteLinkFunction.class,
Constants.FN_JOB, DeleteJobFunction.class
)
);