You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/02/20 17:45:36 UTC
sqoop git commit: SQOOP-1577: Sqoop2: Refactor repository dump and
load tools
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 98d0db1f5 -> 70a08a029
SQOOP-1577: Sqoop2: Refactor repository dump and load tools
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/70a08a02
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/70a08a02
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/70a08a02
Branch: refs/heads/sqoop2
Commit: 70a08a029a2c48cb9b23ad5e06654c7325872575
Parents: 98d0db1
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Fri Feb 20 08:44:57 2015 -0800
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Fri Feb 20 08:44:57 2015 -0800
----------------------------------------------------------------------
.../apache/sqoop/tools/tool/JSONConstants.java | 3 +-
.../sqoop/tools/tool/RepositoryDumpTool.java | 38 ++++++----
.../sqoop/tools/tool/RepositoryLoadTool.java | 78 +++++++++++---------
3 files changed, 69 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
index 9bec0d0..9301707 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
@@ -25,8 +25,9 @@ public final class JSONConstants {
}
public static final String CONNECTOR_ID = "connector-id";
+ public static final String FROM_CONNECTOR_ID = "from-connector-id";
+ public static final String TO_CONNECTOR_ID = "from-connector-id";
public static final String CONNECTOR_NAME = "connector-name";
- public static final String ALL = "all";
public static final String NAME = "name";
public static final String LINK_ID = "link-id";
public static final String JOB_ID = "job-id";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
index 53dac1b..1027baf 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
@@ -21,6 +21,7 @@ import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Iterator;
+import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -31,10 +32,10 @@ import org.apache.commons.cli.ParseException;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.VersionInfo;
import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.json.JobBean;
-import org.apache.sqoop.json.JsonBean;
-import org.apache.sqoop.json.LinkBean;
+import org.apache.sqoop.json.JobsBean;
+import org.apache.sqoop.json.LinksBean;
import org.apache.sqoop.json.SubmissionsBean;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.repository.Repository;
import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.tools.ConfiguredTool;
@@ -101,17 +102,26 @@ public class RepositoryDumpTool extends ConfiguredTool {
JSONObject result = new JSONObject();
- LOG.info("Dumping Connections with skipSensitive=" + String.valueOf(skipSensitive));
- LinkBean links = new LinkBean(repository.findLinks());
- result.put(JSONConstants.LINKS, addConnectorName(links.extract(skipSensitive)));
+ LOG.info("Dumping Links with skipSensitive=" + String.valueOf(skipSensitive));
+ List<MLink> links = repository.findLinks();
+ LinksBean linkBeans = new LinksBean(links);
+ JSONObject linksJsonObject = linkBeans.extract(skipSensitive);
+ JSONArray linksJsonArray = (JSONArray)linksJsonObject.get(JSONConstants.LINKS);
+ addConnectorName(linksJsonArray, JSONConstants.CONNECTOR_ID);
+ result.put(JSONConstants.LINKS, linksJsonObject);
LOG.info("Dumping Jobs with skipSensitive=" + String.valueOf(skipSensitive));
- JobBean jobs = new JobBean(repository.findJobs());
- result.put(JSONConstants.JOBS, addConnectorName(jobs.extract(skipSensitive)));
+ JobsBean jobs = new JobsBean(repository.findJobs());
+ JSONObject jobsJsonObject = jobs.extract(skipSensitive);
+ JSONArray jobsJsonArray = (JSONArray)jobsJsonObject.get(JSONConstants.JOBS);
+ addConnectorName(jobsJsonArray, JSONConstants.FROM_CONNECTOR_ID);
+ addConnectorName(jobsJsonArray, JSONConstants.TO_CONNECTOR_ID);
+ result.put(JSONConstants.JOBS, jobsJsonObject);
LOG.info("Dumping Submissions with skipSensitive=" + String.valueOf(skipSensitive));
SubmissionsBean submissions = new SubmissionsBean(repository.findSubmissions());
- result.put(JSONConstants.SUBMISSIONS, submissions.extract(skipSensitive));
+ JSONObject submissionsJsonObject = submissions.extract(skipSensitive);
+ result.put(JSONConstants.SUBMISSIONS, submissionsJsonObject);
result.put(JSONConstants.METADATA, repoMetadata(skipSensitive));
@@ -129,19 +139,17 @@ public class RepositoryDumpTool extends ConfiguredTool {
return metadata;
}
- private JSONObject addConnectorName(JSONObject json) {
+ private JSONArray addConnectorName(JSONArray jsonArray, String connectorKey) {
ConnectorManager connectorManager = ConnectorManager.getInstance();
- JSONArray results = (JSONArray) json.get(JsonBean.ALL);
-
- Iterator<JSONObject> iterator = results.iterator();
+ Iterator<JSONObject> iterator = jsonArray.iterator();
while (iterator.hasNext()) {
JSONObject result = iterator.next();
- Long connectorId = (Long) result.get(JSONConstants.CONNECTOR_ID);
+ Long connectorId = (Long) result.get(connectorKey);
result.put(JSONConstants.CONNECTOR_NAME, connectorManager.getConnectorConfigurable(connectorId).getUniqueName());
}
- return json;
+ return jsonArray;
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
index d0ebaed..e13cf1e 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
@@ -43,9 +43,9 @@ import org.apache.sqoop.connector.spi.SqoopConnector;
import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.driver.DriverUpgrader;
import org.apache.sqoop.json.JSONUtils;
-import org.apache.sqoop.json.JobBean;
-import org.apache.sqoop.json.LinkBean;
-import org.apache.sqoop.json.SubmissionBean;
+import org.apache.sqoop.json.JobsBean;
+import org.apache.sqoop.json.LinksBean;
+import org.apache.sqoop.json.SubmissionsBean;
import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
@@ -136,19 +136,23 @@ public class RepositoryLoadTool extends ConfiguredTool {
ConnectorManager.getInstance().initialize();
LOG.info("Loading Connections");
- JSONObject jsonConns = (JSONObject) repo.get(JSONConstants.LINKS);
+ JSONObject jsonLinks = (JSONObject) repo.get(JSONConstants.LINKS);
- if (jsonConns == null) {
+ if (jsonLinks == null) {
LOG.error("Malformed JSON file. Key " + JSONConstants.LINKS + " not found.");
return false;
}
- LinkBean linkBean = new LinkBean();
- linkBean.restore(updateConnectorIDUsingName(jsonConns));
+ updateConnectorIDUsingName(
+ (JSONArray)jsonLinks.get(JSONConstants.LINKS),
+ JSONConstants.CONNECTOR_ID);
- HashMap<Long, Long> connectionIds = new HashMap<Long, Long>();
+ LinksBean linksBean = new LinksBean();
+ linksBean.restore(jsonLinks);
- for (MLink link : linkBean.getLinks()) {
+ HashMap<Long, Long> linkIds = new HashMap<Long, Long>();
+
+ for (MLink link : linksBean.getLinks()) {
long oldId = link.getPersistenceId();
long newId = loadLink(link);
if (newId == link.PERSISTANCE_ID_DEFAULT) {
@@ -156,9 +160,9 @@ public class RepositoryLoadTool extends ConfiguredTool {
+ " failed. Aborting repository load. Check log for details.");
return false;
}
- connectionIds.put(oldId, newId);
+ linkIds.put(oldId, newId);
}
- LOG.info("Loaded " + connectionIds.size() + " connections");
+ LOG.info("Loaded " + linkIds.size() + " links");
LOG.info("Loading Jobs");
JSONObject jsonJobs = (JSONObject) repo.get(JSONConstants.JOBS);
@@ -168,12 +172,20 @@ public class RepositoryLoadTool extends ConfiguredTool {
return false;
}
- JobBean jobBean = new JobBean();
- jobBean.restore(updateIdUsingMap(updateConnectorIDUsingName(jsonJobs), connectionIds,
- JSONConstants.LINK_ID));
+ updateConnectorIDUsingName(
+ (JSONArray)jsonJobs.get(JSONConstants.JOBS),
+ JSONConstants.FROM_CONNECTOR_ID);
+ updateConnectorIDUsingName(
+ (JSONArray)jsonJobs.get(JSONConstants.JOBS),
+ JSONConstants.TO_CONNECTOR_ID);
+ updateIdUsingMap((JSONArray)jsonJobs.get(JSONConstants.JOBS),
+ linkIds, JSONConstants.LINK_ID);
+
+ JobsBean jobsBean = new JobsBean();
+ jobsBean.restore(jsonJobs);
HashMap<Long, Long> jobIds = new HashMap<Long, Long>();
- for (MJob job : jobBean.getJobs()) {
+ for (MJob job : jobsBean.getJobs()) {
long oldId = job.getPersistenceId();
long newId = loadJob(job);
@@ -195,10 +207,12 @@ public class RepositoryLoadTool extends ConfiguredTool {
return false;
}
- SubmissionBean submissionBean = new SubmissionBean();
- submissionBean.restore(updateIdUsingMap(jsonSubmissions, jobIds, JSONConstants.JOB_ID));
+ updateIdUsingMap((JSONArray)jsonSubmissions.get(JSONConstants.SUBMISSIONS), jobIds, JSONConstants.JOB_ID);
+
+ SubmissionsBean submissionsBean = new SubmissionsBean();
+ submissionsBean.restore(jsonSubmissions);
int submissionCount = 0;
- for (MSubmission submission : submissionBean.getSubmissions()) {
+ for (MSubmission submission : submissionsBean.getSubmissions()) {
resetPersistenceId(submission);
repository.createSubmission(submission);
submissionCount++;
@@ -238,7 +252,7 @@ public class RepositoryLoadTool extends ConfiguredTool {
private long loadLink(MLink link) {
- // starting by pretending we have a brand new connection
+ // starting by pretending we have a brand new link
resetPersistenceId(link);
Repository repository = RepositoryManager.getInstance().getRepository();
@@ -345,9 +359,7 @@ public class RepositoryLoadTool extends ConfiguredTool {
}
- private JSONObject updateConnectorIDUsingName(JSONObject json) {
- JSONArray array = (JSONArray) json.get(JSONConstants.ALL);
-
+ private JSONArray updateConnectorIDUsingName(JSONArray jsonArray, String connectorIdKey) {
Repository repository = RepositoryManager.getInstance().getRepository();
List<MConnector> connectors = repository.findConnectors();
@@ -357,34 +369,32 @@ public class RepositoryLoadTool extends ConfiguredTool {
connectorMap.put(connector.getUniqueName(), connector.getPersistenceId());
}
- for (Object obj : array) {
+ for (Object obj : jsonArray) {
JSONObject object = (JSONObject) obj;
- long connectorId = (Long) object.get(JSONConstants.CONNECTOR_ID);
+ long connectorId = (Long) object.get(connectorIdKey);
String connectorName = (String) object.get(JSONConstants.CONNECTOR_NAME);
long currentConnectorId = connectorMap.get(connectorName);
- String connectionName = (String) object.get(JSONConstants.NAME);
+ String linkName = (String) object.get(JSONConstants.NAME);
// If a given connector now has a different ID, we need to update the ID
if (connectorId != currentConnectorId) {
- LOG.warn("Connection " + connectionName + " uses connector " + connectorName + ". "
+ LOG.warn("Link " + linkName + " uses connector " + connectorName + ". "
+ "Replacing previous ID " + connectorId + " with new ID " + currentConnectorId);
- object.put(JSONConstants.CONNECTOR_ID, currentConnectorId);
+ object.put(connectorIdKey, currentConnectorId);
}
}
- return json;
- }
- private JSONObject updateIdUsingMap(JSONObject json, HashMap<Long, Long> idMap, String fieldName) {
- JSONArray array = (JSONArray) json.get(JSONConstants.ALL);
+ return jsonArray;
+ }
- for (Object obj : array) {
+ private JSONArray updateIdUsingMap(JSONArray jsonArray, HashMap<Long, Long> idMap, String fieldName) {
+ for (Object obj : jsonArray) {
JSONObject object = (JSONObject) obj;
object.put(fieldName, idMap.get(object.get(fieldName)));
}
- return json;
+ return jsonArray;
}
-
}