You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2014/10/10 04:52:11 UTC
[42/52] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
index b689854..5d48a29 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.connector.hdfs.configuration;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
-@FormClass
+@ConfigClass
public class LinkConfig {
//Todo: Didn't find anything that belongs here...
- // Since empty forms don't work (DERBYREPO_0008:The form contains no input metadata), I'm putting a dummy form here
+ // Since empty forms don't work (DERBYREPO_0008:The config contains no input metadata), I'm putting a dummy config here
@Input(size = 255) public String dummy;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
index 4970821..c0cd336 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
@@ -18,14 +18,14 @@
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
+import org.apache.sqoop.model.Config;
@ConfigurationClass
public class LinkConfiguration {
- @Form
- public LinkConfig link;
+ @Config
+ public LinkConfig linkConfig;
public LinkConfiguration() {
- link = new LinkConfig();
+ linkConfig = new LinkConfig();
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
index b1308db..2dfd738 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
@@ -17,13 +17,13 @@
*/
package org.apache.sqoop.connector.hdfs.configuration;
-import org.apache.sqoop.model.FormClass;
+import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
/**
*
*/
-@FormClass
+@ConfigClass
public class ToJobConfig {
@Input public ToFormat outputFormat;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
index bba249c..c91a975 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
@@ -18,11 +18,11 @@
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
+import org.apache.sqoop.model.Config;
@ConfigurationClass
public class ToJobConfiguration {
- @Form
+ @Config
public ToJobConfig toJobConfig;
public ToJobConfiguration() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
index b603f2f..9b8c6ba 100644
--- a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
+++ b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
@@ -18,12 +18,12 @@
############################
# Link Config
#
-link.label = Link configuration
-link.help = You must supply the information requested in order to \
+linkConfig.label = Link configuration
+linkConfig.help = You must supply the information requested in order to \
create a connection object.
-link.dummy.label = Dummy parameter needed to get HDFS connector to register
-link.dummy.help = You can write anything here. Doesn't matter.
+linkConfig.dummy.label = Dummy parameter needed to get HDFS connector to register
+linkConfig.dummy.help = You can write anything here. Doesn't matter.
# To Job Config
#
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
index 765bedd..f5fbab7 100644
--- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
+++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
@@ -18,6 +18,13 @@
*/
package org.apache.sqoop.connector.idf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Binary;
@@ -26,13 +33,6 @@ import org.apache.sqoop.schema.type.Text;
import org.junit.Before;
import org.junit.Test;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
public class TestCSVIntermediateDataFormat {
private final String BYTE_FIELD_ENCODING = "ISO-8859-1";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
index dbfdc03..54bdd13 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
@@ -19,19 +19,18 @@ package org.apache.sqoop.connector;
import java.io.IOException;
import java.net.URL;
-import java.util.ArrayList;
import java.util.Properties;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.core.ConfigurationConstants;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.SqoopConnector;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.core.ConfigurationConstants;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MFromConfig;
+import org.apache.sqoop.model.MLinkConfig;
+import org.apache.sqoop.model.MToConfig;
public final class ConnectorHandler {
@@ -92,26 +91,25 @@ public final class ConnectorHandler {
connectorClassName, ex);
}
- // Initialize Metadata
- MJobForms fromJobForms = null;
- MJobForms toJobForms = null;
+ MFromConfig fromConfig = null;
+ MToConfig toConfig = null;
if (connector.getSupportedDirections().contains(Direction.FROM)) {
- fromJobForms = new MJobForms(FormUtils.toForms(
+ fromConfig = new MFromConfig(ConfigUtils.toConfigs(
connector.getJobConfigurationClass(Direction.FROM)));
}
if (connector.getSupportedDirections().contains(Direction.TO)) {
- toJobForms = new MJobForms(FormUtils.toForms(
+ toConfig = new MToConfig(ConfigUtils.toConfigs(
connector.getJobConfigurationClass(Direction.TO)));
}
- MConnectionForms connectionForms = new MConnectionForms(
- FormUtils.toForms(connector.getLinkConfigurationClass()));
+ MLinkConfig connectionForms = new MLinkConfig(
+ ConfigUtils.toConfigs(connector.getLinkConfigurationClass()));
String connectorVersion = connector.getVersion();
mConnector = new MConnector(connectorUniqueName, connectorClassName, connectorVersion,
- connectionForms, fromJobForms, toJobForms);
+ connectionForms, fromConfig, toConfig);
if (LOG.isInfoEnabled()) {
LOG.info("Connector [" + connectorClassName + "] initialized.");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
index c87df84..5226926 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
@@ -119,7 +119,7 @@ public class ConnectorManager implements Reconfigurable {
return handler.getConnector().getBundle(locale);
}
- public MConnector getConnectorMetadata(long connectorId) {
+ public MConnector getConnectorConfig(long connectorId) {
ConnectorHandler handler = handlerMap.get(nameMap.get(connectorId));
if(handler == null) {
return null;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/Driver.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/Driver.java b/core/src/main/java/org/apache/sqoop/driver/Driver.java
index 5297bde..f1b45bb 100644
--- a/core/src/main/java/org/apache/sqoop/driver/Driver.java
+++ b/core/src/main/java/org/apache/sqoop/driver/Driver.java
@@ -17,6 +17,7 @@
*/
package org.apache.sqoop.driver;
+import java.util.List;
import java.util.Locale;
import java.util.ResourceBundle;
@@ -26,12 +27,12 @@ import org.apache.sqoop.core.ConfigurationConstants;
import org.apache.sqoop.core.Reconfigurable;
import org.apache.sqoop.core.SqoopConfiguration;
import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
-import org.apache.sqoop.driver.configuration.LinkConfiguration;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.validation.Validator;
@@ -92,14 +93,14 @@ public class Driver implements Reconfigurable {
}
/**
- * Driver config structure
+ * Driver structure
*/
- private MDriverConfig mDriverConfig;
+ private MDriver mDriver;
/**
* Validator instance
*/
- private final Validator validator;
+ private final Validator driverValidator;
/**
* Driver config upgrader instance
@@ -111,38 +112,30 @@ public class Driver implements Reconfigurable {
*/
private static final boolean DEFAULT_AUTO_UPGRADE = false;
- public static final String CURRENT_DRIVER_VERSION = "1";
-
- public Class getJobConfigurationClass() {
- return JobConfiguration.class;
- }
-
- public Class getLinkConfigurationClass() {
- return LinkConfiguration.class;
+ public Class getDriverConfigurationGroupClass() {
+ return DriverConfiguration.class;
}
public Driver() {
- MConnectionForms connectionForms = new MConnectionForms(
- FormUtils.toForms(getLinkConfigurationClass())
- );
- mDriverConfig = new MDriverConfig(connectionForms, new MJobForms(FormUtils.toForms(getJobConfigurationClass())),
- CURRENT_DRIVER_VERSION);
+ List<MConfig> driverConfig = ConfigUtils.toConfigs(getDriverConfigurationGroupClass());
+ mDriver = new MDriver(new MDriverConfig(driverConfig), DriverBean.CURRENT_DRIVER_VERSION);
// Build validator
- validator = new DriverValidator();
+ driverValidator = new DriverConfigValidator();
// Build upgrader
driverConfigUpgrader = new DriverConfigUpgrader();
}
public synchronized void initialize() {
- initialize(SqoopConfiguration.getInstance().getContext().getBoolean(ConfigurationConstants.DRIVER_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE));
+ initialize(SqoopConfiguration.getInstance().getContext()
+ .getBoolean(ConfigurationConstants.DRIVER_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE));
}
public synchronized void initialize(boolean autoUpgrade) {
LOG.trace("Begin Driver Config initialization");
// Register driver config in repository
- mDriverConfig = RepositoryManager.getInstance().getRepository().registerDriverConfig(mDriverConfig, autoUpgrade);
+ mDriver = RepositoryManager.getInstance().getRepository().registerDriver(mDriver, autoUpgrade);
SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this));
@@ -154,15 +147,15 @@ public class Driver implements Reconfigurable {
}
public Validator getValidator() {
- return validator;
+ return driverValidator;
}
public RepositoryUpgrader getDriverConfigRepositoryUpgrader() {
return driverConfigUpgrader;
}
- public MDriverConfig getDriverConfig() {
- return mDriverConfig;
+ public MDriver getDriver() {
+ return mDriver;
}
public ResourceBundle getBundle(Locale locale) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
index 8d6eb78..847b73d 100644
--- a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
@@ -18,56 +18,54 @@
*/
package org.apache.sqoop.driver;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJobForms;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import org.apache.sqoop.model.MLinkConfig;
public class DriverConfigUpgrader extends RepositoryUpgrader{
private static final Logger LOG = Logger.getLogger(DriverConfigUpgrader.class);
@Override
- public void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
+ public void upgrade(MLinkConfig original, MLinkConfig upgradeTarget) {
+ // NOTE(VB): There are no link configs anymore for driver, this code remains for previous versions
}
@Override
- public void upgrade(MJobForms original, MJobForms upgradeTarget) {
- doUpgrade(original.getForms(), upgradeTarget.getForms());
-
+ public void upgrade(MConfigList original, MConfigList upgradeTarget) {
+ doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@SuppressWarnings("unchecked")
- private void doUpgrade(List<MForm> original, List<MForm> target) {
- // Easier to find the form in the original forms list if we use a map.
- // Since the constructor of MJobForms takes a list,
+ private void doUpgrade(List<MConfig> original, List<MConfig> target) {
+ // Easier to find the config in the original list if we use a map.
+ // Since the constructor takes a list,
// index is not guaranteed to be the same, so we need to look for
// equivalence
- Map<String, MForm> formMap = new HashMap<String, MForm>();
- for (MForm form : original) {
- formMap.put(form.getName(), form);
+ Map<String, MConfig> configMap = new HashMap<String, MConfig>();
+ for (MConfig config : original) {
+ configMap.put(config.getName(), config);
}
- for (MForm form : target) {
- List<MInput<?>> inputs = form.getInputs();
- MForm originalForm = formMap.get(form.getName());
- if(originalForm == null) {
- LOG.warn("Form: " + form.getName() + " not present in old " +
+ for (MConfig config : target) {
+ List<MInput<?>> inputs = config.getInputs();
+ MConfig originalConfig = configMap.get(config.getName());
+ if(originalConfig == null) {
+ LOG.warn("Config: " + config.getName() + " not present in old " +
"driver config. So it will not be transferred by the upgrader.");
continue;
}
for (MInput input : inputs) {
try {
- MInput originalInput = originalForm.getInput(input.getName());
+ MInput originalInput = originalConfig.getInput(input.getName());
input.setValue(originalInput.getValue());
} catch (SqoopException ex) {
LOG.warn("Input: " + input.getName() + " not present in old " +
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
new file mode 100644
index 0000000..9c3b660
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
+import org.apache.sqoop.driver.configuration.ThrottlingConfig;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.ConfigValidator;
+import org.apache.sqoop.validation.Validator;
+
+public class DriverConfigValidator extends Validator {
+ @Override
+ public ConfigValidator validateConfigForJob(Object jobConfiguration) {
+ ConfigValidator validation = new ConfigValidator(DriverConfiguration.class);
+ DriverConfiguration conf = (DriverConfiguration)jobConfiguration;
+ validateThrottlingConfig(validation,conf.throttlingConfig);
+
+ return validation;
+ };
+
+ private void validateThrottlingConfig(ConfigValidator validation, ThrottlingConfig throttlingConfig) {
+ if(throttlingConfig.numExtractors != null && throttlingConfig.numExtractors < 1) {
+ validation.addMessage(Status.UNACCEPTABLE, "throttlingConfig", "numExtractors", "You need to specify more than one extractor");
+ }
+
+ if(throttlingConfig.numLoaders != null && throttlingConfig.numLoaders < 1) {
+ validation.addMessage(Status.UNACCEPTABLE, "throttlingConfig", "numLoaders", "You need to specify more than one loader");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
deleted file mode 100644
index 9cc51dd..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver;
-
-import org.apache.sqoop.driver.configuration.LinkConfiguration;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
-import org.apache.sqoop.driver.configuration.ThrottlingForm;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
-import org.apache.sqoop.validation.Validator;
-
-public class DriverValidator extends Validator {
- @Override
- public Validation validateLink(Object linkConfiguration) {
- Validation validation = new Validation(LinkConfiguration.class);
- // No validation on link object
- return validation;
- }
-
- @Override
- public Validation validateJob(Object jobConfiguration) {
- Validation validation = new Validation(JobConfiguration.class);
- JobConfiguration conf = (JobConfiguration)jobConfiguration;
- validateThrottlingForm(validation,conf.throttling);
-
- return validation;
- };
-
- private void validateThrottlingForm(Validation validation, ThrottlingForm throttling) {
- if(throttling.extractors != null && throttling.extractors < 1) {
- validation.addMessage(Status.UNACCEPTABLE, "throttling", "extractors", "You need to specify more than one extractor");
- }
-
- if(throttling.loaders != null && throttling.loaders < 1) {
- validation.addMessage(Status.UNACCEPTABLE, "throttling", "loaders", "You need to specify more than one loader");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/JobManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobManager.java b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
index e91c436..df2a5ab 100644
--- a/core/src/main/java/org/apache/sqoop/driver/JobManager.java
+++ b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
@@ -30,13 +30,13 @@ import org.apache.sqoop.connector.spi.SqoopConnector;
import org.apache.sqoop.core.Reconfigurable;
import org.apache.sqoop.core.SqoopConfiguration;
import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
import org.apache.sqoop.job.etl.Destroyer;
import org.apache.sqoop.job.etl.DestroyerContext;
import org.apache.sqoop.job.etl.Initializer;
import org.apache.sqoop.job.etl.InitializerContext;
import org.apache.sqoop.job.etl.Transferable;
-import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MSubmission;
@@ -311,35 +311,27 @@ public class JobManager implements Reconfigurable {
SqoopConnector toConnector = getConnector(toConnection.getConnectorId());
validateSupportedDirection(toConnector, Direction.TO);
- // Transform config to fromConnector specific classes
- Object fromConnectionConfig = ClassUtils.instantiate(fromConnector
- .getLinkConfigurationClass());
- FormUtils.fromForms(fromConnection.getConnectorPart().getForms(), fromConnectionConfig);
+ // link config for the FROM part of the job
+ Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass());
+ ConfigUtils.fromConfigs(fromConnection.getConnectorLinkConfig().getConfigs(), fromLinkConfig);
- // Transform config to toConnector specific classes
- Object toConnectorConfig = ClassUtils
- .instantiate(toConnector.getLinkConfigurationClass());
- FormUtils.fromForms(toConnection.getConnectorPart().getForms(), toConnectorConfig);
+ // link config for the TO part of the job
+ Object toLinkConfig = ClassUtils.instantiate(toConnector.getLinkConfigurationClass());
+ ConfigUtils.fromConfigs(toConnection.getConnectorLinkConfig().getConfigs(), toLinkConfig);
+ // from config for the job
Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
- FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromJob);
+ ConfigUtils.fromConfigs(job.getJobConfig(Direction.FROM).getConfigs(), fromJob);
+ // to config for the job
Object toJob = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
- FormUtils.fromForms(job.getConnectorPart(Direction.TO).getForms(), toJob);
+ ConfigUtils.fromConfigs(job.getJobConfig(Direction.TO).getConfigs(), toJob);
- // Transform framework specific configs
- // Q(VB) : Aren't the following 2 exactly the same?
- Object fromDriverConnection = ClassUtils.instantiate(Driver.getInstance()
- .getLinkConfigurationClass());
- FormUtils.fromForms(fromConnection.getFrameworkPart().getForms(), fromDriverConnection);
+ // the only driver config for the job
+ Object driverConfig = ClassUtils
+ .instantiate(Driver.getInstance().getDriverConfigurationGroupClass());
+ ConfigUtils.fromConfigs(job.getDriverConfig().getConfigs(), driverConfig);
- Object toDriverConnection = ClassUtils.instantiate(Driver.getInstance()
- .getLinkConfigurationClass());
- FormUtils.fromForms(toConnection.getFrameworkPart().getForms(), toDriverConnection);
-
- Object frameworkJob = ClassUtils.instantiate(Driver.getInstance()
- .getJobConfigurationClass());
- FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);
// Create a job request for submit/execution
JobRequest jobRequest = executionEngine.createJobRequest();
@@ -347,14 +339,14 @@ public class JobManager implements Reconfigurable {
jobRequest.setSummary(submission);
jobRequest.setConnector(Direction.FROM, fromConnector);
jobRequest.setConnector(Direction.TO, toConnector);
- jobRequest.setConnectorLinkConfig(Direction.FROM, fromConnectionConfig);
- jobRequest.setConnectorLinkConfig(Direction.TO, toConnectorConfig);
- jobRequest.setConnectorJobConfig(Direction.FROM, fromJob);
- jobRequest.setConnectorJobConfig(Direction.TO, toJob);
- // TODO(Abe): Should we actually have 2 different Driver Connection config objects?
- jobRequest.setFrameworkLinkConfig(Direction.FROM, fromDriverConnection);
- jobRequest.setFrameworkLinkConfig(Direction.TO, toDriverConnection);
- jobRequest.setFrameworkJobConfig(frameworkJob);
+
+ jobRequest.setConnectorLinkConfig(Direction.FROM, fromLinkConfig);
+ jobRequest.setConnectorLinkConfig(Direction.TO, toLinkConfig);
+
+ jobRequest.setJobConfig(Direction.FROM, fromJob);
+ jobRequest.setJobConfig(Direction.TO, toJob);
+
+ jobRequest.setDriverConfig(driverConfig);
jobRequest.setJobName(job.getName());
jobRequest.setJobId(job.getPersistenceId());
jobRequest.setNotificationUrl(notificationBaseUrl + jobId);
@@ -453,12 +445,12 @@ public class JobManager implements Reconfigurable {
// Initialize submission from the connector perspective
initializer.initialize(initializerContext, jobRequest.getConnectorLinkConfig(direction),
- jobRequest.getConnectorJobConfig(direction));
+ jobRequest.getJobConfig(direction));
return initializer.getSchema(initializerContext,
jobRequest.getConnectorLinkConfig(direction),
- jobRequest.getConnectorJobConfig(direction));
+ jobRequest.getJobConfig(direction));
}
private void addConnectorInitializerJars(JobRequest jobRequest, Direction direction) {
@@ -468,7 +460,7 @@ public class JobManager implements Reconfigurable {
// Add job specific jars to
jobRequest.addJars(initializer.getJars(initializerContext,
jobRequest.getConnectorLinkConfig(direction),
- jobRequest.getConnectorJobConfig(direction)));
+ jobRequest.getJobConfig(direction)));
}
private Initializer getConnectorInitializer(JobRequest jobRequest, Direction direction) {
@@ -488,12 +480,12 @@ public class JobManager implements Reconfigurable {
}
void prepareJob(JobRequest request) {
- JobConfiguration jobConfiguration = (JobConfiguration) request.getFrameworkJobConfig();
+ DriverConfiguration jobConfiguration = (DriverConfiguration) request.getDriverConfig();
// We're directly moving configured number of extractors and loaders to
// underlying request object. In the future we might need to throttle this
// count based on other running jobs to meet our SLAs.
- request.setExtractors(jobConfiguration.throttling.extractors);
- request.setLoaders(jobConfiguration.throttling.loaders);
+ request.setExtractors(jobConfiguration.throttlingConfig.numExtractors);
+ request.setLoaders(jobConfiguration.throttlingConfig.numLoaders);
// Delegate rest of the job to execution engine
executionEngine.prepareJob(request);
@@ -532,9 +524,9 @@ public class JobManager implements Reconfigurable {
// destroy submission from connector perspective
fromDestroyer.destroy(fromDestroyerContext, request.getConnectorLinkConfig(Direction.FROM),
- request.getConnectorJobConfig(Direction.FROM));
+ request.getJobConfig(Direction.FROM));
toDestroyer.destroy(toDestroyerContext, request.getConnectorLinkConfig(Direction.TO),
- request.getConnectorJobConfig(Direction.TO));
+ request.getJobConfig(Direction.TO));
}
public MSubmission stop(long jobId, HttpEventContext ctx) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
index 63e1e49..2666320 100644
--- a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
+++ b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
@@ -78,11 +78,11 @@ public class JobRequest {
*/
Object fromConnectorLinkConfig;
Object toConnectorLinkConfig;
- Object fromConnectorJobConfig;
- Object toConnectorJobConfig;
- Object fromFrameworkLinkConfig;
- Object toFrameworkLinkConfig;
- Object frameworkJobConfig;
+
+ Object fromConfig;
+ Object toConfig;
+
+ Object driverConfig;
/**
* Connector context (submission specific configuration)
@@ -124,10 +124,9 @@ public class JobRequest {
this.toConnector = null;
this.fromConnectorLinkConfig = null;
this.toConnectorLinkConfig = null;
- this.fromConnectorJobConfig = null;
- this.toConnectorJobConfig = null;
- this.fromFrameworkLinkConfig = null;
- this.toFrameworkLinkConfig = null;
+ this.fromConfig = null;
+ this.toConfig = null;
+ this.driverConfig = null;
}
public MSubmission getSummary() {
@@ -244,64 +243,38 @@ public class JobRequest {
}
}
- public Object getConnectorJobConfig(Direction type) {
- switch(type) {
- case FROM:
- return fromConnectorJobConfig;
-
- case TO:
- return toConnectorJobConfig;
-
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public void setConnectorJobConfig(Direction type, Object config) {
- switch(type) {
- case FROM:
- fromConnectorJobConfig = config;
- break;
- case TO:
- toConnectorJobConfig = config;
- break;
- default:
- throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
- }
- }
-
- public Object getFrameworkLinkConfig(Direction type) {
+ public Object getJobConfig(Direction type) {
switch(type) {
case FROM:
- return fromFrameworkLinkConfig;
+ return fromConfig;
case TO:
- return toFrameworkLinkConfig;
+ return toConfig;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
- public void setFrameworkLinkConfig(Direction type, Object config) {
+ public void setJobConfig(Direction type, Object config) {
switch(type) {
case FROM:
- fromFrameworkLinkConfig = config;
+ fromConfig = config;
break;
case TO:
- toFrameworkLinkConfig = config;
+ toConfig = config;
break;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
- public Object getFrameworkJobConfig() {
- return frameworkJobConfig;
+ public Object getDriverConfig() {
+ return driverConfig;
}
- public void setFrameworkJobConfig(Object config) {
- frameworkJobConfig = config;
+ public void setDriverConfig(Object config) {
+ driverConfig = config;
}
public MutableMapContext getConnectorContext(Direction type) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
new file mode 100644
index 0000000..d4e2254
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.ConfigurationClass;
+import org.apache.sqoop.model.Config;
+
+/**
+ * Representing the core job configuration
+ */
+@ConfigurationClass
+public class DriverConfiguration {
+ @Config
+ public ThrottlingConfig throttlingConfig;
+
+ public DriverConfiguration() {
+ throttlingConfig = new ThrottlingConfig();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
deleted file mode 100644
index 908a4eb..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-
-/**
- * Representing the core job configuration
- */
-@ConfigurationClass
-public class JobConfiguration {
- @Form
- public ThrottlingForm throttling;
-
- public JobConfiguration() {
- throttling = new ThrottlingForm();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
deleted file mode 100644
index 3202844..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-
-/**
- * Representing the core link configuration
- */
-@ConfigurationClass
-public class LinkConfiguration {
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
new file mode 100644
index 0000000..357d8e5
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.ConfigClass;
+import org.apache.sqoop.model.Input;
+
+/**
+ * Config to set up number of loaders and extractors
+ */
+@ConfigClass
+public class ThrottlingConfig {
+
+ @Input public Integer numExtractors;
+
+ @Input public Integer numLoaders;
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
deleted file mode 100644
index e73007e..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-
-/**
- * Form to set up number of loaders and extractors
- */
-@FormClass
-public class ThrottlingForm {
-
- @Input public Integer extractors;
-
- @Input public Integer loaders;
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
index 3466116..3ade247 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
@@ -25,7 +25,7 @@ import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
@@ -51,7 +51,7 @@ public class JdbcRepository extends Repository {
/**
* Do what is needed to be done with given link object.
*
- * @param conn Connection to metadata repository.
+ * @param conn Connection to the repository.
* @return Arbitrary value
*/
Object doIt(Connection conn) throws Exception;
@@ -158,8 +158,8 @@ public class JdbcRepository extends Repository {
public Object doIt(Connection conn) throws Exception {
String connectorUniqueName = mConnector.getUniqueName();
- MConnector result = handler.findConnector(connectorUniqueName, conn);
- if (result == null) {
+ MConnector connectorResult = handler.findConnector(connectorUniqueName, conn);
+ if (connectorResult == null) {
handler.registerConnector(mConnector, conn);
return mConnector;
} else {
@@ -167,23 +167,23 @@ public class JdbcRepository extends Repository {
// For now, use the "string" versions itself - later we should
// probably include a build number or something that is
// monotonically increasing.
- if (result.getUniqueName().equals(mConnector.getUniqueName()) &&
- mConnector.getVersion().compareTo(result.getVersion()) > 0) {
+ if (connectorResult.getUniqueName().equals(mConnector.getUniqueName()) &&
+ mConnector.getVersion().compareTo(connectorResult.getVersion()) > 0) {
if (autoUpgrade) {
- upgradeConnector(result, mConnector);
+ upgradeConnector(connectorResult, mConnector);
return mConnector;
} else {
throw new SqoopException(RepositoryError.JDBCREPO_0026,
"Connector: " + mConnector.getUniqueName());
}
}
- if (!result.equals(mConnector)) {
+ if (!connectorResult.equals(mConnector)) {
throw new SqoopException(RepositoryError.JDBCREPO_0013,
"Connector: " + mConnector.getUniqueName()
+ " given: " + mConnector
- + " found: " + result);
+ + " found: " + connectorResult);
}
- return result;
+ return connectorResult;
}
}
});
@@ -220,27 +220,27 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public MDriverConfig registerDriverConfig(final MDriverConfig mDriverConfig, final boolean autoUpgrade) {
- return (MDriverConfig) doWithConnection(new DoWithConnection() {
+ public MDriver registerDriver(final MDriver mDriver, final boolean autoUpgrade) {
+ return (MDriver) doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
- MDriverConfig result = handler.findDriverConfig(conn);
- if (result == null) {
- handler.registerDriverConfig(mDriverConfig, conn);
- return mDriverConfig;
+ MDriver existingDriverConfig = handler.findDriver(conn);
+ if (existingDriverConfig == null) {
+ handler.registerDriver(mDriver, conn);
+ return mDriver;
} else {
// We're currently not serializing version into repository
// so let's just compare the structure to see if we need upgrade.
- if(!mDriverConfig.equals(result)) {
+ if(!mDriver.equals(existingDriverConfig)) {
if (autoUpgrade) {
- upgradeDriverConfig(mDriverConfig);
- return mDriverConfig;
+ upgradeDriver(mDriver);
+ return mDriver;
} else {
throw new SqoopException(RepositoryError.JDBCREPO_0026,
- "DriverConfig: " + mDriverConfig.getPersistenceId());
+ "DriverConfig: " + mDriver.getPersistenceId());
}
}
- return result;
+ return existingDriverConfig;
}
}
});
@@ -664,11 +664,11 @@ public class JdbcRepository extends Repository {
}
- protected void updateDriverConfig(final MDriverConfig mDriverConfig, RepositoryTransaction tx) {
+ protected void updateDriver(final MDriver mDriver, RepositoryTransaction tx) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) throws Exception {
- handler.updateDriverConfig(mDriverConfig, conn);
+ handler.updateDriver(mDriver, conn);
return null;
}
}, (JdbcRepositoryTransaction) tx);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
index a743491..97de893 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
@@ -113,11 +113,11 @@ public abstract class JdbcRepositoryHandler {
* which may not have changed). After this operation the repository is
* guaranteed to only have the new forms specified in this object.
*
- * @param mDriverConfig The new data to be inserted into the repository for
+ * @param mDriver The new data to be inserted into the repository for
* the driverConfig.
* @param conn JDBC link for querying repository
*/
- public abstract void updateDriverConfig(MDriverConfig mDriverConfig, Connection conn);
+ public abstract void updateDriver(MDriver mDriver, Connection conn);
/**
@@ -127,7 +127,7 @@ public abstract class JdbcRepositoryHandler {
* @return null if driverConfig are not yet present in repository or
* loaded representation.
*/
- public abstract MDriverConfig findDriverConfig(Connection conn);
+ public abstract MDriver findDriver(Connection conn);
/**
* Register driver config in repository.
@@ -138,7 +138,7 @@ public abstract class JdbcRepositoryHandler {
* @param driverConfig Driver config that should be registered.
* @param conn JDBC link for querying repository.
*/
- public abstract void registerDriverConfig(MDriverConfig driverConfig, Connection conn);
+ public abstract void registerDriver(MDriver driverConfig, Connection conn);
/**
* Return true if repository tables exists and are suitable for use.
@@ -243,7 +243,7 @@ public abstract class JdbcRepositoryHandler {
*
* @param linkId Link id
* @param conn Connection to the repository
- * @return Deserialized form of the link that is saved in repository
+ * @return Deserialized config of the link that is saved in repository
*/
public abstract MLink findLink(long linkId, Connection conn);
@@ -323,7 +323,7 @@ public abstract class JdbcRepositoryHandler {
*
* @param jobId Job id
* @param conn Connection to the repository
- * @return Deserialized form of the job that is present in the repository
+ * @return Deserialized config of the job that is present in the repository
*/
public abstract MJob findJob(long jobId, Connection conn);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/Repository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java
index ea1608a..95c7a4d 100644
--- a/core/src/main/java/org/apache/sqoop/repository/Repository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java
@@ -28,18 +28,21 @@ import org.apache.sqoop.connector.ConnectorManager;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.connector.spi.SqoopConnector;
import org.apache.sqoop.driver.Driver;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.utils.ClassUtils;
-import org.apache.sqoop.validation.Validation;
+import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Validator;
@@ -76,15 +79,26 @@ public abstract class Repository {
/**
* Registers given connector in the repository and return registered
- * variant. This method might return an exception in case that
- * given connector are already registered with different structure.
+ * variant. This method might return an exception in case that
+ * given connector are already registered with different structure
*
* @param mConnector the connector to be registered
- * autoupgrade whether to upgrade driver config automatically
+ * @param autoUpgrade whether to upgrade driver config automatically
* @return Registered connector structure
*/
public abstract MConnector registerConnector(MConnector mConnector, boolean autoUpgrade);
+ /**
+ * Registers given driver and its config in the repository and return registered
+ * variant. This method might return an exception in case that the
+ * given driverConfig are already registered with different structure
+ *
+ * @param mDriverConfig driverConfig to be registered
+ * @param autoUpgrade whether to upgrade driverConfig automatically
+ * @return Registered connector structure
+ */
+ public abstract MDriver registerDriver(MDriver mDriverConfig, boolean autoUpgrade);
+
/**
* Search for connector with given name in repository.
*
@@ -103,18 +117,6 @@ public abstract class Repository {
*/
public abstract List<MConnector> findConnectors();
-
- /**
- * Registers given driverConfig in the repository and return registered
- * variant. This method might return an exception in case that the
- * given driverConfig are already registered with different structure.
- *
- * @param mDriverConfig driverConfig to be registered
- * autoupgrade whether to upgrade driverConfig automatically
- * @return Registered connector structure
- */
- public abstract MDriverConfig registerDriverConfig(MDriverConfig mDriverConfig, boolean autoUpgrade);
-
/**
* Save given link to repository. This link must not be already
* present in the repository otherwise exception will be thrown.
@@ -164,7 +166,7 @@ public abstract class Repository {
* Find link with given id in repository.
*
* @param id Link id
- * @return Deserialized form of the link that is saved in repository
+ * @return Deserialized config of the link that is saved in repository
*/
public abstract MLink findLink(long id);
@@ -184,7 +186,7 @@ public abstract class Repository {
public abstract void createJob(MJob job);
/**
- * Update given job metadata in repository. This object must already be saved
+ * Update given job entity in repository. This object must already be saved
* in repository otherwise exception will be thrown.
*
* @param job Job object that should be updated in the repository
@@ -192,7 +194,7 @@ public abstract class Repository {
public abstract void updateJob(MJob job);
/**
- * Update given job metadata in repository. This object must already be saved
+ * Update given job entity in repository. This object must already be saved
* in repository otherwise exception will be thrown.
*
* @param job Job object that should be updated in the repository
@@ -204,7 +206,7 @@ public abstract class Repository {
public abstract void updateJob(MJob job, RepositoryTransaction tx);
/**
- * Enable or disable job with given id from metadata repository
+ * Enable or disable job with given id from entity repository
*
* @param id Job object that is going to be enabled or disabled
* @param enabled Enable or disable
@@ -212,7 +214,7 @@ public abstract class Repository {
public abstract void enableJob(long id, boolean enabled);
/**
- * Delete job with given id from metadata repository.
+ * Delete job with given id from entity repository.
*
* @param id Job id that should be removed
*/
@@ -222,7 +224,7 @@ public abstract class Repository {
* Find job object with given id.
*
* @param id Job id
- * @return Deserialized form of job loaded from repository
+ * @return Deserialized config of job loaded from repository
*/
public abstract MJob findJob(long id);
@@ -288,8 +290,7 @@ public abstract class Repository {
* @param connectorID Connector ID whose links should be fetched
* @return List of MLink that use <code>connectorID</code>.
*/
- public abstract List<MLink> findLinksForConnector(long
- connectorID);
+ public abstract List<MLink> findLinksForConnector(long connectorID);
/**
* Retrieve jobs which use the given link.
@@ -297,17 +298,16 @@ public abstract class Repository {
* @param connectorID Connector ID whose jobs should be fetched
* @return List of MJobs that use <code>linkID</code>.
*/
- public abstract List<MJob> findJobsForConnector(long
- connectorID);
+ public abstract List<MJob> findJobsForConnector(long connectorID);
/**
* Update the connector with the new data supplied in the
- * <tt>newConnector</tt>. Also Update all forms associated with this
- * connector in the repository with the forms specified in
+ * <tt>newConnector</tt>. Also Update all configs associated with this
+ * connector in the repository with the configs specified in
* <tt>mConnector</tt>. <tt>mConnector </tt> must
- * minimally have the connectorID and all required forms (including ones
+ * minimally have the connectorID and all required configs (including ones
* which may not have changed). After this operation the repository is
- * guaranteed to only have the new forms specified in this object.
+ * guaranteed to only have the new configs specified in this object.
*
* @param newConnector The new data to be inserted into the repository for
* this connector.
@@ -319,22 +319,22 @@ public abstract class Repository {
protected abstract void updateConnector(MConnector newConnector, RepositoryTransaction tx);
/**
- * Update the driverConfig with the new data supplied in the
- * <tt>mDriverConfig</tt>. Also Update all forms associated with the driverConfig
- * in the repository with the forms specified in
+ * Update the driver with the new data supplied in the
+ * <tt>mDriverConfig</tt>. Also Update all configs associated with the driverConfig
+ * in the repository with the configs specified in
* <tt>mDriverConfig</tt>. <tt>mDriverConfig </tt> must
- * minimally have the connectorID and all required forms (including ones
+ * minimally have the connectorID and all required configs (including ones
* which may not have changed). After this operation the repository is
- * guaranteed to only have the new forms specified in this object.
+ * guaranteed to only have the new configs specified in this object.
*
- * @param mDriverConfig The new data to be inserted into the repository for
+ * @param mDriver The new data to be inserted into the repository for
* the driverConfig.
* @param tx The repository transaction to use to push the data to the
* repository. If this is null, a new transaction will be created.
* method will not call begin, commit,
* rollback or close on this transaction.
*/
- protected abstract void updateDriverConfig(MDriverConfig mDriverConfig, RepositoryTransaction tx);
+ protected abstract void updateDriver(MDriver mDriver, RepositoryTransaction tx);
/**
* Delete all inputs for a job
@@ -365,6 +365,13 @@ public abstract class Repository {
}
}
+ private void deleteJobs(List<MJob> jobs, RepositoryTransaction tx) {
+ for (MJob job : jobs) {
+ deleteJobInputs(job.getPersistenceId(), tx);
+ }
+ }
+
+
/**
* Upgrade the connector with the same {@linkplain MConnector#uniqueName}
* in the repository with values from <code>newConnector</code>.
@@ -377,7 +384,7 @@ public abstract class Repository {
* upgraded.
*/
public final void upgradeConnector(MConnector oldConnector, MConnector newConnector) {
- LOG.info("Upgrading metadata for connector: " + oldConnector.getUniqueName());
+ LOG.info("Upgrading connector: " + oldConnector.getUniqueName());
long connectorID = oldConnector.getPersistenceId();
newConnector.setPersistenceId(connectorID);
/* Algorithms:
@@ -385,8 +392,8 @@ public abstract class Repository {
* 2. Get all links associated with the connector.
* 3. Get all jobs associated with the connector.
* 4. Delete the inputs for all of the jobs and links (in that order)
- * 5. Remove all inputs and forms associated with the connector, and
- * register the new forms and inputs.
+ * 5. Remove all inputs and configs associated with the connector, and
+ * register the new configs and inputs.
* 6. Create new links and jobs with connector part being the ones
* returned by the upgrader.
* 7. Validate new links and jobs with connector's validator
@@ -401,85 +408,63 @@ public abstract class Repository {
ConnectorManager.getInstance().getConnector(newConnector
.getUniqueName());
- Validator validator = connector.getValidator();
-
+ Validator connectorConfigValidator = connector.getConfigValidator();
boolean upgradeSuccessful = true;
-
RepositoryUpgrader upgrader = connector.getRepositoryUpgrader();
- List<MLink> links = findLinksForConnector(
- connectorID);
- List<MJob> jobs = findJobsForConnector(connectorID);
+ List<MLink> linksByConnector = findLinksForConnector(connectorID);
+ List<MJob> jobsByConnector = findJobsForConnector(connectorID);
// -- BEGIN TXN --
tx = getTransaction();
tx.begin();
- deletelinksAndJobs(links, jobs, tx);
+ deletelinksAndJobs(linksByConnector, jobsByConnector, tx);
updateConnector(newConnector, tx);
- for (MLink link : links) {
- // Make a new copy of the forms from the connector,
- // else the values will get set in the forms in the connector for
- // each link.
- List<MForm> forms = newConnector.getConnectionForms().clone(false).getForms();
- MConnectionForms newlinkForms = new MConnectionForms(forms);
- upgrader.upgrade(link.getConnectorPart(), newlinkForms);
- MLink newlink = new MLink(link, newlinkForms, link.getFrameworkPart());
-
- // Transform form structures to objects for validations
+ for (MLink oldLink : linksByConnector) {
+ // Make a new copy of the configs
+ List<MConfig> linkConfig = newConnector.getLinkConfig().clone(false).getConfigs();
+ MLinkConfig newLinkConfig = new MLinkConfig(linkConfig);
+ MLinkConfig oldLinkConfig = oldLink.getConnectorLinkConfig();
+ upgrader.upgrade(oldLinkConfig, newLinkConfig);
+
+ MLink newlink = new MLink(oldLink, newLinkConfig);
+
Object newConfigurationObject = ClassUtils.instantiate(connector.getLinkConfigurationClass());
- FormUtils.fromForms(newlink.getConnectorPart().getForms(), newConfigurationObject);
+ ConfigUtils.fromConfigs(newlink.getConnectorLinkConfig().getConfigs(), newConfigurationObject);
- Validation validation = validator.validateLink(newConfigurationObject);
- if (validation.getStatus().canProceed()) {
+ ConfigValidator configValidator = connectorConfigValidator.validateConfigForLink(newConfigurationObject);
+ if (configValidator.getStatus().canProceed()) {
updateLink(newlink, tx);
} else {
- logInvalidModelObject("link", newlink, validation);
+ logInvalidModelObject("link", newlink, configValidator);
upgradeSuccessful = false;
}
}
- for (MJob job : jobs) {
- // Make a new copy of the forms from the connector,
- // else the values will get set in the forms in the connector for
+ for (MJob job : jobsByConnector) {
+ // Make a new copy of the configs
+ // else the values will get set in the configs in the connector for
// each job.
- List<MForm> fromForms = newConnector.getJobForms(Direction.FROM).clone(false).getForms();
- List<MForm> toForms = newConnector.getJobForms(Direction.TO).clone(false).getForms();
+ List<MConfig> fromConfig = newConnector.getConfig(Direction.FROM).clone(false).getConfigs();
+ List<MConfig> toConfig = newConnector.getConfig(Direction.TO).clone(false).getConfigs();
- // New FROM direction forms, old TO direction forms.
+ // New FROM direction configs, old TO direction configs.
if (job.getConnectorId(Direction.FROM) == newConnector.getPersistenceId()) {
- MJobForms newFromJobForms = new MJobForms(fromForms);
- MJobForms oldToJobForms = job.getConnectorPart(Direction.TO);
- upgrader.upgrade(job.getConnectorPart(Direction.FROM), newFromJobForms);
- MJob newJob = new MJob(job, newFromJobForms, oldToJobForms, job.getFrameworkPart());
- updateJob(newJob, tx);
+ MFromConfig newFromConfig = new MFromConfig(fromConfig);
+ MFromConfig oldFromCOnfig = job.getFromJobConfig();
+ upgrader.upgrade(oldFromCOnfig, newFromConfig);
- // Transform form structures to objects for validations
-// Object newFromConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.FROM));
-// FormUtils.fromForms(newJob.getConnectorPart(Direction.FROM).getForms(), newFromConfigurationObject);
-// Validation fromValidation = validator.validateJob(newFromConfigurationObject);
-// if (fromValidation.getStatus().canProceed()) {
-// updateJob(newJob, tx);
-// } else {
-// logInvalidModelObject("job", newJob, fromValidation);
-// upgradeSuccessful = false;
-// }
+ MToConfig oldToConfig = job.getToJobConfig();
+ MJob newJob = new MJob(job, newFromConfig, oldToConfig, job.getDriverConfig());
+ updateJob(newJob, tx);
}
- // Old FROM direction forms, new TO direction forms.
+ // Old FROM direction configs, new TO direction configs.
if (job.getConnectorId(Direction.TO) == newConnector.getPersistenceId()) {
- MJobForms oldFromJobForms = job.getConnectorPart(Direction.FROM);
- MJobForms newToJobForms = new MJobForms(toForms);
- upgrader.upgrade(job.getConnectorPart(Direction.TO), newToJobForms);
- MJob newJob = new MJob(job, oldFromJobForms, newToJobForms, job.getFrameworkPart());
- updateJob(newJob, tx);
- // Transform form structures to objects for validations
-// Object newToConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.TO));
-// FormUtils.fromForms(newJob.getConnectorPart(Direction.TO).getForms(), newToConfigurationObject);
-// Validation toValidation = validator.validateJob(newToConfigurationObject);
-// if (toValidation.getStatus().canProceed()) {
-// updateJob(newJob, tx);
-// } else {
-// logInvalidModelObject("job", newJob, toValidation);
-// upgradeSuccessful = false;
-// }
+ MToConfig oldToConfig = job.getToJobConfig();
+ MToConfig newToConfig = new MToConfig(toConfig);
+ upgrader.upgrade(oldToConfig, newToConfig);
+ MFromConfig oldFromConfig = job.getFromJobConfig();
+ MJob newJob = new MJob(job, oldFromConfig, newToConfig, job.getDriverConfig());
+ updateJob(newJob, tx);
}
}
@@ -506,60 +491,35 @@ public abstract class Repository {
}
}
- public final void upgradeDriverConfig(MDriverConfig driverConfig) {
- LOG.info("Upgrading driver config");
+ public final void upgradeDriver(MDriver driver) {
+ LOG.info("Upgrading driver");
RepositoryTransaction tx = null;
try {
- RepositoryUpgrader upgrader = Driver.getInstance()
+ RepositoryUpgrader driverConfigUpgrader = Driver.getInstance()
.getDriverConfigRepositoryUpgrader();
- List<MLink> links = findLinks();
List<MJob> jobs = findJobs();
Validator validator = Driver.getInstance().getValidator();
-
boolean upgradeSuccessful = true;
// -- BEGIN TXN --
tx = getTransaction();
tx.begin();
- deletelinksAndJobs(links, jobs, tx);
- updateDriverConfig(driverConfig, tx);
- for (MLink link : links) {
- // Make a new copy of the forms from the connector,
- // else the values will get set in the forms in the connector for
- // each link.
- // @TODO(Abe): From/To link forms.
- List<MForm> forms = driverConfig.getConnectionForms().clone(false).getForms();
- MConnectionForms newlinkForms = new MConnectionForms(forms);
- upgrader.upgrade(link.getFrameworkPart(), newlinkForms);
- MLink newlink = new MLink(link, link.getConnectorPart(), newlinkForms);
-
- // Transform form structures to objects for validations
- Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getLinkConfigurationClass());
- FormUtils.fromForms(newlink.getFrameworkPart().getForms(), newConfigurationObject);
-
- Validation validation = validator.validateLink(newConfigurationObject);
- if (validation.getStatus().canProceed()) {
- updateLink(newlink, tx);
- } else {
- logInvalidModelObject("link", newlink, validation);
- upgradeSuccessful = false;
- }
- }
+ deleteJobs(jobs, tx);
+ updateDriver(driver, tx);
+
for (MJob job : jobs) {
- // Make a new copy of the forms from the framework,
- // else the values will get set in the forms in the connector for
- // each link.
- List<MForm> forms = driverConfig.getJobForms().clone(false).getForms();
- MJobForms newJobForms = new MJobForms(forms);
- upgrader.upgrade(job.getFrameworkPart(), newJobForms);
- MJob newJob = new MJob(job, job.getConnectorPart(Direction.FROM), job.getConnectorPart(Direction.TO), newJobForms);
-
- // Transform form structures to objects for validations
- Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getJobConfigurationClass());
- FormUtils.fromForms(newJob.getFrameworkPart().getForms(), newConfigurationObject);
-
- Validation validation = validator.validateJob(newConfigurationObject);
+ // Make a new copy of the configs
+ MDriverConfig driverConfig = driver.getDriverConfig().clone(false);
+ MDriver newDriver = new MDriver(driverConfig, DriverBean.CURRENT_DRIVER_VERSION);
+ driverConfigUpgrader.upgrade(job.getDriverConfig(), newDriver.getDriverConfig());
+ MJob newJob = new MJob(job, job.getFromJobConfig(), job.getToJobConfig(), newDriver.getDriverConfig());
+
+ // Transform config structures to objects for validations
+ Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getDriverConfigurationGroupClass());
+ ConfigUtils.fromConfigs(newJob.getDriverConfig().getConfigs(), newConfigurationObject);
+
+ ConfigValidator validation = validator.validateConfigForJob(newConfigurationObject);
if (validation.getStatus().canProceed()) {
updateJob(newJob, tx);
} else {
@@ -587,14 +547,14 @@ public abstract class Repository {
if(tx != null) {
tx.close();
}
- LOG.info("Driver config upgrade finished");
+ LOG.info("Driver upgrade finished");
}
}
- private void logInvalidModelObject(String objectType, MPersistableEntity entity, Validation validation) {
+ private void logInvalidModelObject(String objectType, MPersistableEntity entity, ConfigValidator validation) {
LOG.error("Upgrader created invalid " + objectType + " with id" + entity.getPersistenceId());
- for(Map.Entry<Validation.FormInput, Validation.Message> entry : validation.getMessages().entrySet()) {
+ for(Map.Entry<ConfigValidator.ConfigInput, ConfigValidator.Message> entry : validation.getMessages().entrySet()) {
LOG.error("\t" + entry.getKey() + ": " + entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
index ae7be82..c2f8505 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
@@ -118,7 +118,7 @@ public class RepositoryManager implements Reconfigurable {
provider.initialize(context);
if(!immutableRepository) {
- LOG.info("Creating or upgrading on disk structures if necessary");
+ LOG.info("Creating or update respository internals at bootup");
provider.getRepository().createOrUpdateInternals();
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/resources/driver-config.properties
----------------------------------------------------------------------
diff --git a/core/src/main/resources/driver-config.properties b/core/src/main/resources/driver-config.properties
index 9ec5d9e..78c3b38 100644
--- a/core/src/main/resources/driver-config.properties
+++ b/core/src/main/resources/driver-config.properties
@@ -19,12 +19,12 @@
# Throttling From
#
-throttling.label = Throttling resources
-throttling.help = Set throttling boundaries to not overload your systems
+throttthrottlingConfigling.label = Throttling resources
+throttlingConfig.help = Set throttling boundaries to not overload your systems
-throttling.extractors.label = Extractors
-throttling.extractors.help = Number of extractors that Sqoop will use
+throttlingConfig.numExtractors.label = Extractors
+throttlingConfig.numExtractors.help = Number of extractors that Sqoop will use
-throttling.loaders.label = Loaders
-throttling.loaders.help = Number of loaders that Sqoop will use
+throttlingConfig.numLoaders.label = Loaders
+throttlingConfig.numLoaders.help = Number of loaders that Sqoop will use
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
index 9c39d23..dc4e8c8 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
@@ -18,16 +18,19 @@
*/
package org.apache.sqoop.driver;
-import org.apache.sqoop.driver.DriverConfigUpgrader;
-import org.apache.sqoop.model.*;
-import org.junit.Before;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import java.util.LinkedList;
import java.util.List;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MStringInput;
+import org.junit.Before;
+import org.junit.Test;
/**
*/
@@ -39,35 +42,14 @@ public class TestDriverConfigUpgrader {
public void initializeUpgrader() {
upgrader = new DriverConfigUpgrader();
}
-
- /**
- * We take the same forms on input and output and we
- * expect that all values will be correctly transferred.
- */
- @Test
- public void testConnectionUpgrade() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection1();
-
- original.getStringInput("f1.s1").setValue("A");
- original.getStringInput("f1.s2").setValue("B");
- original.getIntegerInput("f1.i").setValue(3);
-
- upgrader.upgrade(original, target);
-
- assertEquals("A", target.getStringInput("f1.s1").getValue());
- assertEquals("B", target.getStringInput("f1.s2").getValue());
- assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
- }
-
/**
- * We take the same forms on input and output and we
+ * We take the same configs on input and output and we
* expect that all values will be correctly transferred.
*/
@Test
- public void testJobUpgrade() {
- MJobForms original = job1();
- MJobForms target = job1();
+ public void testJobConfigTyeUpgrade() {
+ MConfigList original = job();
+ MConfigList target = job();
original.getStringInput("f1.s1").setValue("A");
original.getStringInput("f1.s2").setValue("B");
@@ -85,8 +67,8 @@ public class TestDriverConfigUpgrader {
*/
@Test
public void testNonExistingInput() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection2();
+ MConfigList original = job1();
+ MConfigList target = job2();
original.getStringInput("f1.s1").setValue("A");
original.getStringInput("f1.s2").setValue("B");
@@ -104,9 +86,9 @@ public class TestDriverConfigUpgrader {
* therefore is missing in the original.
*/
@Test
- public void testNonExistingForm() {
- MConnectionForms original = connection1();
- MConnectionForms target = connection3();
+ public void testNonExistingConfig() {
+ MConfigList original = job1();
+ MConfigList target = job3();
original.getStringInput("f1.s1").setValue("A");
original.getStringInput("f1.s2").setValue("B");
@@ -119,25 +101,25 @@ public class TestDriverConfigUpgrader {
assertNull(target.getIntegerInput("f2.i").getValue());
}
- MJobForms job1() {
- return new MJobForms(forms1());
+ MConfigList job() {
+ return new MConfigList(configs1());
}
- MConnectionForms connection1() {
- return new MConnectionForms(forms1());
+ MConfigList job1() {
+ return new MConfigList(configs1());
}
- MConnectionForms connection2() {
- return new MConnectionForms(forms2());
+ MConfigList job2() {
+ return new MConfigList(configs2());
}
- MConnectionForms connection3() {
- return new MConnectionForms(forms3());
+ MConfigList job3() {
+ return new MConfigList(configs3());
}
- List<MForm> forms1() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f1", inputs1("f1")));
+ List<MConfig> configs1() {
+ List<MConfig> list = new LinkedList<MConfig>();
+ list.add(new MConfig("f1", inputs1("f1")));
return list;
}
@@ -149,9 +131,9 @@ public class TestDriverConfigUpgrader {
return list;
}
- List<MForm> forms2() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f1", inputs2("f1")));
+ List<MConfig> configs2() {
+ List<MConfig> list = new LinkedList<MConfig>();
+ list.add(new MConfig("f1", inputs2("f1")));
return list;
}
@@ -163,9 +145,9 @@ public class TestDriverConfigUpgrader {
return list;
}
- List<MForm> forms3() {
- List<MForm> list = new LinkedList<MForm>();
- list.add(new MForm("f2", inputs1("f2")));
+ List<MConfig> configs3() {
+ List<MConfig> list = new LinkedList<MConfig>();
+ list.add(new MConfig("f2", inputs1("f2")));
return list;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
index aa3af89..3b475c6 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
@@ -20,6 +20,13 @@ package org.apache.sqoop.driver;
import java.util.Arrays;
import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.ConnectorManager;
@@ -34,13 +41,6 @@ import org.apache.sqoop.request.HttpEventContext;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
public class TestJobManager {
private JobManager jobManager;
private SqoopConnector sqoopConnectorMock;
@@ -99,7 +99,7 @@ public class TestJobManager {
@Test
public void testGetLink() {
- MLink testLink = new MLink(123l, null, null);
+ MLink testLink = new MLink(123l, null);
testLink.setEnabled(true);
MLink mConnectionSpy = org.mockito.Mockito.spy(testLink);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
@@ -111,7 +111,7 @@ public class TestJobManager {
@Test
public void testDisabledLink() {
- MLink testConnection = new MLink(123l, null, null);
+ MLink testConnection = new MLink(123l, null);
testConnection.setPersistenceId(1234);
testConnection.setEnabled(false);
SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection id: "