You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2014/10/10 05:07:27 UTC
[35/50] [abbrv] SQOOP-1498: Sqoop2: Repository Object refactoring
(objects prefixed with M)
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
deleted file mode 100644
index a0a5dc2..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormFiller.java
+++ /dev/null
@@ -1,939 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import jline.ConsoleReader;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.lang.StringUtils;
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MEnumInput;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MIntegerInput;
-import org.apache.sqoop.model.MMapInput;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MNamedElement;
-import org.apache.sqoop.model.MStringInput;
-import org.apache.sqoop.model.MValidatedElement;
-import org.apache.sqoop.validation.Message;
-import org.apache.sqoop.validation.Status;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.ResourceBundle;
-
-import static org.apache.sqoop.shell.ShellEnvironment.*;
-
-/**
- * Convenient methods for retrieving user input and CLI options.
- */
-public final class FormFiller {
-
- /**
- * Internal input that will be reused for loading names for link and
- * job objects.
- */
- private static MStringInput nameInput = new MStringInput("object-name", false, (short)25);
-
- /**
- * Fill job object based on CLI options.
- *
- * @param line Associated console reader object
- * @param job Job that user is suppose to fill in
- * @return True if we filled all inputs, false if user has stopped processing
- * @throws IOException
- */
- public static boolean fillJob(CommandLine line,
- MJob job)
- throws IOException {
-
- job.setName(line.getOptionValue("name"));
-
- // Fill in data from user
- return fillForms(line,
- job.getConnectorPart(Direction.FROM).getForms(),
- job.getFrameworkPart().getForms());
- }
-
- /**
- * Fill job object based on user input.
- *
- * @param reader Associated console reader object
- * @param job Job that user is suppose to fill in
- * @param fromConnectorBundle Connector resource bundle
- * @param driverConfigBundle Driver config resource bundle
- * @return True if we filled all inputs, false if user has stopped processing
- * @throws IOException
- */
- public static boolean fillJob(ConsoleReader reader,
- MJob job,
- ResourceBundle fromConnectorBundle,
- ResourceBundle driverConfigBundle,
- ResourceBundle toConnectorBundle)
- throws IOException {
-
- job.setName(getName(reader, job.getName()));
-
- // Fill in data from user
- return fillForms(reader,
- job.getConnectorPart(Direction.FROM).getForms(),
- fromConnectorBundle,
- job.getFrameworkPart().getForms(),
- driverConfigBundle,
- job.getConnectorPart(Direction.TO).getForms(),
- toConnectorBundle);
- }
-
- /**
- * Fill link object based on CLI options.
- *
- * @param line Associated command line options
- * @param link Link that user is suppose to fill in
- * @return True if we filled all inputs, false if user has stopped processing
- * @throws IOException
- */
- public static boolean fillConnection(CommandLine line,
- MLink link)
- throws IOException {
-
- link.setName(line.getOptionValue("name"));
-
- // Fill in data from user
- return fillForms(line,
- link.getConnectorPart().getForms(),
- link.getFrameworkPart().getForms());
- }
-
- /**
- * Fill link object based on user input.
- *
- * @param reader Associated console reader object
- * @param link Link that user is suppose to fill in
- * @param connectorConfigBundle Connector resource bundle
- * @param driverConfigBundle Driver config resource bundle
- * @return True if we filled all inputs, false if user has stopped processing
- * @throws IOException
- */
- public static boolean fillLink(ConsoleReader reader,
- MLink link,
- ResourceBundle connectorConfigBundle,
- ResourceBundle driverConfigBundle)
- throws IOException {
-
- link.setName(getName(reader, link.getName()));
-
- // Fill in data from user
- return fillForms(reader,
- link.getConnectorPart().getForms(),
- connectorConfigBundle,
- link.getFrameworkPart().getForms(),
- driverConfigBundle);
- }
-
- /**
- * Load CLI options for framework forms and connector forms.
- *
- * @param line CLI options container
- * @param connectorForms Connector forms to read or edit
- * @param frameworkForms Framework forms to read or edit
- * @return
- * @throws IOException
- */
- public static boolean fillForms(CommandLine line,
- List<MForm> connectorForms,
- List<MForm> frameworkForms)
- throws IOException {
- // Query connector forms and framework forms
- return fillForms("connector", connectorForms, line)
- && fillForms("framework", frameworkForms, line);
- }
-
- /**
- * Load all CLI options for a list of forms.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param forms Forms to read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- public static boolean fillForms(String prefix,
- List<MForm> forms,
- CommandLine line)
- throws IOException {
- for (MForm form : forms) {
- if (!fillForm(prefix, form, line)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Load all CLI options for a particular form.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param form Form to read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- @SuppressWarnings("rawtypes")
- public static boolean fillForm(String prefix,
- MForm form,
- CommandLine line) throws IOException {
- for (MInput input : form.getInputs()) {
- if (!fillInput(prefix, input, line)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Load CLI option.
- * Chooses the appropriate 'fill' method to use based on input type.
- *
- * Keys for CLI options are automatically created from the 'prefix' argument
- * and 'input' argument: <prefix>-<form name>-<input name>
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- @SuppressWarnings("rawtypes")
- public static boolean fillInput(String prefix,
- MInput input,
- CommandLine line) throws IOException {
- // Based on the input type, let's perform specific load
- switch (input.getType()) {
- case STRING:
- return fillInputString(prefix, (MStringInput) input, line);
- case INTEGER:
- return fillInputInteger(prefix, (MIntegerInput) input, line);
- case BOOLEAN:
- return fillInputBoolean(prefix, (MBooleanInput) input, line);
- case MAP:
- return fillInputMap(prefix, (MMapInput) input, line);
- case ENUM:
- return fillInputEnum(prefix, (MEnumInput) input, line);
- default:
- println("Unsupported data type " + input.getType());
- return true;
- }
- }
-
- /**
- * Load CLI option for enum type.
- *
- * Currently only supports numeric values.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- private static boolean fillInputEnum(String prefix,
- MEnumInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- String value = line.getOptionValue(opt);
- int index = java.util.Arrays.asList(input.getValues()).indexOf(value);
-
- if(index < 0) {
- errorMessage(input, String.format("Invalid option %s. Please use one of %s.", value, StringUtils.join(input.getValues(), ", ")));
- return false;
- }
-
- input.setValue(value);
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- /**
- * Load CLI options for map type.
- *
- * Parses Key-Value pairs that take the form "<key>=<value>&<key>=<value>&...".
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- private static boolean fillInputMap(String prefix,
- MMapInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- String value = line.getOptionValue(opt);
- Map<String, String> values = new HashMap<String, String>();
- String[] entries = value.split("&");
- for (String entry : entries) {
- if (entry.contains("=")) {
- String[] keyValue = entry.split("=");
- values.put(keyValue[0], keyValue[1]);
- } else {
- errorMessage(input, "Don't know what to do with " + entry);
- return false;
- }
- }
- input.setValue(values);
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- /**
- * Load integer input from CLI option.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- private static boolean fillInputInteger(String prefix,
- MIntegerInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- try {
- input.setValue(Integer.valueOf(line.getOptionValue(FormOptions.getOptionKey(prefix, input))));
- } catch (NumberFormatException ex) {
- errorMessage(input, "Input is not valid integer number");
- return false;
- }
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- /**
- * Load string input from CLI option.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- public static boolean fillInputString(String prefix,
- MStringInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- String value = line.getOptionValue(FormOptions.getOptionKey(prefix, input));
- if(value.length() > input.getMaxLength()) {
- errorMessage(input, "Size of input exceeds allowance for this input"
- + " field. Maximal allowed size is " + input.getMaxLength());
- }
- input.setValue(value);
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- /**
- * Load boolean input from CLI option.
- *
- * @param prefix placed at the beginning of the CLI option key
- * @param input Input that we should read or edit
- * @param line CLI options container
- * @return
- * @throws IOException
- */
- public static boolean fillInputBoolean(String prefix,
- MBooleanInput input,
- CommandLine line)
- throws IOException {
- String opt = FormOptions.getOptionKey(prefix, input);
- if (line.hasOption(opt)) {
- input.setValue(Boolean.valueOf(line.getOptionValue(FormOptions.getOptionKey(prefix, input))));
- } else {
- input.setEmpty();
- }
- return true;
- }
-
- public static boolean fillForms(ConsoleReader reader,
- List<MForm> connectorForms,
- ResourceBundle connectorConfigBundle,
- List<MForm> frameworkForms,
- ResourceBundle driverConfigBundle) throws IOException {
-
-
- // Query connector forms
- if(!fillForms(connectorForms, reader, connectorConfigBundle)) {
- return false;
- }
-
- // Query framework forms
- if(!fillForms(frameworkForms, reader, driverConfigBundle)) {
- return false;
- }
- return true;
- }
-
- public static boolean fillForms(ConsoleReader reader,
- List<MForm> fromConnectorForms,
- ResourceBundle fromConnectorBundle,
- List<MForm> frameworkForms,
- ResourceBundle driverConfigBundle,
- List<MForm> toConnectorForms,
- ResourceBundle toConnectorBundle) throws IOException {
-
-
- // From connector forms
- if(!fillForms(fromConnectorForms, reader, fromConnectorBundle)) {
- return false;
- }
-
- // Query framework forms
- if(!fillForms(frameworkForms, reader, driverConfigBundle)) {
- return false;
- }
-
- // To connector forms
- if(!fillForms(toConnectorForms, reader, toConnectorBundle)) {
- return false;
- }
-
- return true;
- }
-
- public static boolean fillForms(List<MForm> forms,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- for (MForm form : forms) {
- if(!fillForm(form, reader, bundle)) {
- return false;
- }
- }
-
- return true;
- }
-
- @SuppressWarnings("rawtypes")
- public static boolean fillForm(MForm form,
- ConsoleReader reader,
- ResourceBundle bundle) throws IOException {
- println("");
- println(bundle.getString(form.getLabelKey()));
-
- // Print out form validation
- printValidationMessage(form, false);
- println("");
-
- for (MInput input : form.getInputs()) {
- if(!fillInput(input, reader, bundle)) {
- return false;
- }
- }
-
- return true;
- }
-
- @SuppressWarnings("rawtypes")
- public static boolean fillInput(MInput input,
- ConsoleReader reader,
- ResourceBundle bundle) throws IOException {
- // Print out validation
- printValidationMessage(input, false);
-
- // Based on the input type, let's perform specific load
- switch (input.getType()) {
- case STRING:
- return fillInputString((MStringInput) input, reader, bundle);
- case INTEGER:
- return fillInputInteger((MIntegerInput) input, reader, bundle);
- case BOOLEAN:
- return fillInputBoolean((MBooleanInput) input, reader, bundle);
- case MAP:
- return fillInputMap((MMapInput) input, reader, bundle);
- case ENUM:
- return fillInputEnum((MEnumInput) input, reader, bundle);
- default:
- println("Unsupported data type " + input.getType());
- return true;
- }
- }
-
- /**
- * Load user input for enum type.
- *
- * Print out numbered list of all available options and let user choose one
- * item from that.
- *
- * @param input Input that we should read or edit
- * @param reader Associated console reader
- * @param bundle Resource bundle
- * @return True if user with to continue with loading addtional inputs
- * @throws IOException
- */
- private static boolean fillInputEnum(MEnumInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- // Prompt in enum case
- println(bundle.getString(input.getLabelKey()) + ": ");
-
- // Indexes
- int i = -1;
- int lastChoice = -1;
-
- // Print out all values as a numbered list
- for(String value : input.getValues()) {
- i++;
-
- println(" " + i + " : " + value);
-
- // Only show last choice if not sensitive
- if(!input.isEmpty() && value.equals(input.getValue()) && !input.isSensitive()) {
- lastChoice = i;
- }
- }
-
- // Prompt
- reader.printString("Choose: ");
-
- // Fill previously filled index when available
- if(lastChoice != -1) {
- reader.putString(Integer.toString(lastChoice));
- }
-
- reader.flushConsole();
- String userTyped;
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if (userTyped == null) {
- return false;
- } else if (userTyped.isEmpty()) {
- input.setEmpty();
- } else {
- Integer index;
- try {
- index = Integer.valueOf(userTyped);
-
- if(index < 0 || index >= input.getValues().length) {
- errorMessage("Invalid index");
- return fillInputEnum(input, reader, bundle);
- }
-
- input.setValue(input.getValues()[index]);
- } catch (NumberFormatException ex) {
- errorMessage("Input is not valid integer number");
- return fillInputEnum(input, reader, bundle);
- }
- }
-
- return true;
- }
-
- /**
- * Load user input for map type.
- *
- * This implementation will load one map entry at the time. Current flows is
- * as follows: if user did not enter anything (empty input) finish loading
- * and return from function. If user specified input with equal sign (=),
- * lets add new key value pair. Otherwise consider entire input as a key name
- * and try to remove it from the map.
- *
- * Please note that following code do not supports equal sign in property
- * name. It's however perfectly fine to have equal sign in value.
- *
- * @param input Input that we should read or edit
- * @param reader Associated console reader
- * @param bundle Resource bundle
- * @return True if user wish to continue with loading additional inputs
- * @throws IOException
- */
- private static boolean fillInputMap(MMapInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- // Special prompt in Map case
- println(bundle.getString(input.getLabelKey()) + ": ");
-
- // Internal loading map
- Map<String, String> values = input.getValue();
- if(values == null) {
- values = new HashMap<String, String>();
- }
-
- String userTyped;
-
- while(true) {
- // Print all current items in each iteration
- // However do not printout if this input contains sensitive information.
- println("There are currently " + values.size() + " values in the map:");
- if (!input.isSensitive()) {
- for(Map.Entry<String, String> entry : values.entrySet()) {
- println(entry.getKey() + " = " + entry.getValue());
- }
- }
-
- // Special prompt for Map entry
- reader.printString("entry# ");
- reader.flushConsole();
-
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if(userTyped == null) {
- // Finish loading and return back to Sqoop shell
- return false;
- } else if(userTyped.isEmpty()) {
- // User has finished loading data to Map input, either set input empty
- // if there are no entries or propagate entries to the input
- if(values.size() == 0) {
- input.setEmpty();
- } else {
- input.setValue(values);
- }
- return true;
- } else {
- // User has specified regular input, let's check if it contains equals
- // sign. Save new entry (or update existing one) if it does. Otherwise
- // try to remove entry that user specified.
- if(userTyped.contains("=")) {
- String []keyValue = userTyped.split("=", 2);
- values.put(handleUserInput(keyValue[0]), handleUserInput(keyValue[1]));
- } else {
- String key = handleUserInput(userTyped);
- if(values.containsKey(key)) {
- values.remove(key);
- } else {
- errorMessage("Don't know what to do with " + userTyped);
- }
- }
- }
-
- }
- }
-
- /**
- * Handle special cases in user input.
- *
- * Preserve null and empty values, remove whitespace characters before and
- * after loaded string and de-quote the string if it's quoted (to preserve
- * spaces for example).
- *
- * @param input String loaded from user
- * @return Unquoted transformed string
- */
- private static String handleUserInput(String input) {
- // Preserve null and empty values
- if(input == null) {
- return null;
- }
- if(input.isEmpty()) {
- return input;
- }
-
- // Removes empty characters at the begging and end of loaded string
- input = input.trim();
-
- int lastIndex = input.length() - 1;
- char first = input.charAt(0);
- char last = input.charAt(lastIndex);
-
- // Remove quoting if present
- if(first == '\'' && last == '\'') {
- input = input.substring(1, lastIndex);
- } else if(first == '"' && last == '"') {
- input = input.substring(1, lastIndex);
- }
-
- // Return final string
- return input;
- }
-
- private static boolean fillInputInteger(MIntegerInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- generatePrompt(reader, bundle, input);
-
- // Fill already filled data when available
- // However do not printout if this input contains sensitive information.
- if(!input.isEmpty() && !input.isSensitive()) {
- reader.putString(input.getValue().toString());
- }
-
- // Get the data
- String userTyped;
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if (userTyped == null) {
- return false;
- } else if (userTyped.isEmpty()) {
- input.setEmpty();
- } else {
- Integer value;
- try {
- value = Integer.valueOf(userTyped);
- input.setValue(value);
- } catch (NumberFormatException ex) {
- errorMessage("Input is not valid integer number");
- return fillInputInteger(input, reader, bundle);
- }
-
- input.setValue(Integer.valueOf(userTyped));
- }
-
- return true;
- }
-
- /**
- * Load string input from the user.
- *
- * @param input Input that we should load in
- * @param reader Associated console reader
- * @param bundle Resource bundle for this input
- * @return
- * @throws IOException
- */
- public static boolean fillInputString(MStringInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- generatePrompt(reader, bundle, input);
-
- // Fill already filled data when available
- // However do not printout if this input contains sensitive information.
- if(!input.isEmpty() && !input.isSensitive()) {
- reader.putString(input.getValue());
- }
-
- // Get the data
- String userTyped;
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if (userTyped == null) {
- // Propagate end of loading process
- return false;
- } else if (userTyped.isEmpty()) {
- // Empty input in case that nothing was given
- input.setEmpty();
- } else {
- // Set value that user has entered
- input.setValue(userTyped);
-
- // Check that it did not exceeds maximal allowance for given input
- if(userTyped.length() > input.getMaxLength()) {
- errorMessage("Size of input exceeds allowance for this input"
- + " field. Maximal allowed size is " + input.getMaxLength());
- return fillInputString(input, reader, bundle);
- }
- }
-
- return true;
- }
-
- /**
- * Load boolean input from the user.
- *
- * @param input Input that we should load in
- * @param reader Associated console reader
- * @param bundle Resource bundle for this input
- * @return
- * @throws IOException
- */
- public static boolean fillInputBoolean(MBooleanInput input,
- ConsoleReader reader,
- ResourceBundle bundle)
- throws IOException {
- generatePrompt(reader, bundle, input);
-
- // Fill already filled data when available
- // However do not printout if this input contains sensitive information.
- if(!input.isEmpty() && !input.isSensitive()) {
- reader.putString(input.getValue().toString());
- }
-
- // Get the data
- String userTyped;
- if(input.isSensitive()) {
- userTyped = reader.readLine('*');
- } else {
- userTyped = reader.readLine();
- }
-
- if (userTyped == null) {
- // Propagate end of loading process
- return false;
- } else if (userTyped.isEmpty()) {
- // Empty input in case that nothing was given
- input.setEmpty();
- } else {
- // Set value that user has entered
- input.setValue(Boolean.valueOf(userTyped));
- }
-
- return true;
- }
-
- @SuppressWarnings("rawtypes")
- public static void generatePrompt(ConsoleReader reader,
- ResourceBundle bundle,
- MInput input)
- throws IOException {
- reader.printString(bundle.getString(input.getLabelKey()) + ": ");
- reader.flushConsole();
- }
-
- public static String getName(ConsoleReader reader,
- String name) throws IOException {
- if(name == null) {
- nameInput.setEmpty();
- } else {
- nameInput.setValue(name);
- }
-
- fillInputString(nameInput, reader, getResourceBundle());
-
- return nameInput.getValue();
- }
-
- /**
- * Print validation message in cases that it's not in state "FINE"
- *
- * @param element Validated element
- */
- public static void printValidationMessage(MValidatedElement element, boolean includeInputPrefix) {
- if(element.getValidationStatus() == Status.getDefault()) {
- return;
- }
-
- for(Message message : element.getValidationMessages())
- switch (message.getStatus()) {
- case UNACCEPTABLE:
- if (includeInputPrefix) {
- errorMessage(element, message.getMessage());
- } else {
- errorMessage(message.getMessage());
- }
- break;
- case ACCEPTABLE:
- if (includeInputPrefix) {
- warningMessage(element, message.getMessage());
- } else {
- warningMessage(message.getMessage());
- }
- break;
- default:
- // Simply ignore all other states for the moment
- break;
- }
- }
-
- public static void errorMessage(String message) {
- println("Error message: @|red " + message + " |@");
- }
-
- public static void errorMessage(MNamedElement input, String message) {
- print(input.getName());
- print(": ");
- errorMessage(message);
- }
-
- public static void warningMessage(String message) {
- println("Warning message: @|yellow " + message + " |@");
- }
-
- public static void warningMessage(MNamedElement input, String message) {
- print(input.getName());
- print(": ");
- warningMessage(message);
- }
-
- public static void errorIntroduction() {
- println();
- println("@|red There are issues with entered data, please revise your input:|@");
- }
-
- public static void printLinkValidationMessages(MLink link) {
- for (MForm form : link.getConnectorPart().getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- for (MForm form : link.getFrameworkPart().getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- }
-
- public static void printJobValidationMessages(MJob job) {
- for (MForm form : job.getConnectorPart(Direction.FROM).getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- for (MForm form : job.getFrameworkPart().getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- for (MForm form : job.getConnectorPart(Direction.TO).getForms()) {
- for (MInput<?> input : form.getInputs()) {
- printValidationMessage(input, true);
- }
- }
- }
-
- private FormFiller() {
- // Do not instantiate
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java
deleted file mode 100644
index efd002e..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/FormOptions.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.model.MForm;
-import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MInputType;
-import org.apache.sqoop.shell.core.ShellError;
-
-/**
- * Utilities for automatically creating org.apache.commons.cli.Option objects.
- */
-public class FormOptions {
- /**
- * This method is used to automatically generate keys
- * for a particular input.
- *
- * @param prefix Prefix to prepend to CLI option keys
- * @param input
- * @return
- */
- @SuppressWarnings("rawtypes")
- public static String getOptionKey(String prefix, MInput input) {
- return prefix + "-" + input.getName().replace('.', '-');
- }
-
- /**
- * This method is used to automatically generate CLI options
- * for a list of forms.
- *
- * @param prefix Prefix to prepend to CLI option keys
- * @param forms Forms to get options for
- * @return
- */
- public static List<Option> getFormsOptions(String prefix, List<MForm> forms) {
- List<Option> options = new LinkedList<Option>();
- for (MForm form : forms) {
- List<Option> formOptions = getFormOptions(prefix, form);
- options.addAll(formOptions);
- }
- return options;
- }
-
- /**
- * This method is used to automatically generate CLI options
- * for a particular form.
- *
- * @param prefix Prefix to prepend to CLI option keys
- * @param form Form to get options for
- * @return List<Option>
- */
- @SuppressWarnings({ "rawtypes", "static-access" })
- public static List<Option> getFormOptions(String prefix, MForm form) {
- List<Option> options = new LinkedList<Option>();
- for (MInput input : form.getInputs()) {
- if (input.getType().equals(MInputType.BOOLEAN)) {
- options.add(OptionBuilder
- .withLongOpt(getOptionKey(prefix, input))
- .create());
- } else {
- options.add(OptionBuilder
- .withLongOpt(getOptionKey(prefix, input))
- .hasArg()
- .create());
- }
- }
- return options;
- }
-
- /**
- * Parses command line options.
- *
- * @param options parse arglist against these.
- * @param start beginning index in arglist.
- * @param arglist arguments to parse.
- * @param stopAtNonOption stop parsing when nonoption found in arglist.
- * @return CommandLine object
- */
- public static CommandLine parseOptions(Options options, int start, List<String> arglist, boolean stopAtNonOption) {
- String[] args = arglist.subList(start, arglist.size()).toArray(new String[arglist.size() - start]);
-
- CommandLineParser parser = new GnuParser();
- CommandLine line;
- try {
- line = parser.parse(options, args, stopAtNonOption);
- } catch (ParseException e) {
- throw new SqoopException(ShellError.SHELL_0003, e.getMessage(), e);
- }
- return line;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java
new file mode 100644
index 0000000..49b09c7
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicConfigOptions.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.model.MJob;
+
+/**
+ * Automatically create dynamic options for jobs.
+ */
+@SuppressWarnings("serial")
+public class JobDynamicConfigOptions extends DynamicConfigOptions<MJob> {
+
+ @SuppressWarnings("static-access")
+ @Override
+ public void prepareOptions(MJob job) {
+ this.addOption(OptionBuilder
+ .withLongOpt("name")
+ .hasArg()
+ .create());
+ for (Option option : ConfigOptions.getConfigsOptions("from", job.getJobConfig(Direction.FROM).getConfigs())) {
+ this.addOption(option);
+ }
+ for (Option option : ConfigOptions.getConfigsOptions("driver", job.getDriverConfig().getConfigs())) {
+ this.addOption(option);
+ }
+ for (Option option : ConfigOptions.getConfigsOptions("to", job.getJobConfig(Direction.TO).getConfigs())) {
+ this.addOption(option);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java
deleted file mode 100644
index 87c0776..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/JobDynamicFormOptions.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MJob;
-
-/**
- * Automatically create dynamic options for jobs.
- */
-@SuppressWarnings("serial")
-public class JobDynamicFormOptions extends DynamicFormOptions<MJob> {
-
- @SuppressWarnings("static-access")
- @Override
- public void prepareOptions(MJob job) {
- this.addOption(OptionBuilder
- .withLongOpt("name")
- .hasArg()
- .create());
- for (Option option : FormOptions.getFormsOptions("connector", job.getConnectorPart(Direction.FROM).getForms())) {
- this.addOption(option);
- }
- for (Option option : FormOptions.getFormsOptions("framework", job.getFrameworkPart().getForms())) {
- this.addOption(option);
- }
- for (Option option : FormOptions.getFormsOptions("connector", job.getConnectorPart(Direction.TO).getForms())) {
- this.addOption(option);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java
new file mode 100644
index 0000000..ef279f7
--- /dev/null
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicConfigOptions.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.shell.utils;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.model.MLink;
+
+/**
+ * Automatically create dynamic options for connections.
+ */
+@SuppressWarnings("serial")
+public class LinkDynamicConfigOptions extends DynamicConfigOptions<MLink> {
+
+ @SuppressWarnings("static-access")
+ @Override
+ public void prepareOptions(MLink link) {
+ this.addOption(OptionBuilder.withLongOpt("name").hasArg().create());
+ for (Option option : ConfigOptions.getConfigsOptions("link", link.getConnectorLinkConfig()
+ .getConfigs())) {
+ this.addOption(option);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java b/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
deleted file mode 100644
index 2952be2..0000000
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/LinkDynamicFormOptions.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.shell.utils;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.sqoop.model.MLink;
-
-/**
- * Automatically create dynamic options for connections.
- */
-@SuppressWarnings("serial")
-public class LinkDynamicFormOptions extends DynamicFormOptions<MLink> {
-
- @SuppressWarnings("static-access")
- @Override
- public void prepareOptions(MLink link) {
- this.addOption(OptionBuilder.withLongOpt("name").hasArg().create());
- for (Option option : FormOptions.getFormsOptions("connector", link.getConnectorPart()
- .getForms())) {
- this.addOption(option);
- }
- for (Option option : FormOptions.getFormsOptions("framework", link.getFrameworkPart()
- .getForms())) {
- this.addOption(option);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/shell/src/main/resources/shell-resource.properties
----------------------------------------------------------------------
diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties
index c0f86f7..e2381dd 100644
--- a/shell/src/main/resources/shell-resource.properties
+++ b/shell/src/main/resources/shell-resource.properties
@@ -16,7 +16,7 @@
# Client Resources in default language (english)
############################
-# Security Form
+# Security Config
#############################
object-name.label = Name
object-name.help = Non unique name of the entity to help you remember \
@@ -200,22 +200,22 @@ table.header.status = Status
table.header.date = Last Update Date
table.header.enabled = Enabled
-#Form displayer resources
-formdisplayer.link = link
-formdisplayer.job = Job
-formdisplayer.form = form
-formdisplayer.name = Name
-formdisplayer.label = Label
-formdisplayer.help = Help
-formdisplayer.input = Input
-formdisplayer.type = Type
-formdisplayer.sensitive = Sensitive
-formdisplayer.size = Size
-formdisplayer.possible_values = Possible values
-formdisplayer.unsupported_datatype = Unsupported data type
-formdisplayer.input_sensitive = This input is sensitive
-
-formdisplayer.warning_message = There were warnings while create or update, but saved successfully.
+#Config displayer resources
+config.displayer.link = link
+config.displayer.job = Job
+config.displayer.config = config
+config.displayer.name = Name
+config.displayer.label = Label
+config.displayer.help = Help
+config.displayer.input = Input
+config.displayer.type = Type
+config.displayer.sensitive = Sensitive
+config.displayer.size = Size
+config.displayer.possible_values = Possible values
+config.displayer.unsupported_datatype = Unsupported data type
+config.displayer.input_sensitive = This input is sensitive
+
+config.displayer.warning_message = There were warnings while create or update, but saved successfully.
submission.submission_detail = Submission details
submission.job_id = Job ID
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java b/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
index e00b404..879e428 100644
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/RepositoryUpgrader.java
@@ -18,8 +18,8 @@
*/
package org.apache.sqoop.connector.spi;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MLinkConfig;
/**
* Repository represents the sqoop entity store. Sqoop entities include
@@ -29,23 +29,23 @@ import org.apache.sqoop.model.MJobForms;
public abstract class RepositoryUpgrader {
/**
- * Upgrade the original connection and fill into the upgradeTarget. Note
- * that any metadata already in {@code upgradeTarget} maybe overwritten.
- * @param original - original connection metadata as in the repository
+ * Upgrade the original link config and fill into the upgradeTarget. Note
+ * that any data already in {@code upgradeTarget} maybe overwritten.
+ * @param original - original link config as in the repository
* @param upgradeTarget - the instance that will be filled in with the
- * upgraded metadata.
+ * upgraded link config.
*/
- public abstract void upgrade(MConnectionForms original,
- MConnectionForms upgradeTarget);
+ public abstract void upgrade(MLinkConfig original, MLinkConfig upgradeTarget);
/**
- * Upgrade the original job and fill into the upgradeTarget. Note
- * that any metadata already in {@code upgradeTarget} maybe overwritten.
- * This method must be called only after the connection metadata has
+ * Upgrade the original job config and fill into the upgradeTarget. Note
+ * that any config data already in {@code upgradeTarget} maybe overwritten.
+ * This method must be called only after the link config has
* already been upgraded.
- * @param original - original connection metadata as in the repository
+ * @param original - original job config as in the repository
* @param upgradeTarget - the instance that will be filled in with the
- * upgraded metadata.
+ * upgraded job config.
+ * NOTE(VB): This api will be revisited to accomodate from and to job config update
*/
- public abstract void upgrade(MJobForms original, MJobForms upgradeTarget);
+ public abstract void upgrade(MConfigList original, MConfigList upgradeTarget);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
index 7b78ba0..849f091 100644
--- a/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
+++ b/spi/src/main/java/org/apache/sqoop/connector/spi/SqoopConnector.java
@@ -58,12 +58,12 @@ public abstract class SqoopConnector {
}
/**
- * @return Get connection configuration class
+ * @return Get link configuration group class
*/
public abstract Class getLinkConfigurationClass();
/**
- * @return Get job configuration class for given type or null if not supported
+ * @return Get job configuration group class per direction type or null if not supported
*/
public abstract Class getJobConfigurationClass(Direction jobType);
@@ -79,12 +79,10 @@ public abstract class SqoopConnector {
/**
* Returns validation object that Sqoop can use to validate user
- * supplied forms before accepting them. This object will be used both for
- * connection and job forms.
- *
+ * supplied configs before accepting them. This object will be used both link and job configs
* @return Validator object
*/
- public abstract Validator getValidator();
+ public abstract Validator getConfigValidator();
/**
* Returns an {@linkplain RepositoryUpgrader} object that can upgrade the
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
index f965bdd..a133106 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Destroyer.java
@@ -28,7 +28,7 @@ public abstract class Destroyer<LinkConfiguration, JobConfiguration> {
*
* @param context Destroyer context
* @param linkConfiguration link configuration object
- * @param jobConfiguration Job configuration object
+ * @param jobConfiguration job configuration object
*/
public abstract void destroy(DestroyerContext context,
LinkConfiguration linkConfiguration,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java b/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
index 34f02c9..d6c186d 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Extractor.java
@@ -27,8 +27,8 @@ public abstract class Extractor<LinkConfiguration, JobConfiguration, Partition>
* Extract data from source and pass them into the Sqoop.
*
* @param context Extractor context object
- * @param linkConfiguration link configuration
- * @param jobConfiguration Job configuration
+ * @param linkConfiguration link configuration object
+ * @param jobConfiguration job configuration object
* @param partition Partition that this extract should work on
*/
public abstract void extract(ExtractorContext context,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
index c9ae540..5c48fc3 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java
@@ -34,28 +34,28 @@ public abstract class Initializer<LinkConfiguration, JobConfiguration> {
* promoted to all other part of the workflow automatically.
*
* @param context Initializer context object
- * @param linkConfiguration Connector's link configuration object
- * @param jobConfiguration Connector's job configuration object
+ * @param linkConfiguration link configuration object
+ * @param jobConfiguration job configuration object
*/
- public abstract void initialize(InitializerContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration);
+ public abstract void initialize(InitializerContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration);
- /**
- * Return list of all jars that this particular connector needs to operate
- * on following job. This method will be called after running initialize
- * method.
- *
- * @return
- */
- public List<String> getJars(InitializerContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration) {
- return new LinkedList<String>();
- }
+ /**
+ * Return list of all jars that this particular connector needs to operate
+ * on following job. This method will be called after running initialize
+ * method.
+ *
+ * @return
+ */
+ public List<String> getJars(InitializerContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration) {
+ return new LinkedList<String>();
+ }
- public abstract Schema getSchema(InitializerContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration);
+ public abstract Schema getSchema(InitializerContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java b/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
index 434dcf6..cc32ada 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Loader.java
@@ -26,12 +26,12 @@ public abstract class Loader<LinkConfiguration, JobConfiguration> {
* Load data to target.
*
* @param context Loader context object
- * @param linkConfiguration link configuration
- * @param jobConfiguration Job configuration
+ * @param linkConfiguration link configuration object
+ * @param jobConfiguration job configuration object
* @throws Exception
*/
- public abstract void load(LoaderContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration) throws Exception;
+ public abstract void load(LoaderContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java b/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
index 8156762..57507df 100644
--- a/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
+++ b/spi/src/main/java/org/apache/sqoop/job/etl/Partitioner.java
@@ -31,12 +31,12 @@ public abstract class Partitioner<LinkConfiguration, JobConfiguration> {
* Each partition will be then processed in separate extractor.
*
* @param context Partitioner context object
- * @param linkConfiguration link configuration
- * @param jobConfiguration Job configuration
+ * @param linkConfiguration link configuration object
+ * @param jobConfiguration job configuration object
* @return
*/
- public abstract List<Partition> getPartitions(PartitionerContext context,
- LinkConfiguration linkConfiguration,
- JobConfiguration jobConfiguration);
+ public abstract List<Partition> getPartitions(PartitionerContext context,
+ LinkConfiguration linkConfiguration,
+ JobConfiguration jobConfiguration);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/spi/src/main/java/org/apache/sqoop/validation/Validator.java
----------------------------------------------------------------------
diff --git a/spi/src/main/java/org/apache/sqoop/validation/Validator.java b/spi/src/main/java/org/apache/sqoop/validation/Validator.java
index 894f412..2909905 100644
--- a/spi/src/main/java/org/apache/sqoop/validation/Validator.java
+++ b/spi/src/main/java/org/apache/sqoop/validation/Validator.java
@@ -29,21 +29,21 @@ public class Validator {
/**
* Validate link configuration object.
*
- * @param linkConfiguration Connection object to be validated
+ * @param linkConfiguration link config object to be validated
* @return Validation status
*/
- public Validation validateLink(Object linkConfiguration) {
- return new Validation(EmptyClass.class);
+ public ConfigValidator validateConfigForLink(Object linkConfiguration) {
+ return new ConfigValidator(EmptyClass.class);
}
/**
* Validate configuration object for job .
*
- * @param jobConfiguration Job to be validated
+ * @param jobConfiguration Job config to be validated
* @return Validation status
*/
- public Validation validateJob(Object jobConfiguration) {
- return new Validation(EmptyClass.class);
+ public ConfigValidator validateConfigForJob(Object jobConfiguration) {
+ return new ConfigValidator(EmptyClass.class);
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
index 25255ae..0c492ef 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
@@ -141,7 +141,7 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
* {@inheritDoc}
*/
@Override
- public boolean isExecutionEngineSupported(Class executionEngineClass) {
+ public boolean isExecutionEngineSupported(Class<?> executionEngineClass) {
return executionEngineClass == MapreduceExecutionEngine.class;
}
@@ -201,16 +201,16 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
try {
Job job = new Job(configuration);
- // And finally put all configuration objects to credentials cache
- ConfigurationUtils.setConnectorConnectionConfig(Direction.FROM, job, request.getConnectorLinkConfig(Direction.FROM));
- ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getConnectorJobConfig(Direction.FROM));
- ConfigurationUtils.setConnectorConnectionConfig(Direction.TO, job, request.getConnectorLinkConfig(Direction.TO));
- ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getConnectorJobConfig(Direction.TO));
+ // link configs
+ ConfigurationUtils.setConnectorLinkConfig(Direction.FROM, job, request.getConnectorLinkConfig(Direction.FROM));
+ ConfigurationUtils.setConnectorLinkConfig(Direction.TO, job, request.getConnectorLinkConfig(Direction.TO));
- ConfigurationUtils.setFrameworkConnectionConfig(Direction.FROM, job, request.getFrameworkLinkConfig(Direction.FROM));
- ConfigurationUtils.setFrameworkConnectionConfig(Direction.TO, job, request.getFrameworkLinkConfig(Direction.TO));
- ConfigurationUtils.setFrameworkJobConfig(job, request.getFrameworkJobConfig());
+ // from and to configs
+ ConfigurationUtils.setConnectorJobConfig(Direction.FROM, job, request.getJobConfig(Direction.FROM));
+ ConfigurationUtils.setConnectorJobConfig(Direction.TO, job, request.getJobConfig(Direction.TO));
+ ConfigurationUtils.setDriverConfig(job, request.getDriverConfig());
+ // @TODO(Abe): Persist TO schema.
ConfigurationUtils.setConnectorSchema(Direction.FROM, job, request.getSummary().getFromSchema());
ConfigurationUtils.setConnectorSchema(Direction.TO, job, request.getSummary().getToSchema());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
index 9ecc9da..7504e76 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
@@ -17,6 +17,12 @@
*/
package org.apache.sqoop.test.minicluster;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.conf.Configuration;
import org.codehaus.cargo.container.ContainerType;
import org.codehaus.cargo.container.InstalledLocalContainer;
@@ -25,16 +31,9 @@ import org.codehaus.cargo.container.configuration.LocalConfiguration;
import org.codehaus.cargo.container.deployable.WAR;
import org.codehaus.cargo.container.installer.Installer;
import org.codehaus.cargo.container.installer.ZipURLInstaller;
-import org.codehaus.cargo.container.property.GeneralPropertySet;
import org.codehaus.cargo.generic.DefaultContainerFactory;
import org.codehaus.cargo.generic.configuration.DefaultConfigurationFactory;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
/**
* Embedded tomcat Sqoop server mini cluster.
*
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
index 6074d36..af31769 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
@@ -17,15 +17,18 @@
*/
package org.apache.sqoop.test.testcases;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
import org.apache.log4j.Logger;
import org.apache.sqoop.client.SubmissionCallback;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.test.asserts.ProviderAsserts;
@@ -39,9 +42,6 @@ import org.apache.sqoop.validation.Status;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-
/**
* Base test case suitable for connector testing.
*
@@ -121,41 +121,50 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
}
/**
- * Fill link form based on currently active provider.
+ * Fill link config based on currently active provider.
*
* @param link MLink object to fill
*/
- protected void fillRdbmsLinkForm(MLink link) {
- MFormList forms = link.getConnectorPart();
- forms.getStringInput("link.jdbcDriver").setValue(provider.getJdbcDriver());
- forms.getStringInput("link.connectionString").setValue(provider.getConnectionUrl());
- forms.getStringInput("link.username").setValue(provider.getConnectionUsername());
- forms.getStringInput("link.password").setValue(provider.getConnectionPassword());
+ protected void fillRdbmsLinkConfig(MLink link) {
+ MConfigList configs = link.getConnectorLinkConfig();
+ configs.getStringInput("linkConfig.jdbcDriver").setValue(provider.getJdbcDriver());
+ configs.getStringInput("linkConfig.connectionString").setValue(provider.getConnectionUrl());
+ configs.getStringInput("linkConfig.username").setValue(provider.getConnectionUsername());
+ configs.getStringInput("linkConfig.password").setValue(provider.getConnectionPassword());
}
/**
- * Fill TO form with specific storage and output type.
+ * Fill TO config with specific storage and output type.
*
* @param job MJob object to fill
* @param output Output type that should be set
*/
- protected void fillToJobForm(MJob job, ToFormat output) {
- MFormList toForms = job.getConnectorPart(Direction.TO);
- toForms.getEnumInput("toJobConfig.outputFormat").setValue(output);
- toForms.getStringInput("toJobConfig.outputDirectory").setValue(getMapreduceDirectory());
+ protected void fillHdfsToConfig(MJob job, ToFormat output) {
+ MConfigList toConfig = job.getJobConfig(Direction.TO);
+ toConfig.getEnumInput("toJobConfig.outputFormat").setValue(output);
+ toConfig.getStringInput("toJobConfig.outputDirectory").setValue(getMapreduceDirectory());
}
/**
- * Fill FROM form
+ * Fill FROM config
*
* @param job MJob object to fill
*/
- protected void fillFromJobForm(MJob job) {
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.inputDirectory").setValue(getMapreduceDirectory());
+ protected void fillHdfsFromConfig(MJob job) {
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fromConfig.getStringInput("fromJobConfig.inputDirectory").setValue(getMapreduceDirectory());
}
/**
+ * Fill Driver config
+ * @param job
+ */
+ protected void fillDriverConfig(MJob job) {
+ job.getDriverConfig().getStringInput("throttlingConfig.numExtractors").setValue("3");
+ }
+
+
+ /**
* Create table cities.
*/
protected void createTableCities() {
@@ -186,7 +195,7 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
/**
* Assert row in testing table.
*
- * @param conditions Conditions in form that are expected by the database provider
+ * @param conditions Conditions in config that are expected by the database provider
* @param values Values that are expected in the table (with corresponding types)
*/
protected void assertRow(Object []conditions, Object ...values) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
index 63736ab..5e1d564 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
@@ -165,7 +165,7 @@ abstract public class TomcatTestCase {
* @throws IOException
*/
protected void assertTo(String... lines) throws IOException {
- // TODO(VB): fix this to be not directly dependent on mapreduce
+ // TODO(VB): fix this to be not directly dependent on hdfs/MR
HdfsAsserts.assertMapreduceOutput(hdfsClient, getMapreduceDirectory(), lines);
}
@@ -175,7 +175,7 @@ abstract public class TomcatTestCase {
* @param expectedFiles Expected number of files
*/
protected void assertToFiles(int expectedFiles) throws IOException {
- // TODO(VB): fix this to be not directly dependent on mapreduce
+ // TODO(VB): fix this to be not directly dependent on hdfs/MR
HdfsAsserts.assertMapreduceOutputFiles(hdfsClient, getMapreduceDirectory(), expectedFiles);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
index 5e1abc1..b1b3b16 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
@@ -19,8 +19,9 @@ package org.apache.sqoop.integration.connector.jdbc.generic;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
import org.junit.Test;
@@ -43,7 +44,7 @@ public class FromHDFSToRDBMSTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -53,10 +54,16 @@ public class FromHDFSToRDBMSTest extends ConnectorTestCase {
// Job creation
MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId());
- // Connector values
- MFormList toForms = job.getConnectorPart(Direction.TO);
- toForms.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- fillFromJobForm(job);
+ // set hdfs "FROM" config for the job, since the connector test case base class only has utilities for hdfs!
+ fillHdfsFromConfig(job);
+
+ // set the rdms "TO" config here
+ MConfigList toConfig = job.getJobConfig(Direction.TO);
+ toConfig.getStringInput("toJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+
+ // driver config
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
saveJob(job);
executeJob(job);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
index 2dc0613..36f7443 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
@@ -17,17 +17,18 @@
*/
package org.apache.sqoop.integration.connector.jdbc.generic;
+import static org.junit.Assert.assertTrue;
+
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.junit.Test;
-import static org.junit.Assert.assertTrue;
-
/**
* Import simple table with various configurations.
*/
@@ -39,7 +40,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsConnection);
+ fillRdbmsLinkConfig(rdbmsConnection);
saveLink(rdbmsConnection);
// HDFS link
@@ -49,11 +50,17 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
// Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());
- // Connector values
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
- fillToJobForm(job, ToFormat.TEXT_FILE);
+ // srt rdbms "FROM" config
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
+
+ // fill the hdfs "TO" config
+ fillHdfsToConfig(job, ToFormat.TEXT_FILE);
+ // driver config
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
+
saveJob(job);
executeJob(job);
@@ -76,7 +83,7 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -87,11 +94,11 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase {
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
// Connector values
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
- forms.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country"));
- fillToJobForm(job, ToFormat.TEXT_FILE);
+ MConfigList configs = job.getJobConfig(Direction.FROM);
+ configs.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ configs.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
+ configs.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country"));
+ fillHdfsToConfig(job, ToFormat.TEXT_FILE);
saveJob(job);
MSubmission submission = getClient().startSubmission(job.getPersistenceId());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
index 729f95e..824a51d 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
@@ -19,8 +19,9 @@ package org.apache.sqoop.integration.connector.jdbc.generic;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
+import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.apache.sqoop.test.utils.ParametrizedUtils;
@@ -59,8 +60,8 @@ public class PartitionerTest extends ConnectorTestCase {
}
private String partitionColumn;
- private int extractors;
private int maxOutputFiles;
+ private int extractors;
public PartitionerTest(String partitionColumn, int expectedOutputFiles, int extractors) {
this.partitionColumn = partitionColumn;
@@ -74,7 +75,7 @@ public class PartitionerTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -84,13 +85,17 @@ public class PartitionerTest extends ConnectorTestCase {
// Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
- // Connector values
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn));
- fillToJobForm(job, ToFormat.TEXT_FILE);
- forms = job.getFrameworkPart();
- forms.getIntegerInput("throttling.extractors").setValue(extractors);
+ // set the rdbms "FROM" config
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName(partitionColumn));
+ // fill hdfs "TO" config
+ fillHdfsToConfig(job, ToFormat.TEXT_FILE);
+
+ // set driver config
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(extractors);
+
saveJob(job);
executeJob(job);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
index 562a6a6..f42fa32 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
@@ -17,16 +17,16 @@
*/
package org.apache.sqoop.integration.connector.jdbc.generic;
+import static org.junit.Assert.assertEquals;
+
import org.apache.sqoop.common.Direction;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.test.data.Cities;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
/**
*
*/
@@ -46,7 +46,7 @@ public class TableStagedRDBMSTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -57,13 +57,19 @@ public class TableStagedRDBMSTest extends ConnectorTestCase {
MJob job = getClient().createJob(hdfsLink.getPersistenceId(),
rdbmsLink.getPersistenceId());
- // Connector values
- MFormList forms = job.getConnectorPart(Direction.TO);
- forms.getStringInput("toJobConfig.tableName").setValue(
+ // fill HDFS "FROM" config
+ fillHdfsFromConfig(job);
+
+ // fill rdbms "TO" config here
+ MConfigList configs = job.getJobConfig(Direction.TO);
+ configs.getStringInput("toJobConfig.tableName").setValue(
provider.escapeTableName(getTableName()));
- forms.getStringInput("toJobConfig.stageTableName").setValue(
+ configs.getStringInput("toJobConfig.stageTableName").setValue(
provider.escapeTableName(stageTableName));
- fillFromJobForm(job);
+ // driver config
+ MConfigList driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
+
saveJob(job);
executeJob(job);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
index 507ac53..055bc3d 100644
--- a/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
@@ -23,7 +23,7 @@ import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.hdfs.configuration.ToFormat;
import org.apache.sqoop.driver.DriverError;
import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.junit.runner.RunWith;
@@ -67,7 +67,7 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase {
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkForm(rdbmsLink);
+ fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
@@ -77,15 +77,21 @@ public class SubmissionWithDisabledModelObjectsTest extends ConnectorTestCase {
// Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
- // Connector values
- MFormList forms = job.getConnectorPart(Direction.FROM);
- forms.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
- forms.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
- // Framework values
- fillToJobForm(job, ToFormat.TEXT_FILE);
+ // rdms "FROM" config
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fromConfig.getStringInput("fromJobConfig.tableName").setValue(provider.escapeTableName(getTableName()));
+ fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id"));
+
+ // hdfs "TO" config
+ fillHdfsToConfig(job, ToFormat.TEXT_FILE);
+
+ // driver config
+ MConfigList driverConfig = job.getDriverConfig();
+ //driverConfig.getIntegerInput("throttlingConfig.extractors").setValue(3);
+
saveJob(job);
- // Disable model entities as per parametrized run
+ // Disable model entities as per parameterized run
getClient().enableLink(rdbmsLink.getPersistenceId(), enabledLink);
getClient().enableJob(job.getPersistenceId(), enabledJob);