You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2013/04/05 20:08:30 UTC
git commit: SQOOP-916: Add an abort validation handler
Updated Branches:
refs/heads/trunk 75bc21b16 -> ef093399b
SQOOP-916: Add an abort validation handler
(Venkatesh Seetharam via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ef093399
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ef093399
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ef093399
Branch: refs/heads/trunk
Commit: ef093399bf01fed4a7c17caebd05676f1e566b8a
Parents: 75bc21b
Author: Jarek Jarcec Cecho <ja...@cloudera.com>
Authored: Fri Apr 5 11:07:50 2013 -0700
Committer: Jarek Jarcec Cecho <ja...@cloudera.com>
Committed: Fri Apr 5 11:07:50 2013 -0700
----------------------------------------------------------------------
src/docs/user/validation.txt | 4 +-
src/java/org/apache/sqoop/SqoopOptions.java | 4 +-
.../sqoop/validation/AbortOnFailureHandler.java | 55 +++++++
.../sqoop/validation/LogOnFailureHandler.java | 5 +-
.../apache/sqoop/validation/RowCountValidator.java | 2 +-
.../sqoop/validation/ValidationException.java | 4 +-
.../validation/AbortOnFailureHandlerTest.java | 62 ++++++++
.../AbsoluteValidationThresholdTest.java | 38 +++++
.../validation/RowCountValidatorImportTest.java | 123 +++++++++++++--
9 files changed, 275 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/docs/user/validation.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/validation.txt b/src/docs/user/validation.txt
index f28b420..282cfd6 100644
--- a/src/docs/user/validation.txt
+++ b/src/docs/user/validation.txt
@@ -90,7 +90,7 @@ described below.
Description: Responsible for handling failures, must implement
org.apache.sqoop.validation.ValidationFailureHandler
Supported values: The value has to be a fully qualified class name.
- Default value: org.apache.sqoop.validation.LogOnFailureHandler
+ Default value: org.apache.sqoop.validation.AbortOnFailureHandler
Limitations
@@ -132,5 +132,5 @@ $ sqoop import --connect jdbc:mysql://db.foo.com/corp --table EMPLOYEES \
--validation-threshold \
org.apache.sqoop.validation.AbsoluteValidationThreshold \
--validation-failurehandler \
- org.apache.sqoop.validation.LogOnFailureHandler
+ org.apache.sqoop.validation.AbortOnFailureHandler
----
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 906542b..f423c0f 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -41,8 +41,8 @@ import com.cloudera.sqoop.util.RandomHash;
import com.cloudera.sqoop.util.StoredAsProperty;
import org.apache.sqoop.util.CredentialsUtil;
import org.apache.sqoop.util.LoggingUtils;
+import org.apache.sqoop.validation.AbortOnFailureHandler;
import org.apache.sqoop.validation.AbsoluteValidationThreshold;
-import org.apache.sqoop.validation.LogOnFailureHandler;
import org.apache.sqoop.validation.RowCountValidator;
/**
@@ -870,7 +870,7 @@ public class SqoopOptions implements Cloneable {
this.isValidationEnabled = false; // validation is disabled by default
this.validatorClass = RowCountValidator.class;
this.validationThresholdClass = AbsoluteValidationThreshold.class;
- this.validationFailureHandlerClass = LogOnFailureHandler.class;
+ this.validationFailureHandlerClass = AbortOnFailureHandler.class;
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java b/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java
new file mode 100644
index 0000000..32f4fd9
--- /dev/null
+++ b/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java
@@ -0,0 +1,55 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+/**
+ * A specific implementation of ValidationFailureHandler that aborts the
+ * processing by throwing an exception with failure message and the reason.
+ *
+ * This is used as the default handler unless overridden in configuration.
+ */
+public class AbortOnFailureHandler implements ValidationFailureHandler {
+
+ static final ValidationFailureHandler INSTANCE = new AbortOnFailureHandler();
+
+ /**
+ * Method that handles the validation failure.
+ *
+ * @param validationContext validation context
+ * @return if failure was handled or not
+ * @throws ValidationException
+ */
+ @Override
+ public boolean handle(ValidationContext validationContext)
+ throws ValidationException {
+
+ StringBuilder messageBuffer = new StringBuilder();
+ messageBuffer.append("Validation failed by ");
+ messageBuffer.append(validationContext.getMessage());
+ messageBuffer.append(". Reason: ").append(validationContext.getReason());
+ messageBuffer.append(", Row Count at Source: ");
+ messageBuffer.append(validationContext.getSourceRowCount());
+ messageBuffer.append(", Row Count at Target: ");
+ messageBuffer.append(validationContext.getTargetRowCount());
+
+ throw new ValidationException(messageBuffer.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java b/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java
index 3ded6ad..3c20136 100644
--- a/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java
+++ b/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java
@@ -23,9 +23,10 @@ import org.apache.commons.logging.LogFactory;
/**
* A specific implementation of ValidationFailureHandler that logs the failure
- * message and the reason with the configured logger.
+ * message and the reason with the configured logger. A note of caution in
+ * using this since it fails silently by logging the failure to a log file.
*
- * This is used as the default handler unless overridden in configuration.
+ * This is mostly used for testing purposes since this fails silently.
*/
public class LogOnFailureHandler implements ValidationFailureHandler {
private static final Log LOG =
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/validation/RowCountValidator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/validation/RowCountValidator.java b/src/java/org/apache/sqoop/validation/RowCountValidator.java
index 2896192..ca21232 100644
--- a/src/java/org/apache/sqoop/validation/RowCountValidator.java
+++ b/src/java/org/apache/sqoop/validation/RowCountValidator.java
@@ -37,7 +37,7 @@ public class RowCountValidator implements Validator {
public boolean validate(ValidationContext context)
throws ValidationException {
return validate(context,
- AbsoluteValidationThreshold.INSTANCE, LogOnFailureHandler.INSTANCE);
+ AbsoluteValidationThreshold.INSTANCE, AbortOnFailureHandler.INSTANCE);
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/validation/ValidationException.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/validation/ValidationException.java b/src/java/org/apache/sqoop/validation/ValidationException.java
index 0c10241..118f1ca 100644
--- a/src/java/org/apache/sqoop/validation/ValidationException.java
+++ b/src/java/org/apache/sqoop/validation/ValidationException.java
@@ -24,8 +24,8 @@ package org.apache.sqoop.validation;
*/
public class ValidationException extends Exception {
- public ValidationException(String s, Throwable throwable) {
- super(s, throwable);
+ public ValidationException(String message) {
+ super(message);
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java b/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java
new file mode 100644
index 0000000..f38164c
--- /dev/null
+++ b/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+import com.cloudera.sqoop.SqoopOptions;
+import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Tests for AbortOnFailureHandler.
+ */
+public class AbortOnFailureHandlerTest extends TestCase {
+
+ public void testAbortOnFailureHandlerIsDefaultOption() {
+ assertEquals(AbortOnFailureHandler.class,
+ new SqoopOptions(new Configuration()).getValidationFailureHandlerClass());
+ }
+
+ /**
+ * Positive case.
+ */
+ public void testAbortOnFailureHandlerAborting() {
+ try {
+ Validator validator = new RowCountValidator();
+ validator.validate(new ValidationContext(100, 90));
+ fail("AbortOnFailureHandler should have thrown an exception");
+ } catch (ValidationException e) {
+ assertEquals("Validation failed by RowCountValidator. "
+ + "Reason: The expected counter value was 100 but the actual value "
+ + "was 90, Row Count at Source: 100, Row Count at Target: 90",
+ e.getMessage());
+ }
+ }
+
+ /**
+ * Negative case.
+ */
+ public void testAbortOnFailureHandlerNotAborting() {
+ try {
+ Validator validator = new RowCountValidator();
+ validator.validate(new ValidationContext(100, 100));
+ } catch (ValidationException e) {
+ fail("AbortOnFailureHandler should NOT have thrown an exception");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java b/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java
new file mode 100644
index 0000000..9ac5074
--- /dev/null
+++ b/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+import junit.framework.TestCase;
+
+/**
+ * Tests for AbsoluteValidationThreshold.
+ */
+public class AbsoluteValidationThresholdTest extends TestCase {
+
+ /**
+ * Test the implementation for AbsoluteValidationThreshold.
+ * Both arguments should be same else fail.
+ */
+ public void testAbsoluteValidationThreshold() {
+ ValidationThreshold validationThreshold = new AbsoluteValidationThreshold();
+ assertTrue(validationThreshold.compare(100, 100));
+ assertFalse(validationThreshold.compare(100, 90));
+ assertFalse(validationThreshold.compare(90, 100));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java b/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java
index a53e281..035d3b1 100644
--- a/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java
+++ b/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java
@@ -18,8 +18,10 @@
package org.apache.sqoop.validation;
+import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.tool.ImportTool;
import java.util.ArrayList;
import java.util.Collections;
@@ -37,22 +39,11 @@ public class RowCountValidatorImportTest extends ImportJobTestCase {
}
/**
- * Test the implementation for AbsoluteValidationThreshold.
- * Both arguments should be same else fail.
- */
- public void testAbsoluteValidationThreshold() {
- ValidationThreshold validationThreshold = new AbsoluteValidationThreshold();
- assertTrue(validationThreshold.compare(100, 100));
- assertFalse(validationThreshold.compare(100, 90));
- assertFalse(validationThreshold.compare(90, 100));
- }
-
- /**
- * Test if teh --validate flag actually made it through the options.
+ * Test if the --validate flag actually made it through the options.
*
* @throws Exception
*/
- public void testValidateOptionIsEnabled() throws Exception {
+ public void testValidateOptionIsEnabledInCLI() throws Exception {
String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
String[] insertVals = {"1", "'Bob'", "'sales'"};
@@ -68,6 +59,112 @@ public class RowCountValidatorImportTest extends ImportJobTestCase {
}
}
+ public void testValidationOptionsParsedCorrectly() throws Exception {
+ String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
+ String[] insertVals = {"1", "'Bob'", "'sales'"};
+
+ try {
+ createTableWithColTypes(types, insertVals);
+
+ String[] args = getArgv(true, null, getConf());
+ ArrayList<String> argsList = new ArrayList<String>();
+ argsList.add("--validator");
+ argsList.add("org.apache.sqoop.validation.RowCountValidator");
+ argsList.add("--validation-threshold");
+ argsList.add("org.apache.sqoop.validation.AbsoluteValidationThreshold");
+ argsList.add("--validation-failurehandler");
+ argsList.add("org.apache.sqoop.validation.AbortOnFailureHandler");
+ Collections.addAll(argsList, args);
+
+ assertTrue("Validate option missing.", argsList.contains("--validate"));
+ assertTrue("Validator option missing.", argsList.contains("--validator"));
+
+ String[] optionArgs = toStringArray(argsList);
+
+ SqoopOptions validationOptions = new ImportTool().parseArguments(
+ optionArgs, getConf(), getSqoopOptions(getConf()), true);
+ assertEquals(RowCountValidator.class,
+ validationOptions.getValidatorClass());
+ assertEquals(AbsoluteValidationThreshold.class,
+ validationOptions.getValidationThresholdClass());
+ assertEquals(AbortOnFailureHandler.class,
+ validationOptions.getValidationFailureHandlerClass());
+ } catch (Exception e) {
+ fail("The validation options are passed correctly: " + e.getMessage());
+ } finally {
+ dropTableIfExists(getTableName());
+ }
+ }
+
+ public void testInvalidValidationOptions() throws Exception {
+ String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
+ String[] insertVals = {"1", "'Bob'", "'sales'"};
+
+ try {
+ createTableWithColTypes(types, insertVals);
+
+ String[] args = getArgv(true, null, getConf());
+ ArrayList<String> argsList = new ArrayList<String>();
+ argsList.add("--validator");
+ argsList.add("org.apache.sqoop.validation.NullValidator");
+ argsList.add("--validation-threshold");
+ argsList.add("org.apache.sqoop.validation.NullValidationThreshold");
+ argsList.add("--validation-failurehandler");
+ argsList.add("org.apache.sqoop.validation.NullFailureHandler");
+ Collections.addAll(argsList, args);
+
+ String[] optionArgs = toStringArray(argsList);
+
+ new ImportTool().parseArguments(optionArgs, getConf(),
+ getSqoopOptions(getConf()), true);
+ fail("The validation options are incorrect and must throw an exception");
+ } catch (Exception e) {
+ System.out.println("e.getMessage() = " + e.getMessage());
+ System.out.println("e.getClass() = " + e.getClass());
+ assertEquals(
+ com.cloudera.sqoop.SqoopOptions.InvalidOptionsException.class,
+ e.getClass());
+ } finally {
+ dropTableIfExists(getTableName());
+ }
+ }
+
+ private String[] toStringArray(ArrayList<String> argsList) {
+ String[] optionArgs = new String[argsList.size()];
+ for (int i = 0; i < argsList.size(); i++) {
+ optionArgs[i] = argsList.get(i);
+ }
+ return optionArgs;
+ }
+
+ /**
+ * Negative case where the row counts do NOT match.
+ */
+ public void testValidatorWithDifferentRowCounts() {
+ try {
+ Validator validator = new RowCountValidator();
+ validator.validate(new ValidationContext(100, 90));
+ fail("FailureHandler should have thrown an exception");
+ } catch (ValidationException e) {
+ assertEquals("Validation failed by RowCountValidator. "
+ + "Reason: The expected counter value was 100 but the actual value "
+ + "was 90, Row Count at Source: 100, Row Count at Target: 90",
+ e.getMessage());
+ }
+ }
+
+ /**
+ * Positive case where the row counts match.
+ */
+ public void testValidatorWithMatchingRowCounts() {
+ try {
+ Validator validator = new RowCountValidator();
+ validator.validate(new ValidationContext(100, 100));
+ } catch (ValidationException e) {
+ fail("FailureHandler should NOT have thrown an exception");
+ }
+ }
+
/**
* Test the validation for a sample import, positive case.
*