You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ma...@apache.org on 2016/11/23 20:51:55 UTC
sqoop git commit: SQOOP-2349: Add command line option for setting
transaction isolation levels for metadata queries
Repository: sqoop
Updated Branches:
refs/heads/trunk 3fba26419 -> b0b7b4779
SQOOP-2349: Add command line option for setting
transaction isolation levels for metadata queries
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b0b7b477
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b0b7b477
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b0b7b477
Branch: refs/heads/trunk
Commit: b0b7b4779c13101930f402fe3c7d832ea0a503f2
Parents: 3fba264
Author: Attila Szabo <ma...@apache.org>
Authored: Wed Nov 23 21:45:03 2016 +0100
Committer: Attila Szabo <ma...@apache.org>
Committed: Wed Nov 23 21:45:03 2016 +0100
----------------------------------------------------------------------
src/java/org/apache/sqoop/SqoopOptions.java | 28 ++++++++---
.../org/apache/sqoop/manager/SqlManager.java | 2 +-
.../org/apache/sqoop/tool/BaseSqoopTool.java | 20 ++++++++
.../sqoop/tool/JDBCTransactionLevels.java | 39 ++++++++++++++
.../org/apache/sqoop/tool/TestImportTool.java | 53 ++++++++++++++++++++
5 files changed, 134 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 30b4705..ef26f16 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Field;
import java.net.URLDecoder;
+import java.sql.Connection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -340,6 +341,8 @@ public class SqoopOptions implements Cloneable {
// explicit split by cols
@StoredAsProperty("reset.onemapper") private boolean autoResetToOneMapper;
+ @StoredAsProperty("sqlconnection.metadata.transaction.isolation.level") private int metadataTransactionIsolationLevel;
+
// These next two fields are not serialized to the metastore.
// If this SqoopOptions is created by reading a saved job, these will
// be populated by the JobStorage to facilitate updating the same
@@ -1033,9 +1036,12 @@ public class SqoopOptions implements Cloneable {
// Relaxed isolation will not enabled by default which is the behavior
// of sqoop until now.
this.relaxedIsolation = false;
-
+
// set default mainframe data set type to partitioned data set
this.mainframeInputDatasetType = MainframeConfiguration.MAINFRAME_INPUT_DATASET_TYPE_PARTITIONED;
+
+ // set default transaction isolation level to TRANSACTION_READ_UNCOMMITED
+ this.metadataTransactionIsolationLevel = Connection.TRANSACTION_READ_COMMITTED;
}
/**
@@ -2659,11 +2665,19 @@ public class SqoopOptions implements Cloneable {
this.customToolOptions = customToolOptions;
}
- public String getToolName() {
- return this.toolName;
- }
+ public String getToolName() {
+ return this.toolName;
+ }
- public void setToolName(String toolName) {
- this.toolName = toolName;
- }
+ public void setToolName(String toolName) {
+ this.toolName = toolName;
+ }
+
+ public int getMetadataTransactionIsolationLevel() {
+ return this.metadataTransactionIsolationLevel;
+ }
+
+ public void setMetadataTransactionIsolationLevel(int transactionIsolationLevel) {
+ this.metadataTransactionIsolationLevel = transactionIsolationLevel;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/java/org/apache/sqoop/manager/SqlManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/SqlManager.java b/src/java/org/apache/sqoop/manager/SqlManager.java
index 768507b..808e330 100644
--- a/src/java/org/apache/sqoop/manager/SqlManager.java
+++ b/src/java/org/apache/sqoop/manager/SqlManager.java
@@ -918,7 +918,7 @@ public abstract class SqlManager
* (queries executed by the ConnManager itself).
*/
protected int getMetadataIsolationLevel() {
- return Connection.TRANSACTION_READ_COMMITTED;
+ return options.getMetadataTransactionIsolationLevel();
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 235f67a..3ed0f77 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -58,6 +58,8 @@ import com.cloudera.sqoop.metastore.JobData;
*/
public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
+ public static final String METADATA_TRANSACTION_ISOLATION_LEVEL = "metadata-transaction-isolation-level";
+
public static final Log LOG = LogFactory.getLog(
BaseSqoopTool.class.getName());
@@ -478,6 +480,13 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
.hasArg()
.withArgName("rootdir")
.create());
+ commonOpts.addOption(OptionBuilder
+ .withDescription("Defines the transaction isolation level for metadata queries. "
+ + "For more details check java.sql.Connection javadoc or the JDBC specificaiton")
+ .withLongOpt(METADATA_TRANSACTION_ISOLATION_LEVEL)
+ .hasArg()
+ .withArgName("isolationlevel")
+ .create());
// relax isolation requirements
commonOpts.addOption(OptionBuilder
.withDescription("Use read-uncommitted isolation for imports")
@@ -1029,6 +1038,17 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
if (in.hasOption(RELAXED_ISOLATION)) {
out.setRelaxedIsolation(true);
}
+
+ if (in.hasOption(METADATA_TRANSACTION_ISOLATION_LEVEL)) {
+ String transactionLevel = in.getOptionValue(METADATA_TRANSACTION_ISOLATION_LEVEL);
+ try {
+ out.setMetadataTransactionIsolationLevel(JDBCTransactionLevels.valueOf(transactionLevel).getTransactionIsolationLevelValue());
+ } catch (IllegalArgumentException e) {
+ throw new RuntimeException("Only transaction isolation levels defined by "
+ + "java.sql.Connection class are supported. Check the "
+ + "java.sql.Connection javadocs for more details", e);
+ }
+ }
}
private void applyCredentialsOptions(CommandLine in, SqoopOptions out)
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java b/src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java
new file mode 100644
index 0000000..c141497
--- /dev/null
+++ b/src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import java.sql.Connection;
+
+enum JDBCTransactionLevels {
+ TRANSACTION_NONE(Connection.TRANSACTION_NONE),
+ TRANSACTION_READ_UNCOMMITTED(Connection.TRANSACTION_READ_UNCOMMITTED),
+ TRANSACTION_READ_COMMITTED(Connection.TRANSACTION_READ_COMMITTED),
+ TRANSACTION_REPEATABLE_READ(Connection.TRANSACTION_REPEATABLE_READ),
+ TRANSACTION_SERIALIZABLE(Connection.TRANSACTION_SERIALIZABLE);
+
+ private final int transactionLevelValue;
+
+ private JDBCTransactionLevels(int transactionLevelValue) {
+ this.transactionLevelValue = transactionLevelValue;
+ }
+
+ public int getTransactionIsolationLevelValue() {
+ return transactionLevelValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/b0b7b477/src/test/org/apache/sqoop/tool/TestImportTool.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/tool/TestImportTool.java b/src/test/org/apache/sqoop/tool/TestImportTool.java
new file mode 100644
index 0000000..4136e9f
--- /dev/null
+++ b/src/test/org/apache/sqoop/tool/TestImportTool.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.tool;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.sql.Connection;
+
+import org.apache.sqoop.SqoopOptions;
+import org.junit.experimental.theories.DataPoints;
+import org.junit.experimental.theories.Theories;
+import org.junit.experimental.theories.Theory;
+import org.junit.runner.RunWith;
+
+@RunWith(Theories.class)
+public class TestImportTool {
+ @DataPoints
+ public static final Object[][] TRANSACTION_ISOLATION_LEVEL_NAMES_AND_VALUES = {
+ {"TRANSACTION_NONE", Connection.TRANSACTION_NONE},
+ {"TRANSACTION_READ_COMMITTED",Connection.TRANSACTION_READ_COMMITTED},
+ {"TRANSACTION_READ_UNCOMMITTED",Connection.TRANSACTION_READ_UNCOMMITTED},
+ {"TRANSACTION_REPEATABLE_READ",Connection.TRANSACTION_REPEATABLE_READ},
+ {"TRANSACTION_SERIALIZABLE",Connection.TRANSACTION_SERIALIZABLE}
+ };
+
+ @Theory
+ public void esnureTransactionIsolationLevelsAreMappedToTheRightValues(Object[] values)
+ throws Exception {
+ ImportTool importTool = new ImportTool();
+ String[] args = { "--" + BaseSqoopTool.METADATA_TRANSACTION_ISOLATION_LEVEL, values[0].toString() };
+ SqoopOptions options = importTool.parseArguments(args, null, null, true);
+ assertThat(options.getMetadataTransactionIsolationLevel(), is(equalTo(values[1])));
+ }
+
+}