You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ka...@apache.org on 2013/06/23 03:06:24 UTC
git commit: SQOOP-943: Sqoop2: Repository wide configuration of
immutable disk structures
Updated Branches:
refs/heads/sqoop2 fd4be5f5e -> 681914c85
SQOOP-943: Sqoop2: Repository wide configuration of immutable disk structures
(Jarek Jarcec Cecho via Kate Ting)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/681914c8
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/681914c8
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/681914c8
Branch: refs/heads/sqoop2
Commit: 681914c851cb275f1d803ffe057c2a7a53436330
Parents: fd4be5f
Author: Kate Ting <ka...@apache.org>
Authored: Sat Jun 22 21:00:42 2013 -0400
Committer: Kate Ting <ka...@apache.org>
Committed: Sat Jun 22 21:00:42 2013 -0400
----------------------------------------------------------------------
.../apache/sqoop/repository/JdbcRepository.java | 34 ++++++++++++++++++--
.../sqoop/repository/JdbcRepositoryContext.java | 9 ------
.../repository/JdbcRepositoryProvider.java | 10 +-----
.../repository/RepoConfigurationConstants.java | 15 ++++-----
.../org/apache/sqoop/repository/Repository.java | 20 ++++++++++++
.../sqoop/repository/RepositoryError.java | 3 ++
.../sqoop/repository/RepositoryManager.java | 9 ++++++
dist/src/main/server/conf/sqoop.properties | 4 +--
.../test/minicluster/SqoopMiniCluster.java | 2 +-
9 files changed, 75 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
index 135c2d2..aa1aa8d 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
@@ -31,8 +31,7 @@ import org.apache.sqoop.model.MSubmission;
public class JdbcRepository extends Repository {
- private static final Logger LOG =
- Logger.getLogger(JdbcRepository.class);
+ private static final Logger LOG = Logger.getLogger(JdbcRepository.class);
private final JdbcRepositoryHandler handler;
private final JdbcRepositoryContext repoContext;
@@ -124,6 +123,37 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
+ public void createOrUpdateInternals() {
+ doWithConnection(new DoWithConnection() {
+ @Override
+ public Object doIt(Connection conn) throws Exception {
+ if (!handler.schemaExists()) {
+ LOG.info("Creating repository schema objects");
+ handler.createSchema();
+ }
+
+ return null;
+ }
+ });
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean haveSuitableInternals() {
+ return (Boolean) doWithConnection(new DoWithConnection() {
+ @Override
+ public Object doIt(Connection conn) throws Exception {
+ return handler.schemaExists();
+ }
+ });
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public MConnector registerConnector(final MConnector mConnector) {
return (MConnector) doWithConnection(new DoWithConnection() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java
index 8989fb6..0a8139a 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java
@@ -34,7 +34,6 @@ public final class JdbcRepositoryContext {
private final MapContext context;
private final String handlerClassName;
- private final boolean createSchema;
private final String connectionUrl;
private final String driverClassName;
private final Properties connectionProperties;
@@ -55,9 +54,6 @@ public final class JdbcRepositoryContext {
RepoConfigurationConstants.SYSCFG_REPO_JDBC_HANDLER);
}
- createSchema = context.getBoolean(
- RepoConfigurationConstants.SYSCFG_REPO_JDBC_CREATE_SCHEMA, false);
-
connectionUrl = context.getString(
RepoConfigurationConstants.SYSCFG_REPO_JDBC_URL);
@@ -132,7 +128,6 @@ public final class JdbcRepositoryContext {
if (LOG.isInfoEnabled()) {
StringBuilder sb = new StringBuilder("[repo-ctx] ");
sb.append("handler=").append(handlerClassName).append(", ");
- sb.append("create-schema=").append(createSchema).append(", ");
sb.append("conn-url=").append(connectionUrl).append(", ");
sb.append("driver=").append(driverClassName).append(", ");
sb.append("user=").append(jdbcUserName).append(", ");
@@ -189,10 +184,6 @@ public final class JdbcRepositoryContext {
return driverClassName;
}
- public boolean shouldCreateSchema() {
- return createSchema;
- }
-
public JdbcTransactionIsolation getTransactionIsolation() {
return transactionIsolation;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java
index 3339c59..1fd092a 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java
@@ -40,8 +40,7 @@ import org.apache.sqoop.utils.ClassUtils;
public class JdbcRepositoryProvider implements RepositoryProvider {
- private static final Logger LOG =
- Logger.getLogger(JdbcRepositoryProvider.class);
+ private static final Logger LOG = Logger.getLogger(JdbcRepositoryProvider.class);
private JdbcRepositoryContext repoContext;
@@ -156,13 +155,6 @@ public class JdbcRepositoryProvider implements RepositoryProvider {
handler.initialize(repoContext);
- if (repoContext.shouldCreateSchema()) {
- if (!handler.schemaExists()) {
- LOG.info("Creating repository schema objects");
- handler.createSchema();
- }
- }
-
repository = new JdbcRepository(handler, repoContext);
LOG.info("JdbcRepositoryProvider initialized");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
index 8939f8a..12cb93e 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
@@ -43,6 +43,13 @@ public final class RepoConfigurationConstants {
+ "provider";
/**
+ * Boolean property defining whether repository is allowed to make changes
+ * on disk structures (schema in databases, changing file format, ...).
+ */
+ public static final String SYSCFG_REPO_SCHEMA_IMMUTABLE=
+ PREFIX_REPO_CONFIG + "schema.immutable";
+
+ /**
* Class name for the JDBC repository handler specified by:
* <tt>org.apache.sqoop.repository.jdbc.handler</tt>.
*/
@@ -50,14 +57,6 @@ public final class RepoConfigurationConstants {
+ "jdbc.handler";
/**
- * Indicates if the repository should create the schema objects as necessary,
- * specified as a boolean value for the key:
- * <tt>org.apache.sqoop.repository.jdbc.create.schema</tt>
- */
- public static final String SYSCFG_REPO_JDBC_CREATE_SCHEMA =
- PREFIX_REPO_CONFIG + "jdbc.create.schema";
-
- /**
* JDBC connection URL specified by:
* <tt>org.apache.sqoop.repository.jdbc.url</tt>
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/Repository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java
index 1ba947a..a2911c5 100644
--- a/core/src/main/java/org/apache/sqoop/repository/Repository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java
@@ -56,6 +56,26 @@ public abstract class Repository {
public abstract RepositoryTransaction getTransaction();
/**
+ * Create or update disk data structures.
+ *
+ * This method will be called only if Sqoop server is enabled with changing
+ * repository on disk structures. Repository should not change its disk structures
+ * outside of this method. This method must be no-op in case that the structures
+ * do not need any maintenance.
+ */
+ public abstract void createOrUpdateInternals();
+
+ /**
+ * Return true if internal repository structures exists and are suitable for use.
+ *
+ * This method should return false in case that the structures do exists, but
+ * are not suitable for use or if they requires upgrade.
+ *
+ * @return Boolean values if internal structures are suitable for use
+ */
+ public abstract boolean haveSuitableInternals();
+
+ /**
* Registers given connector in the repository and return registered
* variant. This method might return an exception in case that metadata for
* given connector are already registered with different structure.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
index 4cae7ba..ab5e5a1 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
@@ -29,6 +29,9 @@ public enum RepositoryError implements ErrorCode {
/** The system was unable to find or load the repository provider. */
REPO_0001("Invalid repository provider specified"),
+ /** Repository on disk structures are not suitable for use */
+ REPO_0002("Repository structures are not initialized or requires upgrade"),
+
// JDBC Repository Errors: Prefix JDBCREP
/** An unknown error has occurred. */
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
index 955306d..a178238 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
@@ -111,6 +111,15 @@ public class RepositoryManager {
provider.initialize(context);
+ if(!context.getBoolean(RepoConfigurationConstants.SYSCFG_REPO_SCHEMA_IMMUTABLE, true)) {
+ LOG.info("Creating or upgrading on disk structures if necessary");
+ provider.getRepository().createOrUpdateInternals();
+ }
+
+ if(!provider.getRepository().haveSuitableInternals()) {
+ throw new SqoopException(RepositoryError.REPO_0002);
+ }
+
LOG.info("Repository initialized: OK");
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/dist/src/main/server/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties
index 5131aad..07b148e 100755
--- a/dist/src/main/server/conf/sqoop.properties
+++ b/dist/src/main/server/conf/sqoop.properties
@@ -62,20 +62,20 @@ org.apache.sqoop.log4j.category.org.apache.derby=INFO
# is "org.apache.sqoop.repository.sysprop". Any property that
# is specified with this prefix is parsed out and set as a
# system property. For example, if the built in Derby repository
-# is being used, the sysprop prefixed proeprties can be used
+# is being used, the sysprop prefixed properties can be used
# to affect Derby configuration at startup time by setting
# the appropriate system properties.
#
# Repository provider
org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider
+org.apache.sqoop.repository.schema.immutable=false
# JDBC repository provider configuration
org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler
org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED
org.apache.sqoop.repository.jdbc.maximum.connections=10
org.apache.sqoop.repository.jdbc.url=jdbc:derby:@BASEDIR@/repository/db;create=true
-org.apache.sqoop.repository.jdbc.create.schema=true
org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
org.apache.sqoop.repository.jdbc.user=sa
org.apache.sqoop.repository.jdbc.password=
http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index 3620c28..9c71688 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -169,11 +169,11 @@ public abstract class SqoopMiniCluster {
Map<String, String> properties = new HashMap<String, String>();
properties.put("org.apache.sqoop.repository.provider", "org.apache.sqoop.repository.JdbcRepositoryProvider");
+ properties.put("org.apache.sqoop.repository.schema.immutable", "false");
properties.put("org.apache.sqoop.repository.jdbc.handler", "org.apache.sqoop.repository.derby.DerbyRepositoryHandler");
properties.put("org.apache.sqoop.repository.jdbc.transaction.isolation", "READ_COMMITTED");
properties.put("org.apache.sqoop.repository.jdbc.maximum.connections", "10");
properties.put("org.apache.sqoop.repository.jdbc.url=jdbc:derby:memory:myDB;create", "true");
- properties.put("org.apache.sqoop.repository.jdbc.create.schema", "true");
properties.put("org.apache.sqoop.repository.jdbc.driver", "org.apache.derby.jdbc.EmbeddedDriver");
properties.put("org.apache.sqoop.repository.jdbc.user", "sa");
properties.put("org.apache.sqoop.repository.jdbc.password", "");