You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2016/04/11 22:21:25 UTC
[2/2] sqoop git commit: SQOOP-2890: Provide tooling to encrypt
non-encrypted repository and rotate keys
SQOOP-2890: Provide tooling to encrypt non-encrypted repository and rotate keys
(Abraham Fine via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c6fc7f95
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c6fc7f95
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c6fc7f95
Branch: refs/heads/sqoop2
Commit: c6fc7f95ac0e70aa2cb8902da3e74111d7c142bf
Parents: 5b897a4
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Mon Apr 11 13:20:57 2016 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Mon Apr 11 13:20:57 2016 -0700
----------------------------------------------------------------------
.../sqoop/error/code/CommonRepositoryError.java | 7 +
.../java/org/apache/sqoop/driver/Driver.java | 13 +-
.../apache/sqoop/repository/JdbcRepository.java | 36 +-
.../sqoop/repository/JdbcRepositoryHandler.java | 21 +-
.../sqoop/repository/MasterKeyManager.java | 251 ++++++-----
.../org/apache/sqoop/repository/Repository.java | 28 +-
.../apache/sqoop/security/SecurityError.java | 6 +
.../sqoop/repository/TestMasterKeyManager.java | 13 +-
dist/src/main/conf/sqoop.properties | 4 +-
docs/src/site/sphinx/security/Encryption.rst | 218 ++++++++-
.../common/CommonRepositoryHandler.java | 136 +++++-
...RepositoryInsertUpdateDeleteSelectQuery.java | 91 +++-
.../derby/DerbySchemaCreateQuery.java | 2 +-
test/pom.xml | 5 +
.../tools/RepositoryDumpLoadToolTest.java | 47 +-
.../tools/RepositoryEncryptionToolTest.java | 444 +++++++++++++++++++
.../apache/sqoop/tools/tool/BuiltinTools.java | 1 +
.../tools/tool/RepositoryEncryptionTool.java | 145 ++++++
18 files changed, 1298 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java b/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java
index dd52c82..4d523d2 100644
--- a/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java
+++ b/common/src/main/java/org/apache/sqoop/error/code/CommonRepositoryError.java
@@ -218,6 +218,13 @@ public enum CommonRepositoryError implements ErrorCode {
COMMON_0058("Resource doesn't exist"),
COMMON_0059("Unable to retrieve master key"),
+
+ COMMON_0060("Failed to change the master key for inputs"),
+
+ COMMON_0061("Unable to delete Master Key"),
+
+ COMMON_0062("Invalid primary key after registering entity"),
+
;
private final String message;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/driver/Driver.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/Driver.java b/core/src/main/java/org/apache/sqoop/driver/Driver.java
index 2d1d73e..b6be4f1 100644
--- a/core/src/main/java/org/apache/sqoop/driver/Driver.java
+++ b/core/src/main/java/org/apache/sqoop/driver/Driver.java
@@ -114,10 +114,6 @@ public class Driver implements Reconfigurable {
}
private Driver() {
- List<MConfig> driverConfig = ConfigUtils.toConfigs(getDriverJobConfigurationClass());
- List<MValidator> mValidators = ConfigUtils.getMValidatorsFromConfigurationClass(getDriverJobConfigurationClass());
- mDriver = new MDriver(new MDriverConfig(driverConfig, mValidators), DriverBean.CURRENT_DRIVER_VERSION);
-
// Build upgrader
driverUpgrader = new DriverUpgrader();
}
@@ -130,15 +126,20 @@ public class Driver implements Reconfigurable {
public synchronized void initialize(boolean autoUpgrade) {
LOG.trace("Begin Driver initialization");
+ List<MConfig> driverConfig = ConfigUtils.toConfigs(getDriverJobConfigurationClass());
+ List<MValidator> mValidators = ConfigUtils.getMValidatorsFromConfigurationClass(getDriverJobConfigurationClass());
+
// Register driver in repository
- mDriver = RepositoryManager.getInstance().getRepository().registerDriver(mDriver, autoUpgrade);
+ mDriver = RepositoryManager.getInstance().getRepository().registerDriver(
+ new MDriver(new MDriverConfig(driverConfig, mValidators),
+ DriverBean.CURRENT_DRIVER_VERSION), autoUpgrade);
SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this));
LOG.info("Driver initialized: OK");
}
- public synchronized void destroy() {
+ public synchronized void destroy() {
LOG.trace("Begin Driver destroy");
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
index 44ab41b..b665d79 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
@@ -662,20 +662,20 @@ public class JdbcRepository extends Repository {
* {@inheritDoc}
*/
@Override
- public MMasterKey getMasterKey() {
+ public MMasterKey getMasterKey(RepositoryTransaction txn) {
return (MMasterKey) doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) throws Exception {
return handler.getMasterKey(conn);
}
- });
+ }, (JdbcRepositoryTransaction) txn);
}
/**
* {@inheritDoc}
*/
@Override
- public void createMasterKey(final MMasterKey mMasterKey) {
+ public void createMasterKey(final MMasterKey mMasterKey, RepositoryTransaction txn) {
doWithConnection(new DoWithConnection() {
@Override
public Object doIt(Connection conn) {
@@ -685,7 +685,35 @@ public class JdbcRepository extends Repository {
handler.createMasterKey(mMasterKey, conn);
return null;
}
- });
+ }, (JdbcRepositoryTransaction) txn);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void deleteMasterKey(final long masterKeyId, RepositoryTransaction txn) {
+ doWithConnection(new DoWithConnection() {
+ @Override
+ public Object doIt(Connection conn) {
+ handler.deleteMasterKey(masterKeyId, conn);
+ return null;
+ }
+ }, (JdbcRepositoryTransaction) txn);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void changeMasterKeyManager(final MasterKeyManager fromMasterKeyManager, final MasterKeyManager toMasterKeyManager, RepositoryTransaction txn) {
+ doWithConnection(new DoWithConnection() {
+ @Override
+ public Object doIt(Connection conn) {
+ handler.changeMasterKeyManager(fromMasterKeyManager, toMasterKeyManager, conn);
+ return null;
+ }
+ }, (JdbcRepositoryTransaction) txn);
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
index 94f5e6f..efd8806 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
@@ -477,9 +477,28 @@ public abstract class JdbcRepositoryHandler {
/**
* Create the master key in the database
*
- * @param mMasterKey MMasterKey representing the Master Key
+ * @param mMasterKey MMasterKey representing the Master Key, this method populates
+ * its persistence id
* @param conn Connection to the repository
*/
public abstract void createMasterKey(MMasterKey mMasterKey, Connection conn);
+ /**
+ * Delete the master key from the database
+ *
+ * @param conn Connection to the repository
+ */
+ public abstract void deleteMasterKey(long masterKeyId, Connection conn);
+
+ /**
+ * Change MasterKeyManager for inputs
+ *
+ * @param fromMasterKeyManager The master key manager that currently represents
+ * the encryption strategy for the inputs
+ * @param toMasterKeyManager The master key manager that will represent the
+ * encryption strategy for the inputs
+ * @param conn Connection to the repository
+ */
+ public abstract void changeMasterKeyManager(MasterKeyManager fromMasterKeyManager, MasterKeyManager toMasterKeyManager, Connection conn);
+
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java b/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java
index df84d54..bf251e0 100644
--- a/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java
+++ b/core/src/main/java/org/apache/sqoop/repository/MasterKeyManager.java
@@ -53,6 +53,9 @@ public class MasterKeyManager {
private int pbkdf2Rounds;
private int ivLength;
+ private RepositoryTransaction repositoryTransaction;
+ private MMasterKey mMasterKey;
+
private SecretKey masterEncryptionKey;
private SecretKey masterHmacKey;
@@ -64,7 +67,7 @@ public class MasterKeyManager {
instance = new MasterKeyManager();
}
- private MasterKeyManager() {
+ public MasterKeyManager() {
}
public static MasterKeyManager getInstance() {
@@ -79,128 +82,168 @@ public class MasterKeyManager {
initialize(true);
}
- public synchronized void initialize(boolean createMasterKey) throws SqoopException {
- // This is used for the generation of random initialization vectors and salts
- random = new SecureRandom();
+ public void initialize(boolean createMasterKey) throws SqoopException {
+ initialize(createMasterKey, false, null);
+ }
+ public void initialize(boolean createMasterKey, boolean createKeyEvenIfKeyExists, RepositoryTransaction repositoryTransactionArg) throws SqoopException {
MapContext configurationContext = SqoopConfiguration.getInstance().getContext();
if (configurationContext.getBoolean(SecurityConstants.REPO_ENCRYPTION_ENABLED, false)) {
// Grab configuration from the sqoop properties file. All of this configuration is required
// and an exception will be thrown if any of it is missing
- hmacAlgorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM);
- cipherAlgorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM);
- cipherSpec = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC);
- cipherKeySize = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE);
- ivLength = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE);
- pbkdf2Algorithm = populateStringConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM);
- pbkdf2Rounds = populateIntConfiguration(configurationContext, SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS);
-
- // The size of the hmac key can be derived from the provided HMAC algorithm
- try {
- hmacKeySizeBytes = Mac.getInstance(hmacAlgorithm).getMacLength();
- } catch (NoSuchAlgorithmException e) {
- throw new SqoopException(SecurityError.ENCRYPTION_0011, e);
- }
-
- Repository repository = RepositoryManager.getInstance().getRepository();
- String password = PasswordUtils.readPassword(configurationContext, SecurityConstants.REPO_ENCRYPTION_PASSWORD,
+ String hmacAlgorithm = populateStringConfiguration(configurationContext,
+ SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM);
+ String cipherAlgorithm = populateStringConfiguration(configurationContext,
+ SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM);
+ String cipherSpec = populateStringConfiguration(configurationContext,
+ SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC);
+ int cipherKeySize = populateIntConfiguration(configurationContext,
+ SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE);
+ int ivLength = populateIntConfiguration(configurationContext,
+ SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE);
+ String pbkdf2Algorithm = populateStringConfiguration(configurationContext,
+ SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM);
+ int pbkdf2Rounds = populateIntConfiguration(configurationContext,
+ SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS);
+
+ String password = PasswordUtils.readPassword(configurationContext,
+ SecurityConstants.REPO_ENCRYPTION_PASSWORD,
SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR);
- if (StringUtils.isEmpty(password)) {
- throw new SqoopException(SecurityError.ENCRYPTION_0008);
- }
- MMasterKey existingEncryptedMasterKey = repository.getMasterKey();
- String salt;
+ initialize(createMasterKey, hmacAlgorithm, cipherAlgorithm, cipherSpec, cipherKeySize,
+ ivLength, pbkdf2Algorithm, pbkdf2Rounds, password, createKeyEvenIfKeyExists, repositoryTransactionArg);
+ }
+ }
- if (existingEncryptedMasterKey == null) {
- // Since the master key does not exist, we can generate a random salt that we will use
- // for encryption of the Master Key
- // We will use a salt that is the same size as the encryption key
- salt = Base64.encodeBase64String(generateRandomByteArray(hmacKeySizeBytes));
- } else {
- // Since the master key already exists, we will read the salt from the repository
- salt = existingEncryptedMasterKey.getSalt();
- }
- // Derive two keys (that we will be used to encrypt and verify the master key)
- // from the configuration provided password and the salt we just read/created.
- byte[] keyBytes = getKeysFromPassword(password, salt);
- SecretKey passwordEncryptionKey = new SecretKeySpec(keyBytes, 0,
- cipherKeySize, cipherAlgorithm);
- SecretKey passwordHmacKey = new SecretKeySpec(keyBytes,
- cipherKeySize, hmacKeySizeBytes, hmacAlgorithm);
-
- byte[] masterEncryptionKeyBytes;
- byte[] masterHmacKeyBytes;
- if (existingEncryptedMasterKey == null) {
- if (createMasterKey) {
- // A master key does not exist so we must create one. We will simply
- // use two random byte arrays for the encryption and hmac components.
- // The sizes of these keys is determined by the values provided to
- // configuration.
- masterEncryptionKeyBytes = generateRandomByteArray(cipherKeySize);
- masterHmacKeyBytes = generateRandomByteArray(hmacKeySizeBytes);
-
- // The initialization vector for the encryption of the master key is
- // randomly generated.
- String iv = Base64.encodeBase64String(generateRandomByteArray(ivLength));
-
- // We append our two keys together and encrypt the resulting byte array.
- // This is the secret that all of the encryption in the repository depends upon
- byte[] secret = ArrayUtils.addAll(masterEncryptionKeyBytes, masterHmacKeyBytes);
- String encryptedSecret = encryptToString(passwordEncryptionKey, secret, iv);
-
- // We store our new master key in the repository in its encrypted form
- // along with an HMAC to verify the key when we read it, the salt needed to
- // generate keys to decrypt it, and the initialization vector used
- repository.createMasterKey(new MMasterKey(encryptedSecret, generateHmac(passwordHmacKey,
- encryptedSecret), salt, iv));
- } else {
- // If a master key does not exist and we are trying to initialize the
- // manager without allowing it to create a master key, we should fail
- throw new SqoopException(SecurityError.ENCRYPTION_0002);
- }
+ public synchronized void initialize(boolean createMasterKey, String hmacAlgorithmArg,
+ String cipherAlgorithmArg, String cipherSpecArg,
+ int cipherKeySizeArg, int ivLengthArg,
+ String pbkdf2AlgorithmArg, int pbkdf2RoundsArg,
+ String password, boolean createKeyEvenIfKeyExists, RepositoryTransaction repositoryTransactionArg) throws SqoopException {
+ hmacAlgorithm = hmacAlgorithmArg;
+ cipherAlgorithm = cipherAlgorithmArg;
+ cipherSpec = cipherSpecArg;
+ cipherKeySize = cipherKeySizeArg;
+ ivLength = ivLengthArg;
+ pbkdf2Algorithm = pbkdf2AlgorithmArg;
+ pbkdf2Rounds = pbkdf2RoundsArg;
+
+ repositoryTransaction = repositoryTransactionArg;
+
+ // This is used for the generation of random initialization vectors and salts
+ random = new SecureRandom();
+ // The size of the hmac key can be derived from the provided HMAC algorithm
+ try {
+ hmacKeySizeBytes = Mac.getInstance(hmacAlgorithm).getMacLength();
+ } catch (NoSuchAlgorithmException e) {
+ throw new SqoopException(SecurityError.ENCRYPTION_0011, e);
+ }
+
+ Repository repository = RepositoryManager.getInstance().getRepository();
+ if (StringUtils.isEmpty(password)) {
+ throw new SqoopException(SecurityError.ENCRYPTION_0008);
+ }
+
+ MMasterKey existingEncryptedMasterKey = repository.getMasterKey(repositoryTransaction);
+ String salt;
+
+ if (existingEncryptedMasterKey == null || createKeyEvenIfKeyExists) {
+ // Since the master key does not exist, we can generate a random salt that we will use
+ // for encryption of the Master Key
+ // We will use a salt that is the same size as the encryption key
+ salt = Base64.encodeBase64String(generateRandomByteArray(hmacKeySizeBytes));
+ } else {
+ // Since the master key already exists, we will read the salt from the repository
+ salt = existingEncryptedMasterKey.getSalt();
+ }
+
+ // Derive two keys (that we will be used to encrypt and verify the master key)
+ // from the configuration provided password and the salt we just read/created.
+ byte[] keyBytes = getKeysFromPassword(password, salt);
+ SecretKey passwordEncryptionKey = new SecretKeySpec(keyBytes, 0,
+ cipherKeySize, cipherAlgorithm);
+ SecretKey passwordHmacKey = new SecretKeySpec(keyBytes,
+ cipherKeySize, hmacKeySizeBytes, hmacAlgorithm);
+
+ byte[] masterEncryptionKeyBytes;
+ byte[] masterHmacKeyBytes;
+ if (existingEncryptedMasterKey == null || createKeyEvenIfKeyExists) {
+ if (createMasterKey) {
+ // A master key does not exist so we must create one. We will simply
+ // use two random byte arrays for the encryption and hmac components.
+ // The sizes of these keys is determined by the values provided to
+ // configuration.
+ masterEncryptionKeyBytes = generateRandomByteArray(cipherKeySize);
+ masterHmacKeyBytes = generateRandomByteArray(hmacKeySizeBytes);
+
+ // The initialization vector for the encryption of the master key is
+ // randomly generated.
+ String iv = Base64.encodeBase64String(generateRandomByteArray(ivLength));
+
+ // We append our two keys together and encrypt the resulting byte array.
+ // This is the secret that all of the encryption in the repository depends upon
+ byte[] secret = ArrayUtils.addAll(masterEncryptionKeyBytes, masterHmacKeyBytes);
+ String encryptedSecret = encryptToString(passwordEncryptionKey, secret, iv);
+
+ // We store our new master key in the repository in its encrypted form
+ // along with an HMAC to verify the key when we read it, the salt needed to
+ // generate keys to decrypt it, and the initialization vector used
+ mMasterKey = new MMasterKey(encryptedSecret, generateHmac(passwordHmacKey, encryptedSecret), salt, iv);
+ repository.createMasterKey(mMasterKey, repositoryTransaction);
} else {
- // A master key exists so we need to read it from the repository and
- // decrypt it.
- String iv = existingEncryptedMasterKey.getIv();
- String encryptedSecret = existingEncryptedMasterKey.getEncryptedSecret();
-
- // Before we go about decrypting the master key we should verify the hmac
- // to ensure that it has not been tampered with
- String hmac = existingEncryptedMasterKey.getHmac();
- if (!validHmac(passwordHmacKey, encryptedSecret, hmac)) {
- throw new SqoopException(SecurityError.ENCRYPTION_0001);
- }
-
- // The master key has not been tampered with, lets decrypt it using the key
- // derived from the password and the initialization vector from the repository
- byte[] decryptedKey = decryptToBytes(passwordEncryptionKey, encryptedSecret, iv);
-
- // Since the master key is stored as the concatenation of an encryption
- // key and an hmac key, we need to split it according to the sizes derived
- // from the configuration
- masterEncryptionKeyBytes = new byte[cipherKeySize];
- masterHmacKeyBytes = new byte[hmacKeySizeBytes];
- System.arraycopy(decryptedKey, 0, masterEncryptionKeyBytes, 0,
- cipherKeySize);
- System.arraycopy(decryptedKey, cipherKeySize,
- masterHmacKeyBytes, 0, hmacKeySizeBytes);
+ // If a master key does not exist and we are trying to initialize the
+ // manager without allowing it to create a master key, we should fail
+ throw new SqoopException(SecurityError.ENCRYPTION_0002);
+ }
+ } else {
+ // A master key exists so we need to read it from the repository and
+ // decrypt it.
+ mMasterKey = existingEncryptedMasterKey;
+ String iv = existingEncryptedMasterKey.getIv();
+ String encryptedSecret = existingEncryptedMasterKey.getEncryptedSecret();
+
+ // Before we go about decrypting the master key we should verify the hmac
+ // to ensure that it has not been tampered with
+ String hmac = existingEncryptedMasterKey.getHmac();
+ if (!validHmac(passwordHmacKey, encryptedSecret, hmac)) {
+ throw new SqoopException(SecurityError.ENCRYPTION_0001);
}
- // Place the master encryption and master hmac key in SecretKey objects
- // so we can use them to encrypt and decrypt data
- masterEncryptionKey = new SecretKeySpec(masterEncryptionKeyBytes, 0, cipherKeySize, cipherAlgorithm);
- masterHmacKey = new SecretKeySpec(masterHmacKeyBytes, 0, hmacKeySizeBytes, hmacAlgorithm);
+ // The master key has not been tampered with, lets decrypt it using the key
+ // derived from the password and the initialization vector from the repository
+ byte[] decryptedKey = decryptToBytes(passwordEncryptionKey, encryptedSecret, iv);
+
+ // Since the master key is stored as the concatenation of an encryption
+ // key and an hmac key, we need to split it according to the sizes derived
+ // from the configuration
+ masterEncryptionKeyBytes = new byte[cipherKeySize];
+ masterHmacKeyBytes = new byte[hmacKeySizeBytes];
+ System.arraycopy(decryptedKey, 0, masterEncryptionKeyBytes, 0,
+ cipherKeySize);
+ System.arraycopy(decryptedKey, cipherKeySize,
+ masterHmacKeyBytes, 0, hmacKeySizeBytes);
}
+
+ // Place the master encryption and master hmac key in SecretKey objects
+ // so we can use them to encrypt and decrypt data
+ masterEncryptionKey = new SecretKeySpec(masterEncryptionKeyBytes, 0, cipherKeySize, cipherAlgorithm);
+ masterHmacKey = new SecretKeySpec(masterHmacKeyBytes, 0, hmacKeySizeBytes, hmacAlgorithm);
}
public synchronized void destroy() {
hmacAlgorithm = null;
+ hmacKeySizeBytes = 0;
cipherAlgorithm = null;
+ cipherKeySize = 0;
cipherSpec = null;
pbkdf2Algorithm = null;
+ pbkdf2Rounds = 0;
+ ivLength = 0;
+
+ repositoryTransaction = null;
+ mMasterKey = null;
masterEncryptionKey = null;
masterHmacKey = null;
@@ -208,6 +251,10 @@ public class MasterKeyManager {
random = null;
}
+ public void deleteMasterKeyFromRepository() {
+ RepositoryManager.getInstance().getRepository().deleteMasterKey(mMasterKey.getPersistenceId(), repositoryTransaction);
+ }
+
/**
* Returns a Base64 representation of the encrypted cleartext, using the Master Key
*
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/repository/Repository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java
index c2e3c74..ff02f3a 100644
--- a/core/src/main/java/org/apache/sqoop/repository/Repository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java
@@ -33,7 +33,6 @@ import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.driver.DriverUpgrader;
import org.apache.sqoop.json.DriverBean;
import org.apache.sqoop.model.ConfigUtils;
-import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriver;
@@ -332,19 +331,40 @@ public abstract class Repository {
/**
* Get the encrypted master key from the repository
+ * There should only be one unless the repository encryption tool is in use
*
* @return The encrypted master key, null if no master key exists
*/
- public abstract MMasterKey getMasterKey();
+ public abstract MMasterKey getMasterKey(RepositoryTransaction tx);
/**
* Create the encrypted master key in the repository
*
- * @param mMasterKey The encrypted master key
+ * @param mMasterKey The encrypted master key. Its persistenceId will be populated
*/
- public abstract void createMasterKey(MMasterKey mMasterKey);
+ public abstract void createMasterKey(MMasterKey mMasterKey, RepositoryTransaction tx);
+ /**
+ * Delete the master key record for the given id
+ *
+ * @param masterKeyId id of the master key to delete
+ * @param tx Transaction to perform the change within
+ */
+ public abstract void deleteMasterKey(long masterKeyId, RepositoryTransaction tx);
+
+ /**
+ * Change MasterKeyManager for inputs
+ *
+ * @param fromMasterKeyManager The master key manager that currently represents
+ * the encryption strategy for the inputs.
+ * null if the inputs are not currently encrypted
+ * @param toMasterKeyManager The master key manager that will represent the
+ * encryption strategy for the inputs.
+ * null if we would like to decrypt the inputs
+ * @param tx Transaction to perform the change within
+ */
+ public abstract void changeMasterKeyManager(MasterKeyManager fromMasterKeyManager, MasterKeyManager toMasterKeyManager, RepositoryTransaction tx);
/*********************Configurable Upgrade APIs ******************************/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/main/java/org/apache/sqoop/security/SecurityError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/security/SecurityError.java b/core/src/main/java/org/apache/sqoop/security/SecurityError.java
index 2ba849c..fc7b6b2 100644
--- a/core/src/main/java/org/apache/sqoop/security/SecurityError.java
+++ b/core/src/main/java/org/apache/sqoop/security/SecurityError.java
@@ -102,6 +102,12 @@ public enum SecurityError implements ErrorCode {
/** The HMAC algorithm specified in configuration could not be found */
ENCRYPTION_0011("HMAC algorithm not found"),
+ /** An error occured when running the repository encryption tool */
+ ENCRYPTION_0012("Failed to run repository encryption tool"),
+
+ /** An error occured when running the repository encryption tool */
+ ENCRYPTION_0013("Repository is encrypted, need configuration to decrypt"),
+
;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java b/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java
index f9579bf..9e368a7 100644
--- a/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java
+++ b/core/src/test/java/org/apache/sqoop/repository/TestMasterKeyManager.java
@@ -81,7 +81,7 @@ public class TestMasterKeyManager {
jdbcRepoMock = mock(JdbcRepository.class);
- when(jdbcRepoMock.getMasterKey()).thenReturn(null);
+ when(jdbcRepoMock.getMasterKey(null)).thenReturn(null);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
masterKeyManager = MasterKeyManager.getInstance();
@@ -106,8 +106,7 @@ public class TestMasterKeyManager {
ArgumentCaptor<MMasterKey> mMasterKeyArgumentCaptor = ArgumentCaptor
.forClass(MMasterKey.class);
- verify(jdbcRepoMock, times(1)).createMasterKey(mMasterKeyArgumentCaptor
- .capture());
+ verify(jdbcRepoMock, times(1)).createMasterKey(mMasterKeyArgumentCaptor.capture(), any(RepositoryTransaction.class));
// Encrypt something with that master key
String secret = "imasecret";
@@ -120,12 +119,12 @@ public class TestMasterKeyManager {
// Create a new MasterKeyManager instance with existing master key
// coming from the "db"
jdbcRepoMock = mock(JdbcRepository.class);
- when(jdbcRepoMock.getMasterKey()).thenReturn(mMasterKeyArgumentCaptor
+ when(jdbcRepoMock.getMasterKey(null)).thenReturn(mMasterKeyArgumentCaptor
.getValue());
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
masterKeyManager.initialize();
- verify(jdbcRepoMock, times(1)).getMasterKey();
+ verify(jdbcRepoMock, times(1)).getMasterKey(null);
// Try to decrypt
assertEquals(masterKeyManager.decryptWithMasterKey(encrypted, iv, masterKeyManager.generateHmacWithMasterHmacKey(encrypted)), secret);
@@ -135,7 +134,7 @@ public class TestMasterKeyManager {
public void testInitializeWithKeyCreationWithoutExistingKey() {
masterKeyManager.initialize();
- verify(jdbcRepoMock, times(1)).createMasterKey(any(MMasterKey.class));
+ verify(jdbcRepoMock, times(1)).createMasterKey(any(MMasterKey.class), any(RepositoryTransaction.class));
}
@Test(
@@ -144,7 +143,7 @@ public class TestMasterKeyManager {
)
public void testMasterKeyWithInvalidHmac() {
jdbcRepoMock = mock(JdbcRepository.class);
- when(jdbcRepoMock.getMasterKey()).thenReturn(new MMasterKey(
+ when(jdbcRepoMock.getMasterKey(null)).thenReturn(new MMasterKey(
Base64.encodeBase64String(generateRandomByteArray(CIPHER_KEY_SIZE_BYTES)),
Base64.encodeBase64String(generateRandomByteArray(HMAC_KEY_SIZE_BYTES)),
Base64.encodeBase64String(generateRandomByteArray(CIPHER_KEY_SIZE_BYTES)),
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/dist/src/main/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties
index 58b60fd..d5aa0c3 100755
--- a/dist/src/main/conf/sqoop.properties
+++ b/dist/src/main/conf/sqoop.properties
@@ -197,9 +197,9 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce
#org.apache.sqoop.security.repo_encryption.password_generator=
#org.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256
#org.apache.sqoop.security.repo_encryption.cipher_algorithm=AES
-#org.apache.sqoop.security.repo_encryption.cipher_key_size=128
+#org.apache.sqoop.security.repo_encryption.cipher_key_size=16
#org.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding
-#org.apache.sqoop.security.repo_encryption.initialization_vector_size=128
+#org.apache.sqoop.security.repo_encryption.initialization_vector_size=16
#org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1
#org.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/docs/src/site/sphinx/security/Encryption.rst
----------------------------------------------------------------------
diff --git a/docs/src/site/sphinx/security/Encryption.rst b/docs/src/site/sphinx/security/Encryption.rst
index 6471a07..45ffc17 100644
--- a/docs/src/site/sphinx/security/Encryption.rst
+++ b/docs/src/site/sphinx/security/Encryption.rst
@@ -14,9 +14,9 @@
limitations under the License.
-==========
-Encryption
-==========
+===========
+API TLS/SSL
+===========
Sqoop 2 offers an HTTP REST-like API as the mechanism by which clients can
communicate with the Sqoop 2 server. The Sqoop 2 server and the Sqoop 2 shell
@@ -73,7 +73,7 @@ is configured.
org.apache.sqoop.security.tls.keymanager_password=echo keymanagerpassword
Client/Shell Configuration
---------------------------
+==========================
When using TLS on the Sqoop 2 server, especially with a self-signed certificate,
it may be useful to specify a truststore for the client/shell to use.
@@ -99,4 +99,212 @@ You may also use a password generator.
::
sqoop:000> set truststore --truststore /Users/abefine/keystore/node2.truststore --truststore-password-generator "echo changeme"
- Truststore set successfully
\ No newline at end of file
+ Truststore set successfully
+
+=====================
+Repository Encryption
+=====================
+
+Sqoop 2 uses a database to store metadata about the various data sources it talks to, we call this database the repository.
+
+The repository can store passwords and other pieces of information that are security sensitive, within the context of Sqoop
+2, this information is referred to as sensitive inputs. Which inputs are considered sensitive is determined by the connector.
+
+We support encrypting sensitive inputs in the repository using a provided password or password generator. Sqoop 2 uses the
+provided password and the provided key generation algorithm (such as PBKDF2) to generate a key to encrypt sensitive inputs
+and another hmac key to verify their integrity.
+
+Only the sensitive inputs are encrypted. If an input is not defined as sensitive by the connector, it is NOT encrypted.
+
+Server Configuration
+=====================
+
+Note: This configuration will allow a new Sqoop instance to encrypt information or read from an already encrypted repository.
+It will not encrypt sensitive inputs in an existing repository. Please see below for instructions on how to encrypt an existing repository.
+
+First, repository encryption must be enabled.
+::
+
+ org.apache.sqoop.security.repo_encryption.enabled=true
+
+Then we configure the password:
+
+::
+
+ org.apache.sqoop.security.repo_encryption.password=supersecret
+
+Or the password generator:
+
+::
+
+ org.apache.sqoop.security.repo_encryption.password_generator=echo supersecret
+
+The plaintext password is always given preference to the password generator if both are present.
+
+Then we can configure the HMAC algorithm. Please find the list of possibilities here:
+`Standard Algorithm Name Documentation - Mac <http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#Mac>`_
+We can store digests with up to 1024 bits.
+
+::
+
+ org.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256
+
+Then we configure the cipher algorithm. Possibilities can be found here:
+`Standard Algorithm Name Documentation - Cipher <http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#Cipher>`_
+
+::
+
+ org.apache.sqoop.security.repo_encryption.cipher_algorithm=AES
+
+Then we configure the key size for the cipher in bytes. We can store up to 1024 bit keys.
+
+::
+
+ org.apache.sqoop.security.repo_encryption.cipher_key_size=16
+
+Next we need to specify the cipher transformation. The options for this field are listed here:
+`Cipher (Java Platform SE 7) <http://docs.oracle.com/javase/7/docs/api/javax/crypto/Cipher.html>`_
+
+::
+
+ org.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding
+
+The size of the initialization vector to use in bytes. We support up to 1024 bit initialization vectors.
+
+::
+
+ org.apache.sqoop.security.repo_encryption.initialization_vector_size=16
+
+Next we need to specfy the algorithm for secret key generation. Please refer to:
+`Standard Algorithm Name Documentation - SecretKeyFactory <http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#SecretKeyFactory>`_
+
+::
+
+ org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1
+
+Finally specify the number of rounds/iterations for the generation of a key from a password.
+
+::
+
+ org.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000
+
+Repository Encryption Tool
+==========================
+
+Sometimes we may want to change the password that is used to encrypt our data, generate a new key for our existing password,
+encrypt an existing unencrypted repository, or decrypt an existing encrypting repository. Sqoop 2 provides the
+Repository Encryption Tool to allow us to do this.
+
+Before using the tool it is important to shut down the Sqoop 2 server.
+
+All changes that the tool makes occur in a single transaction with the repository, which should prevent leaving the
+repository in a bad state.
+
+The Repository Encryption Tool is very simple, it uses the exact same configuration specified above (with the exception
+of ``useConf``). Configuration prefixed with a "-F" represents the existing repository state, configuration prefixed with
+a "-T" represents the desired repository state. If one of these configuration sets is left out that means unencrypted.
+
+Changing the Password
+---------------------
+
+In order to change the password, we need to specify the current configuration with the existing password and the desired
+configuration with the new password. It looks like this:
+
+::
+
+ sqoop.sh tool repositoryencryption \
+ -Forg.apache.sqoop.security.repo_encryption.password=old_password \
+ -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
+ -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
+ -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
+ -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 \
+ -Torg.apache.sqoop.security.repo_encryption.password=new_password \
+ -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
+ -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
+ -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
+ -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000
+
+Generate a New Key for the Existing Password
+--------------------------------------------
+
+Just like with the previous scenario you could copy the same configuration twice like this:
+
+::
+
+ sqoop.sh tool repositoryencryption \
+ -Forg.apache.sqoop.security.repo_encryption.password=password \
+ -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
+ -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
+ -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
+ -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 \
+ -Torg.apache.sqoop.security.repo_encryption.password=password \
+ -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
+ -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
+ -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
+ -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000
+
+But we do have a shortcut to make this easier:
+
+::
+
+ sqoop.sh tool repositoryencryption -FuseConf -TuseConf
+
+The ``useConf`` option will read whatever configuration is already in the configured sqoop properties file and apply it
+for the specified direction.
+
+Encrypting an Existing Unencrypted Repository
+---------------------------------------------
+
+::
+
+ sqoop.sh tool repositoryencryption \
+ -Torg.apache.sqoop.security.repo_encryption.password=password \
+ -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
+ -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
+ -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
+ -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
+ -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000
+
+If the configuration for the encrypted repository has already been written to the sqoop properties file, one can simply
+execute:
+
+::
+
+ sqoop.sh tool repositoryencryption -TuseConf
+
+
+Decrypting an Existing Encrypted Repository
+-------------------------------------------
+
+::
+
+ sqoop.sh tool repositoryencryption \
+ -Forg.apache.sqoop.security.repo_encryption.password=password \
+ -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
+ -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
+ -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
+ -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
+ -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000
+
+If the configuration for the encrypted repository has not yet been removed from the sqoop properties file, one can simply
+execute:
+
+::
+
+ sqoop.sh tool repositoryencryption -FuseConf
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
index a9168ae..4cae811 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
@@ -1164,12 +1164,15 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
return null;
}
- String aesKey = rs.getString(1);
- String hmac = rs.getString(2);
- String salt = rs.getString(3);
- String iv = rs.getString(4);
+ String aesKey = rs.getString(2);
+ String hmac = rs.getString(3);
+ String salt = rs.getString(4);
+ String iv = rs.getString(5);
- return new MMasterKey(aesKey, hmac, salt, iv);
+ MMasterKey mMasterKey = new MMasterKey(aesKey, hmac, salt, iv);
+ mMasterKey.setPersistenceId(rs.getLong(1));
+
+ return mMasterKey;
}
} catch (SQLException ex) {
logException(ex);
@@ -1183,8 +1186,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
@Override
public void createMasterKey(MMasterKey mMasterKey, Connection conn) {
int result;
- try (PreparedStatement preparedStatement = conn.prepareStatement(crudQueries.getStmtInsertSqMasterKey(),
- Statement.RETURN_GENERATED_KEYS)) {
+ try (PreparedStatement preparedStatement = conn.prepareStatement(crudQueries.getStmtInsertSqMasterKey(), Statement.RETURN_GENERATED_KEYS)) {
preparedStatement.setString(1, mMasterKey.getEncryptedSecret());
preparedStatement.setString(2, mMasterKey.getHmac());
preparedStatement.setString(3, mMasterKey.getSalt());
@@ -1195,12 +1197,60 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
throw new SqoopException(CommonRepositoryError.COMMON_0009,
Integer.toString(result));
}
+
+ long masterKeyId = -1;
+ try(ResultSet primaryKeyResultSet = preparedStatement.getGeneratedKeys()) {
+ if (primaryKeyResultSet.next()) {
+ masterKeyId = primaryKeyResultSet.getLong(1);
+ }
+ }
+
+ if (masterKeyId < 0) {
+ throw new SqoopException(CommonRepositoryError.COMMON_0062);
+ }
+
+ mMasterKey.setPersistenceId(masterKeyId);
+
} catch (SQLException ex) {
logException(ex, mMasterKey);
throw new SqoopException(CommonRepositoryError.COMMON_0031, ex);
}
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void deleteMasterKey(long masterKeyId, Connection conn) {
+ try (PreparedStatement deleteMasterKeyPreparedStatement = conn.prepareStatement(crudQueries.getDeleteSqMasterKey())) {
+ deleteMasterKeyPreparedStatement.setLong(1, masterKeyId);
+ deleteMasterKeyPreparedStatement.executeUpdate();
+ } catch (SQLException ex) {
+ logException(ex);
+ throw new SqoopException(CommonRepositoryError.COMMON_0061, ex);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void changeMasterKeyManager(MasterKeyManager fromMasterKeyManager, MasterKeyManager toMasterKeyManager, Connection conn) {
+ try (
+ PreparedStatement fetchLinkInputs = conn.prepareStatement(crudQueries.getStmtFetchLinkInput());
+ PreparedStatement updateLinkInput = conn.prepareStatement(crudQueries.getUpdateLinkInput());
+ PreparedStatement fetchJobInputs = conn.prepareStatement(crudQueries.getStmtFetchJobInput());
+ PreparedStatement updateJobInput = conn.prepareStatement(crudQueries.getUpdateJobInput())
+ ) {
+ transitionInputs(fromMasterKeyManager, toMasterKeyManager, fetchLinkInputs, updateLinkInput);
+ transitionInputs(fromMasterKeyManager, toMasterKeyManager, fetchJobInputs, updateJobInput);
+ } catch (SQLException exception) {
+ logException(exception);
+ throw new SqoopException(CommonRepositoryError.COMMON_0060, exception);
+ }
+ }
+
private void insertConnectorDirection(Long connectorId, Direction direction, Connection conn)
throws SQLException {
try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtInsertSqConnectorDirections())) {
@@ -1646,7 +1696,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
try (ResultSet rsConnection = stmt.executeQuery();
PreparedStatement connectorConfigFetchStatement = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable());
- PreparedStatement connectorConfigInputStatement = conn.prepareStatement(crudQueries.getStmtFetchLinkInput());) {
+ PreparedStatement connectorConfigInputStatement = conn.prepareStatement(crudQueries.getStmtFetchLinkInputByJob());) {
while(rsConnection.next()) {
long id = rsConnection.getLong(1);
String name = rsConnection.getString(2);
@@ -1691,7 +1741,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
try (ResultSet rsConnection = stmt.executeQuery();
PreparedStatement configStmt = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfiguration());
- PreparedStatement inputStmt = conn.prepareStatement(crudQueries.getStmtFetchLinkInput());
+ PreparedStatement inputStmt = conn.prepareStatement(crudQueries.getStmtFetchLinkInputByJob());
) {
while(rsConnection.next()) {
long id = rsConnection.getLong(1);
@@ -1734,7 +1784,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
PreparedStatement fromConfigFetchStmt = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable());
PreparedStatement toConfigFetchStmt = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable());
PreparedStatement driverConfigfetchStmt = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable());
- PreparedStatement jobInputFetchStmt = conn.prepareStatement(crudQueries.getStmtFetchJobInput());) {
+ PreparedStatement jobInputFetchStmt = conn.prepareStatement(crudQueries.getStmtFetchJobInputByJob());) {
// Note: Job does not hold a explicit reference to the driver since every
// job has the same driver
@@ -1813,9 +1863,9 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
(crudQueries.getStmtSelectConfigForConfigurable());
PreparedStatement configStmt = conn.prepareStatement(crudQueries
.getStmtSelectConfigForConfiguration());
- PreparedStatement jobInputFetchStmt = conn.prepareStatement(crudQueries.getStmtFetchJobInput());
+ PreparedStatement jobInputFetchStmt = conn.prepareStatement(crudQueries.getStmtFetchJobInputByJob());
PreparedStatement inputStmt = conn.prepareStatement(crudQueries
- .getStmtFetchLinkInput())
+ .getStmtFetchLinkInputByJob())
) {
// Note: Job does not hold a explicit reference to the driver since every
@@ -2125,7 +2175,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
// get the overrides value from the SQ_INPUT_RELATION table
String overrides = getOverrides(inputId, conn);
String inputEnumValues = rsetInput.getString(9);
- String value = readInputValue(rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13));
+ String value = readInputValue(MasterKeyManager.getInstance(), rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13));
MInputType mit = MInputType.valueOf(inputType);
MInput input = null;
switch (mit) {
@@ -2222,6 +2272,58 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
}
}
+ private void transitionInputs(MasterKeyManager fromMasterKeyManager, MasterKeyManager toMasterKeyManager,
+ PreparedStatement selectInputsStatement, PreparedStatement updateInputsStatement) throws SQLException {
+ try (ResultSet inputs = selectInputsStatement.executeQuery()) {
+ while (inputs.next()) {
+ long inputId = inputs.getLong(1);
+ boolean encrypted = inputs.getBoolean(11);
+ boolean sensitive = inputs.getBoolean(6);
+
+ if (encrypted) {
+ assert(fromMasterKeyManager != null);
+ // We need to decrypt the input first
+ String encryptedValue = inputs.getString(10);
+ String iv = inputs.getString(12);
+ String hmac = inputs.getString(13);
+
+ String plainTextValue = readInputValue(fromMasterKeyManager, encryptedValue, encrypted, iv, hmac);
+
+ if (toMasterKeyManager != null && sensitive) {
+ // We need to encrypt the input
+ String newIv = toMasterKeyManager.generateRandomIv();
+ String encryptedInput = toMasterKeyManager.encryptWithMasterKey(plainTextValue, newIv);
+
+ updateInputsStatement.setString(1, encryptedInput);
+ updateInputsStatement.setBoolean(2, true);
+ updateInputsStatement.setString(3, newIv);
+ updateInputsStatement.setString(4, toMasterKeyManager.generateHmacWithMasterHmacKey(encryptedInput));
+ updateInputsStatement.setLong(5, inputId);
+ } else {
+ // Store the plaintext
+ updateInputsStatement.setString(1, plainTextValue);
+ updateInputsStatement.setBoolean(2, false);
+ updateInputsStatement.setNull(3, Types.VARCHAR);
+ updateInputsStatement.setNull(4, Types.VARCHAR);
+ updateInputsStatement.setLong(5, inputId);
+ }
+ updateInputsStatement.executeUpdate();
+ } else if (toMasterKeyManager != null && sensitive) {
+ // We need to encrypt the input
+ String plainTextValue = inputs.getString(10);
+ String newIv = toMasterKeyManager.generateRandomIv();
+ String encryptedInput = toMasterKeyManager.encryptWithMasterKey(plainTextValue, newIv);
+
+ updateInputsStatement.setString(1, encryptedInput);
+ updateInputsStatement.setBoolean(2, true);
+ updateInputsStatement.setString(3, newIv);
+ updateInputsStatement.setString(4, toMasterKeyManager.generateHmacWithMasterHmacKey(encryptedInput));
+ updateInputsStatement.setLong(5, inputId);
+ updateInputsStatement.executeUpdate();
+ }
+ }
+ }
+ }
private void loadInputsForConfigs(MConfigList mConfigList, PreparedStatement configStmt, PreparedStatement inputStmt) throws SQLException {
for (MConfig mConfig : mConfigList.getConfigs()) {
@@ -2237,7 +2339,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
while (inputResults.next()) {
long inputId = inputResults.getLong(1);
String inputName = inputResults.getString(2);
- String value = readInputValue(inputResults.getString(10), inputResults.getBoolean(11), inputResults.getString(12), inputResults.getString(13));
+ String value = readInputValue(MasterKeyManager.getInstance(), inputResults.getString(10), inputResults.getBoolean(11), inputResults.getString(12), inputResults.getString(13));
if (mConfig.getName().equals(configName) && mConfig.getInputNames().contains(inputName)) {
MInput mInput = mConfig.getInput(inputName);
mInput.setPersistenceId(inputId);
@@ -2262,9 +2364,9 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
* @param hmac HMAC for tamper resistance
* @return The input value
*/
- private String readInputValue(String possiblyEncryptedValue, boolean encrypted, String iv, String hmac) throws SqoopException {
+ private String readInputValue(MasterKeyManager masterKeyManager, String possiblyEncryptedValue, boolean encrypted, String iv, String hmac) throws SqoopException {
if (encrypted) {
- return MasterKeyManager.getInstance().decryptWithMasterKey(possiblyEncryptedValue, iv, hmac);
+ return masterKeyManager.decryptWithMasterKey(possiblyEncryptedValue, iv, hmac);
} else {
return possiblyEncryptedValue;
}
@@ -2318,7 +2420,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
// get the overrides value from the SQ_INPUT_RELATION table
String overrides = getOverrides(inputId, conn);
String inputEnumValues = rsetInput.getString(9);
- String value = readInputValue(rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13));
+ String value = readInputValue(MasterKeyManager.getInstance(), rsetInput.getString(10), rsetInput.getBoolean(11), rsetInput.getString(12), rsetInput.getString(13));
MInputType mit = MInputType.valueOf(inputType);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
index d527290..9118722 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
@@ -215,7 +215,7 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
* *******LINK INPUT TABLE *************
*/
//DML: Get inputs and values for a given link
- private static final String STMT_FETCH_LINK_INPUT =
+ private static final String STMT_FETCH_LINK_INPUT_BY_JOB =
"SELECT "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", "
@@ -237,11 +237,38 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?"
+ " ORDER BY " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX);
+ private static final String STMT_FETCH_LINK_INPUT =
+ "SELECT "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ","
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC)
+ + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
+ + " RIGHT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT);
+
+ private static final String UPDATE_LINK_INPUT =
+ "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME) + " SET "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + " = ?, "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + " = ?, "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + " = ?, "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " = ? "
+ + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + " = ?";
+
/**
* *******JOB INPUT TABLE *************
*/
//DML: Fetch inputs and values for a given job
- private static final String STMT_FETCH_JOB_INPUT =
+ private static final String STMT_FETCH_JOB_INPUT_BY_JOB =
"SELECT "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", "
@@ -263,6 +290,35 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?"
+ " ORDER BY " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX);
+
+ private static final String STMT_FETCH_JOB_INPUT =
+ "SELECT "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_TYPE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRMASK) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC)
+ + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
+ + " RIGHT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID)
+ + " ORDER BY " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_INDEX);
+
+ private static final String UPDATE_JOB_INPUT =
+ "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME) + " SET "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + " = ?, "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + " = ?, "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + " = ?, "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " = ? "
+ + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + " = ?";
+
/**
* *******LINK TABLE *************
*/
@@ -758,7 +814,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_CONFIGURABLE) + " = ?)";
private static final String STMT_SELECT_SQ_MASTER_KEY =
- "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", "
+ "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV)
@@ -772,6 +829,10 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV)
+ ") VALUES (?, ?, ?, ?)";
+ private static final String DELETE_SQ_MASTER_KEY =
+ "DELETE FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME)
+ + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + " = ?";
+
public String getStmtSelectSqdIdBySqdName() {
return STMT_SELECT_SQD_ID_BY_SQD_NAME;
@@ -821,14 +882,30 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
return STMT_INSERT_INTO_INPUT;
}
- public String getStmtFetchLinkInput() {
- return STMT_FETCH_LINK_INPUT;
+ public String getStmtFetchLinkInputByJob() {
+ return STMT_FETCH_LINK_INPUT_BY_JOB;
+ }
+
+ public String getUpdateLinkInput() {
+ return UPDATE_LINK_INPUT;
+ }
+
+ public String getUpdateJobInput() {
+ return UPDATE_JOB_INPUT;
}
public String getStmtFetchJobInput() {
return STMT_FETCH_JOB_INPUT;
}
+ public String getStmtFetchJobInputByJob() {
+ return STMT_FETCH_JOB_INPUT_BY_JOB;
+ }
+
+ public String getStmtFetchLinkInput() {
+ return STMT_FETCH_LINK_INPUT;
+ }
+
public String getStmtInsertLink() {
return STMT_INSERT_LINK;
}
@@ -1082,4 +1159,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
public String getStmtInsertSqMasterKey() {
return STMT_INSERT_SQ_MASTER_KEY;
}
+
+ public String getDeleteSqMasterKey() {
+ return DELETE_SQ_MASTER_KEY;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
index b4f66bb..81de886 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
@@ -193,7 +193,7 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* | SQ_LNKI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID)
* | SQ_LNKI_VALUE: LONG VARCHAR|
* | SQ_LNKI_ENCRYPTED: BOOLEAN |
- * | SQ_LNKI_IV: VARCHAR(171) |
+ * | SQ_LNKI_IV: VARCHAR(171) |
* | SQ_LNKI_HMAC: VARCHAR(171) |
* +----------------------------+
* </pre>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index 4bac683..81e599d 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -185,6 +185,11 @@ limitations under the License.
<artifactId>parquet-avro</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+
</dependencies>
<!-- Add classifier name to the JAR name -->
http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java b/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java
index c1a10bc..c17c483 100644
--- a/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java
@@ -19,6 +19,7 @@ package org.apache.sqoop.integration.tools;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
+import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.common.VersionInfo;
import org.apache.sqoop.json.JSONUtils;
import org.apache.sqoop.json.JobBean;
@@ -28,16 +29,18 @@ import org.apache.sqoop.model.*;
import org.apache.sqoop.submission.SubmissionStatus;
import org.apache.sqoop.test.infrastructure.Infrastructure;
import org.apache.sqoop.test.infrastructure.SqoopTestCase;
+import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider;
import org.apache.sqoop.test.infrastructure.providers.SqoopInfrastructureProvider;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
import org.apache.sqoop.test.utils.HdfsUtils;
import org.apache.sqoop.tools.tool.JSONConstants;
import org.apache.sqoop.tools.tool.RepositoryDumpTool;
import org.apache.sqoop.tools.tool.RepositoryLoadTool;
import org.apache.sqoop.utils.UrlSafeUtils;
import org.json.simple.JSONObject;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
+import org.testng.annotations.*;
import java.io.*;
import java.util.List;
@@ -46,14 +49,24 @@ import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
@Test(groups = "no-real-cluster")
-@Infrastructure(dependencies = {KdcInfrastructureProvider.class, SqoopInfrastructureProvider.class})
+@Infrastructure(dependencies = {HadoopInfrastructureProvider.class})
public class RepositoryDumpLoadToolTest extends SqoopTestCase {
+ private JettySqoopMiniCluster jettySqoopMiniCluster;
+ private SqoopClient client;
private String jsonFilePath;
// do the load test and insert data to repo first, then do the dump test.
- @Test(dependsOnMethods = { "testLoad" })
+ @Test
public void testDump() throws Exception {
+ // load data into repository
+ RepositoryLoadTool rlt = new RepositoryLoadTool();
+ rlt.setInTest(true);
+ rlt.runToolWithConfiguration(new String[]{"-i", jsonFilePath});
+ verifyLinks(client.getLinks());
+ verifyJobs(client.getJobs());
+ verifySubmissions(client.getSubmissions());
+
// dump the repository
RepositoryDumpTool rdt = new RepositoryDumpTool();
rdt.setInTest(true);
@@ -87,16 +100,6 @@ public class RepositoryDumpLoadToolTest extends SqoopTestCase {
}
}
- @Test
- public void testLoad() throws Exception {
- RepositoryLoadTool rlt = new RepositoryLoadTool();
- rlt.setInTest(true);
- rlt.runToolWithConfiguration(new String[]{"-i", jsonFilePath});
- verifyLinks(getClient().getLinks());
- verifyJobs(getClient().getJobs());
- verifySubmissions(getClient().getSubmissions());
- }
-
private void verifyLinks(List<MLink> links) {
for (MLink link : links) {
String linkName = link.getName();
@@ -126,8 +129,15 @@ public class RepositoryDumpLoadToolTest extends SqoopTestCase {
assertEquals(submission.getStatus(), SubmissionStatus.SUCCEEDED);
}
- // generate the json file without the license
@BeforeMethod
+ private void startCluster() throws Exception {
+ jettySqoopMiniCluster = new JettySqoopMiniCluster(HdfsUtils.joinPathFragments(super.getTemporaryPath(), this.getClass().getName()), getHadoopConf());
+ jettySqoopMiniCluster.start();
+ client = new SqoopClient(jettySqoopMiniCluster.getServerUrl());
+ }
+
+ // generate the json file without the license
+ @BeforeMethod(dependsOnMethods = { "startCluster" })
public void prepareJsonFile() throws Exception {
String testFilePath = getClass().getResource("/repoLoadToolTest.json").getPath();
jsonFilePath = HdfsUtils.joinPathFragments(getTemporaryPath(), "repoLoadTest.json");
@@ -140,7 +150,7 @@ public class RepositoryDumpLoadToolTest extends SqoopTestCase {
// for hdfs connector, DirectoryExistsValidator is responsible for validation
// replace the link config dir by the local path.
if (line.indexOf("linkConfReplacement") > 0) {
- line = line.replaceAll("linkConfReplacement", UrlSafeUtils.urlEncode(getSqoopMiniClusterTemporaryPath() + "/config/"));
+ line = line.replaceAll("linkConfReplacement", UrlSafeUtils.urlEncode(jettySqoopMiniCluster.getTemporaryPath() + "/config/"));
}
writer.write(line);
}
@@ -148,4 +158,9 @@ public class RepositoryDumpLoadToolTest extends SqoopTestCase {
writer.flush();
}
}
+
+ @AfterMethod
+ private void stopCluster() throws Exception {
+ jettySqoopMiniCluster.stop();
+ }
}