You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sa...@apache.org on 2020/04/20 10:59:24 UTC
[hadoop] branch branch-3.3 updated: HADOOP-16959. Resolve
hadoop-cos dependency conflict. Contributed by Yang Yu.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 9c81b17 HADOOP-16959. Resolve hadoop-cos dependency conflict. Contributed by Yang Yu.
9c81b17 is described below
commit 9c81b17153449f4b58ae26d6ea798c6d077d6458
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Mon Apr 20 18:06:19 2020 +0800
HADOOP-16959. Resolve hadoop-cos dependency conflict. Contributed by Yang Yu.
(cherry picked from commit 82ff7bc9abc8f3ad549db898953d98ef142ab02d)
---
.../hadoop-cloud-storage/pom.xml | 5 +
.../hadoop-cos/dev-support/findbugs-exclude.xml | 5 +
hadoop-cloud-storage-project/hadoop-cos/pom.xml | 29 ++++-
.../java/org/apache/hadoop/fs/cosn/BufferPool.java | 39 +++---
.../apache/hadoop/fs/cosn/CosNFileReadTask.java | 1 -
.../java/org/apache/hadoop/fs/cosn/CosNUtils.java | 46 ++++---
.../hadoop/fs/cosn/CosNativeFileSystemStore.java | 28 +++--
...er.java => AbstractCOSCredentialsProvider.java} | 44 +++----
...erList.java => COSCredentialsProviderList.java} | 42 ++++---
...=> EnvironmentVariableCredentialsProvider.java} | 32 +++--
...rovider.java => SimpleCredentialsProvider.java} | 34 +++---
.../src/site/markdown/cloud-storage/index.md | 13 +-
.../apache/hadoop/fs/cosn/TestCosCredentials.java | 134 +++++++++++++++++++++
hadoop-project/pom.xml | 12 ++
14 files changed, 337 insertions(+), 127 deletions(-)
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
index b5e35b0..7b9d12b 100644
--- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
@@ -128,5 +128,10 @@
<artifactId>hadoop-openstack</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-cos</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml
index 40d78d0..e647e67 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml
+++ b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml
@@ -15,4 +15,9 @@
limitations under the License.
-->
<FindBugsFilter>
+ <Match>
+ <Class name="org.apache.hadoop.fs.cosn.CosNInputStream.ReadBuffer"/>
+ <Method name="getBuffer"/>
+ <Bug pattern="EI_EXPOSE_REP"/>h_LIB
+ </Match>
</FindBugsFilter>
diff --git a/hadoop-cloud-storage-project/hadoop-cos/pom.xml b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
index 839bd04..64e5bf9 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
@@ -81,6 +81,31 @@
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>deplist</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>list</goal>
+ </goals>
+ <configuration>
+ <outputFile>${project.basedir}/target/hadoop-cloud-storage-deps/${project.artifactId}.cloud-storage-optional.txt</outputFile>
+ </configuration>
+ </execution>
+ <execution>
+ <id>package</id>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -93,8 +118,8 @@
<dependency>
<groupId>com.qcloud</groupId>
- <artifactId>cos_api</artifactId>
- <version>5.4.9</version>
+ <artifactId>cos_api-bundle</artifactId>
+ <version>5.6.19</version>
<scope>compile</scope>
</dependency>
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java
index a4ee4d5..409c9cb 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java
@@ -63,32 +63,27 @@ public final class BufferPool {
private File createDir(String dirPath) throws IOException {
File dir = new File(dirPath);
- if (null != dir) {
- if (!dir.exists()) {
- LOG.debug("Buffer dir: [{}] does not exists. create it first.",
- dirPath);
- if (dir.mkdirs()) {
- if (!dir.setWritable(true) || !dir.setReadable(true)
- || !dir.setExecutable(true)) {
- LOG.warn("Set the buffer dir: [{}]'s permission [writable,"
- + "readable, executable] failed.", dir.getAbsolutePath());
- }
- LOG.debug("Buffer dir: [{}] is created successfully.",
- dir.getAbsolutePath());
- } else {
- // Once again, check if it has been created successfully.
- // Prevent problems created by multiple processes at the same time.
- if (!dir.exists()) {
- throw new IOException("buffer dir:" + dir.getAbsolutePath()
- + " is created unsuccessfully");
- }
+ if (!dir.exists()) {
+ LOG.debug("Buffer dir: [{}] does not exists. create it first.",
+ dirPath);
+ if (dir.mkdirs()) {
+ if (!dir.setWritable(true) || !dir.setReadable(true)
+ || !dir.setExecutable(true)) {
+ LOG.warn("Set the buffer dir: [{}]'s permission [writable,"
+ + "readable, executable] failed.", dir.getAbsolutePath());
}
+ LOG.debug("Buffer dir: [{}] is created successfully.",
+ dir.getAbsolutePath());
} else {
- LOG.debug("buffer dir: {} already exists.", dirPath);
+ // Once again, check if it has been created successfully.
+ // Prevent problems created by multiple processes at the same time.
+ if (!dir.exists()) {
+ throw new IOException("buffer dir:" + dir.getAbsolutePath()
+ + " is created unsuccessfully");
+ }
}
} else {
- throw new IOException("creating buffer dir: " + dir.getAbsolutePath()
- + "unsuccessfully.");
+ LOG.debug("buffer dir: {} already exists.", dirPath);
}
return dir;
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java
index a5dcdda..249e9e1 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java
@@ -80,7 +80,6 @@ public class CosNFileReadTask implements Runnable {
public void run() {
int retries = 0;
RetryPolicy.RetryAction retryAction;
- LOG.info(Thread.currentThread().getName() + "read ...");
try {
this.readBuffer.lock();
do {
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java
index 39981ca..cdac15f 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java
@@ -22,15 +22,16 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.net.URI;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.qcloud.cos.auth.COSCredentialsProvider;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList;
-import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialProvider;
-import org.apache.hadoop.fs.cosn.auth.SimpleCredentialProvider;
+import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
+import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialsProvider;
+import org.apache.hadoop.fs.cosn.auth.SimpleCredentialsProvider;
/**
* Utility methods for CosN code.
@@ -48,21 +49,23 @@ public final class CosNUtils {
private CosNUtils() {
}
- public static COSCredentialProviderList createCosCredentialsProviderSet(
+ public static COSCredentialsProviderList createCosCredentialsProviderSet(
+ URI uri,
Configuration conf) throws IOException {
- COSCredentialProviderList credentialProviderList =
- new COSCredentialProviderList();
+ COSCredentialsProviderList credentialProviderList =
+ new COSCredentialsProviderList();
Class<?>[] cosClasses = CosNUtils.loadCosProviderClasses(
conf,
CosNConfigKeys.COSN_CREDENTIALS_PROVIDER);
if (0 == cosClasses.length) {
- credentialProviderList.add(new SimpleCredentialProvider(conf));
- credentialProviderList.add(new EnvironmentVariableCredentialProvider());
+ credentialProviderList.add(
+ new SimpleCredentialsProvider(uri, conf));
+ credentialProviderList.add(
+ new EnvironmentVariableCredentialsProvider(uri, conf));
} else {
for (Class<?> credClass : cosClasses) {
- credentialProviderList.add(createCOSCredentialProvider(
- conf,
+ credentialProviderList.add(createCOSCredentialProvider(uri, conf,
credClass));
}
}
@@ -83,16 +86,17 @@ public final class CosNUtils {
}
public static COSCredentialsProvider createCOSCredentialProvider(
+ URI uri,
Configuration conf,
Class<?> credClass) throws IOException {
COSCredentialsProvider credentialsProvider;
if (!COSCredentialsProvider.class.isAssignableFrom(credClass)) {
- throw new IllegalArgumentException(
- "class " + credClass + " " + NOT_COS_CREDENTIAL_PROVIDER);
+ throw new IllegalArgumentException("class " + credClass + " " +
+ NOT_COS_CREDENTIAL_PROVIDER);
}
if (Modifier.isAbstract(credClass.getModifiers())) {
- throw new IllegalArgumentException(
- "class " + credClass + " " + ABSTRACT_CREDENTIAL_PROVIDER);
+ throw new IllegalArgumentException("class " + credClass + " " +
+ ABSTRACT_CREDENTIAL_PROVIDER);
}
LOG.debug("Credential Provider class: " + credClass.getName());
@@ -112,8 +116,18 @@ public final class CosNUtils {
return credentialsProvider;
}
- Method factory = getFactoryMethod(
- credClass, COSCredentialsProvider.class, "getInstance");
+ // new credClass(uri, conf)
+ constructor = getConstructor(credClass, URI.class,
+ Configuration.class);
+ if (null != constructor) {
+ credentialsProvider =
+ (COSCredentialsProvider) constructor.newInstance(uri,
+ conf);
+ return credentialsProvider;
+ }
+
+ Method factory = getFactoryMethod(credClass,
+ COSCredentialsProvider.class, "getInstance");
if (null != factory) {
credentialsProvider = (COSCredentialsProvider) factory.invoke(null);
return credentialsProvider;
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
index 833f42d..d2484c0 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
@@ -34,6 +34,7 @@ import com.qcloud.cos.COSClient;
import com.qcloud.cos.ClientConfig;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.endpoint.SuffixEndpointBuilder;
import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.exception.CosServiceException;
import com.qcloud.cos.http.HttpProtocol;
@@ -64,7 +65,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList;
+import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
import org.apache.hadoop.util.VersionInfo;
import org.apache.http.HttpStatus;
@@ -89,9 +90,9 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
* @throws IOException Initialize the COS client failed,
* caused by incorrect options.
*/
- private void initCOSClient(Configuration conf) throws IOException {
- COSCredentialProviderList credentialProviderList =
- CosNUtils.createCosCredentialsProviderSet(conf);
+ private void initCOSClient(URI uri, Configuration conf) throws IOException {
+ COSCredentialsProviderList credentialProviderList =
+ CosNUtils.createCosCredentialsProviderSet(uri, conf);
String region = conf.get(CosNConfigKeys.COSN_REGION_KEY);
String endpointSuffix = conf.get(
CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY);
@@ -113,7 +114,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
ClientConfig config;
if (null == region) {
config = new ClientConfig(new Region(""));
- config.setEndPointSuffix(endpointSuffix);
+ config.setEndpointBuilder(new SuffixEndpointBuilder(endpointSuffix));
} else {
config = new ClientConfig(new Region(region));
}
@@ -146,7 +147,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
try {
- initCOSClient(conf);
+ initCOSClient(uri, conf);
this.bucketName = uri.getHost();
} catch (Exception e) {
handleException(e, "");
@@ -174,8 +175,8 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
PutObjectResult putObjectResult =
(PutObjectResult) callCOSClientWithRetry(putObjectRequest);
- LOG.debug("Store file successfully. COS key: [{}], ETag: [{}], "
- + "MD5: [{}].", key, putObjectResult.getETag(), new String(md5Hash));
+ LOG.debug("Store file successfully. COS key: [{}], ETag: [{}].",
+ key, putObjectResult.getETag());
} catch (Exception e) {
String errMsg = String.format("Store file failed. COS key: [%s], "
+ "exception: [%s]", key, e.toString());
@@ -196,8 +197,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
public void storeFile(String key, File file, byte[] md5Hash)
throws IOException {
LOG.info("Store file from local path: [{}]. file length: [{}] COS key: " +
- "[{}] MD5: [{}].", file.getCanonicalPath(), file.length(), key,
- new String(md5Hash));
+ "[{}]", file.getCanonicalPath(), file.length(), key);
storeFileWithRetry(key, new BufferedInputStream(new FileInputStream(file)),
md5Hash, file.length());
}
@@ -218,7 +218,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
byte[] md5Hash,
long contentLength) throws IOException {
LOG.info("Store file from input stream. COS key: [{}], "
- + "length: [{}], MD5: [{}].", key, contentLength, md5Hash);
+ + "length: [{}].", key, contentLength);
storeFileWithRetry(key, inputStream, md5Hash, contentLength);
}
@@ -250,7 +250,11 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
public PartETag uploadPart(File file, String key, String uploadId,
int partNum) throws IOException {
InputStream inputStream = new FileInputStream(file);
- return uploadPart(inputStream, key, uploadId, partNum, file.length());
+ try {
+ return uploadPart(inputStream, key, uploadId, partNum, file.length());
+ } finally {
+ inputStream.close();
+ }
}
@Override
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java
similarity index 50%
copy from hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java
copy to hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java
index f0635fc..1363a79 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java
@@ -17,38 +17,32 @@
*/
package org.apache.hadoop.fs.cosn.auth;
-import com.qcloud.cos.auth.BasicCOSCredentials;
-import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
-
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.CosNConfigKeys;
+
+import javax.annotation.Nullable;
+import java.net.URI;
/**
- * Get the credentials from the hadoop configuration.
+ * The base class for COS credential providers which take a URI or
+ * configuration in their constructor.
*/
-public class SimpleCredentialProvider implements COSCredentialsProvider {
- private String secretId;
- private String secretKey;
+public abstract class AbstractCOSCredentialsProvider
+ implements COSCredentialsProvider {
+ private final URI uri;
+ private final Configuration conf;
- public SimpleCredentialProvider(Configuration conf) {
- this.secretId = conf.get(
- CosNConfigKeys.COSN_SECRET_ID_KEY
- );
- this.secretKey = conf.get(
- CosNConfigKeys.COSN_SECRET_KEY_KEY
- );
+ public AbstractCOSCredentialsProvider(@Nullable URI uri,
+ Configuration conf) {
+ this.uri = uri;
+ this.conf = conf;
}
- @Override
- public COSCredentials getCredentials() {
- if (!StringUtils.isEmpty(this.secretId)
- && !StringUtils.isEmpty(this.secretKey)) {
- return new BasicCOSCredentials(this.secretId, this.secretKey);
- }
- throw new CosClientException("secret id or secret key is unset");
+ public URI getUri() {
+ return uri;
}
-}
+ public Configuration getConf() {
+ return conf;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
similarity index 80%
rename from hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java
rename to hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
index e900b99..e4c59a5 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
@@ -28,7 +28,6 @@ import com.google.common.base.Preconditions;
import com.qcloud.cos.auth.AnonymousCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.utils.StringUtils;
import org.slf4j.Logger;
@@ -37,10 +36,10 @@ import org.slf4j.LoggerFactory;
/**
* a list of cos credentials provider.
*/
-public class COSCredentialProviderList implements
+public class COSCredentialsProviderList implements
COSCredentialsProvider, AutoCloseable {
private static final Logger LOG =
- LoggerFactory.getLogger(COSCredentialProviderList.class);
+ LoggerFactory.getLogger(COSCredentialsProviderList.class);
private static final String NO_COS_CREDENTIAL_PROVIDERS =
"No COS Credential Providers";
@@ -48,17 +47,17 @@ public class COSCredentialProviderList implements
"Credentials requested after provider list was closed";
private final List<COSCredentialsProvider> providers =
- new ArrayList<>(1);
+ new ArrayList<COSCredentialsProvider>(1);
private boolean reuseLastProvider = true;
private COSCredentialsProvider lastProvider;
private final AtomicInteger refCount = new AtomicInteger(1);
private final AtomicBoolean isClosed = new AtomicBoolean(false);
- public COSCredentialProviderList() {
+ public COSCredentialsProviderList() {
}
- public COSCredentialProviderList(
+ public COSCredentialsProviderList(
Collection<COSCredentialsProvider> providers) {
this.providers.addAll(providers);
}
@@ -77,7 +76,7 @@ public class COSCredentialProviderList implements
}
}
- public COSCredentialProviderList share() {
+ public COSCredentialsProviderList share() {
Preconditions.checkState(!this.closed(), "Provider list is closed");
this.refCount.incrementAndGet();
return this;
@@ -100,16 +99,13 @@ public class COSCredentialProviderList implements
}
for (COSCredentialsProvider provider : this.providers) {
- try {
- COSCredentials credentials = provider.getCredentials();
- if (!StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId())
- && !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey())
- || credentials instanceof AnonymousCOSCredentials) {
- this.lastProvider = provider;
- return credentials;
- }
- } catch (CosClientException e) {
- LOG.warn("No credentials provided by {}: {}", provider, e.toString());
+ COSCredentials credentials = provider.getCredentials();
+ if (null != credentials
+ && !StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId())
+ && !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey())
+ || credentials instanceof AnonymousCOSCredentials) {
+ this.lastProvider = provider;
+ return credentials;
}
}
@@ -118,6 +114,17 @@ public class COSCredentialProviderList implements
}
@Override
+ public void refresh() {
+ if (this.closed()) {
+ return;
+ }
+
+ for (COSCredentialsProvider cosCredentialsProvider : this.providers) {
+ cosCredentialsProvider.refresh();
+ }
+ }
+
+ @Override
public void close() throws Exception {
if (this.closed()) {
return;
@@ -135,5 +142,4 @@ public class COSCredentialProviderList implements
}
}
}
-
}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java
similarity index 70%
rename from hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java
rename to hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java
index 0a7786b..baa7690 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java
@@ -20,16 +20,24 @@ package org.apache.hadoop.fs.cosn.auth;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.utils.StringUtils;
-
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.cosn.Constants;
+import javax.annotation.Nullable;
+import java.net.URI;
+
/**
- * the provider obtaining the cos credentials from the environment variables.
+ * The provider obtaining the cos credentials from the environment variables.
*/
-public class EnvironmentVariableCredentialProvider
- implements COSCredentialsProvider {
+public class EnvironmentVariableCredentialsProvider
+ extends AbstractCOSCredentialsProvider implements COSCredentialsProvider {
+
+ public EnvironmentVariableCredentialsProvider(@Nullable URI uri,
+ Configuration conf) {
+ super(uri, conf);
+ }
+
@Override
public COSCredentials getCredentials() {
String secretId = System.getenv(Constants.COSN_SECRET_ID_ENV);
@@ -41,15 +49,19 @@ public class EnvironmentVariableCredentialProvider
if (!StringUtils.isNullOrEmpty(secretId)
&& !StringUtils.isNullOrEmpty(secretKey)) {
return new BasicCOSCredentials(secretId, secretKey);
- } else {
- throw new CosClientException(
- "Unable to load COS credentials from environment variables" +
- "(COS_SECRET_ID or COS_SECRET_KEY)");
}
+
+ return null;
+ }
+
+ @Override
+ public void refresh() {
}
@Override
public String toString() {
- return "EnvironmentVariableCredentialProvider{}";
+ return String.format("EnvironmentVariableCredentialsProvider{%s, %s}",
+ Constants.COSN_SECRET_ID_ENV,
+ Constants.COSN_SECRET_KEY_ENV);
}
}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java
similarity index 66%
rename from hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java
rename to hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java
index f0635fc..107574a 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java
@@ -20,35 +20,41 @@ package org.apache.hadoop.fs.cosn.auth;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
-
-import org.apache.commons.lang3.StringUtils;
+import com.qcloud.cos.utils.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.cosn.CosNConfigKeys;
+import javax.annotation.Nullable;
+import java.net.URI;
+
/**
* Get the credentials from the hadoop configuration.
*/
-public class SimpleCredentialProvider implements COSCredentialsProvider {
+public class SimpleCredentialsProvider
+ extends AbstractCOSCredentialsProvider implements COSCredentialsProvider {
private String secretId;
private String secretKey;
- public SimpleCredentialProvider(Configuration conf) {
- this.secretId = conf.get(
- CosNConfigKeys.COSN_SECRET_ID_KEY
- );
- this.secretKey = conf.get(
- CosNConfigKeys.COSN_SECRET_KEY_KEY
- );
+ public SimpleCredentialsProvider(@Nullable URI uri, Configuration conf) {
+ super(uri, conf);
+ if (null != conf) {
+ this.secretId = conf.get(
+ CosNConfigKeys.COSN_SECRET_ID_KEY);
+ this.secretKey = conf.get(
+ CosNConfigKeys.COSN_SECRET_KEY_KEY);
+ }
}
@Override
public COSCredentials getCredentials() {
- if (!StringUtils.isEmpty(this.secretId)
- && !StringUtils.isEmpty(this.secretKey)) {
+ if (!StringUtils.isNullOrEmpty(this.secretId)
+ && !StringUtils.isNullOrEmpty(this.secretKey)) {
return new BasicCOSCredentials(this.secretId, this.secretKey);
}
- throw new CosClientException("secret id or secret key is unset");
+ return null;
}
+ @Override
+ public void refresh() {
+ }
}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md b/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md
index d4f8728..9c96ac3 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md
@@ -130,20 +130,19 @@ Each user needs to properly configure the credentials ( User's secreteId and sec
```xml
<property>
<name>fs.cosn.credentials.provider</name>
- <value>org.apache.hadoop.fs.auth.SimpleCredentialProvider</value>
+ <value>org.apache.hadoop.fs.auth.SimpleCredentialsProvider</value>
<description>
This option allows the user to specify how to get the credentials.
Comma-separated class names of credential provider classes which implement
com.qcloud.cos.auth.COSCredentialsProvider:
- 1.org.apache.hadoop.fs.auth.SimpleCredentialProvider: Obtain the secret id and secret key
- from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml
- 2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY
+ 1.org.apache.hadoop.fs.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml
+ 2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY
If unspecified, the default order of credential providers is:
- 1. org.apache.hadoop.fs.auth.SimpleCredentialProvider
- 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider
+ 1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider
+ 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider
</description>
</property>
@@ -237,7 +236,7 @@ Hadoop-COS provides rich runtime properties to set, and most of these do not req
| properties | description | default value | required |
|:----------:|:-----------|:-------------:|:--------:|
| fs.defaultFS | Configure the default file system used by Hadoop.| None | NO |
-| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider: <br/> 1. org.apache.hadoop.fs.cos.auth.SimpleCredentialProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the sec [...]
+| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider: <br/> 1. org.apache.hadoop.fs.cos.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the s [...]
| fs.cosn.userinfo.secretId/secretKey | The API key information of your account | None | YES |
| fs.cosn.bucket.region | The region where the bucket is located. | None | YES |
| fs.cosn.impl | The implementation class of the CosN filesystem. | None | YES |
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java
new file mode 100644
index 0000000..8b74f36
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.cosn;
+
+import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.auth.COSCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class TestCosCredentials {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestCosCredentials.class);
+
+ private final URI fsUri;
+
+ private final String testCosNSecretId = "secretId";
+ private final String testCosNSecretKey = "secretKey";
+ private final String testCosNEnvSecretId = "env_secretId";
+ private final String testCosNEnvSecretKey = "env_secretKey";
+
+ public TestCosCredentials() throws URISyntaxException {
+ // A fake uri for tests.
+ this.fsUri = new URI("cosn://test-bucket-1250000000");
+ }
+
+ @Test
+ public void testSimpleCredentialsProvider() throws Throwable {
+ Configuration configuration = new Configuration();
+ configuration.set(CosNConfigKeys.COSN_SECRET_ID_KEY,
+ testCosNSecretId);
+ configuration.set(CosNConfigKeys.COSN_SECRET_KEY_KEY,
+ testCosNSecretKey);
+ validateCredentials(this.fsUri, configuration);
+ }
+
+ @Test
+ public void testEnvironmentCredentialsProvider() throws Throwable {
+ Configuration configuration = new Configuration();
+ // Set EnvironmentVariableCredentialsProvider as the CosCredentials
+ // Provider.
+ configuration.set(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER,
+ "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider");
+ // Set the environment variables storing the secret id and secret key.
+ System.setProperty(Constants.COSN_SECRET_ID_ENV, testCosNEnvSecretId);
+ System.setProperty(Constants.COSN_SECRET_KEY_ENV, testCosNEnvSecretKey);
+ validateCredentials(this.fsUri, configuration);
+ }
+
+ private void validateCredentials(URI uri, Configuration configuration)
+ throws IOException {
+ if (null != configuration) {
+ COSCredentialsProvider credentialsProvider =
+ CosNUtils.createCosCredentialsProviderSet(uri, configuration);
+ COSCredentials cosCredentials = credentialsProvider.getCredentials();
+ assertNotNull("The cos credentials obtained is null.", cosCredentials);
+ if (configuration.get(
+ CosNConfigKeys.COSN_CREDENTIALS_PROVIDER).compareToIgnoreCase(
+ "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider")
+ == 0) {
+ if (null == cosCredentials.getCOSAccessKeyId()
+ || cosCredentials.getCOSAccessKeyId().isEmpty()
+ || null == cosCredentials.getCOSSecretKey()
+ || cosCredentials.getCOSSecretKey().isEmpty()) {
+ String failMessage = String.format(
+ "Test EnvironmentVariableCredentialsProvider failed. The " +
+ "expected is [secretId: %s, secretKey: %s], but got null or" +
+ " empty.", testCosNEnvSecretId, testCosNEnvSecretKey);
+ fail(failMessage);
+ }
+
+ if (cosCredentials.getCOSAccessKeyId()
+ .compareTo(testCosNEnvSecretId) != 0
+ || cosCredentials.getCOSSecretKey()
+ .compareTo(testCosNEnvSecretKey) != 0) {
+ String failMessage = String.format("Test " +
+ "EnvironmentVariableCredentialsProvider failed. " +
+ "The expected is [secretId: %s, secretKey: %s], but got is " +
+ "[secretId:%s, secretKey:%s].", testCosNEnvSecretId,
+ testCosNEnvSecretKey, cosCredentials.getCOSAccessKeyId(),
+ cosCredentials.getCOSSecretKey());
+ }
+ // expected
+ } else {
+ if (null == cosCredentials.getCOSAccessKeyId()
+ || cosCredentials.getCOSAccessKeyId().isEmpty()
+ || null == cosCredentials.getCOSSecretKey()
+ || cosCredentials.getCOSSecretKey().isEmpty()) {
+ String failMessage = String.format(
+ "Test COSCredentials failed. The " +
+ "expected is [secretId: %s, secretKey: %s], but got null or" +
+ " empty.", testCosNSecretId, testCosNSecretKey);
+ fail(failMessage);
+ }
+ if (cosCredentials.getCOSAccessKeyId()
+ .compareTo(testCosNSecretId) != 0
+ || cosCredentials.getCOSSecretKey()
+ .compareTo(testCosNSecretKey) != 0) {
+ String failMessage = String.format("Test " +
+ "EnvironmentVariableCredentialsProvider failed. " +
+ "The expected is [secretId: %s, secretKey: %s], but got is " +
+ "[secretId:%s, secretKey:%s].", testCosNSecretId,
+ testCosNSecretKey, cosCredentials.getCOSAccessKeyId(),
+ cosCredentials.getCOSSecretKey());
+ fail(failMessage);
+ }
+ // expected
+ }
+ }
+ }
+}
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3b016a5..4ef1870 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -644,6 +644,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-cos</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
<version>${hadoop.version}</version>
</dependency>
@@ -1433,6 +1439,12 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.qcloud</groupId>
+ <artifactId>cos_api-bundle</artifactId>
+ <version>5.6.19</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org