You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/02/15 00:56:48 UTC
[11/50] [abbrv] hadoop git commit: HADOOP-13929. ADLS connector
should not check in contract-test-options.xml. (John Zhuge via lei)
HADOOP-13929. ADLS connector should not check in contract-test-options.xml. (John Zhuge via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71c23c9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71c23c9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71c23c9f
Branch: refs/heads/YARN-2915
Commit: 71c23c9fc94cfdf58de80effbc3f51c0925d0cfe
Parents: 4ed33e9
Author: Lei Xu <le...@cloudera.com>
Authored: Mon Feb 13 13:33:13 2017 -0800
Committer: Lei Xu <le...@cloudera.com>
Committed: Mon Feb 13 13:33:13 2017 -0800
----------------------------------------------------------------------
.gitignore | 12 ++--
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 2 +-
.../src/site/markdown/index.md | 4 +-
.../fs/adl/live/AdlStorageConfiguration.java | 42 +++++++-------
.../src/test/resources/adls.xml | 11 ++++
.../test/resources/contract-test-options.xml | 61 --------------------
6 files changed, 39 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index eb98a3f..724162d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,10 @@ target
build
dependency-reduced-pom.xml
+# Filesystem contract test options and credentials
+auth-keys.xml
+azure-auth-keys.xml
+
# External tool builders
*/.externalToolBuilders
*/maven-eclipse.xml
@@ -24,8 +28,6 @@ dependency-reduced-pom.xml
hadoop-common-project/hadoop-kms/downloads/
hadoop-hdfs-project/hadoop-hdfs/downloads
hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads
-hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml
-hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.toolbox
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/dist
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tmp
@@ -41,10 +43,4 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/testem.log
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/dist
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/tmp
yarnregistry.pdf
-hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
-hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
-hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
-hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
patchprocess/
-hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml
-hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 3d41025..303b7bc 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -72,7 +72,7 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.*;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class AdlFileSystem extends FileSystem {
- static final String SCHEME = "adl";
+ public static final String SCHEME = "adl";
static final int DEFAULT_PORT = 443;
private URI uri;
private String userName;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index ced5cff..5037db6 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -224,7 +224,9 @@ commands demonstrate access to a storage account named `youraccount`.
## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
-A selection of tests can run against the Azure Data Lake Storage. To run tests against Adl storage. Please configure contract-test-options.xml with Adl account information mentioned in the above sections. Also turn on contract test execution flag to trigger tests against Azure Data Lake Storage.
+A selection of tests can run against the Azure Data Lake Storage. To run these
+tests, please create `src/test/resources/auth-keys.xml` with Adl account
+information mentioned in the above sections and the following properties.
<property>
<name>dfs.adl.test.contract.enable</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
index f7f0b71..7d6153d 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs.adl.live;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
+import org.apache.hadoop.util.ReflectionUtils;
import java.io.IOException;
import java.net.URI;
@@ -30,15 +32,19 @@ import java.net.URISyntaxException;
* Configure Adl storage file system.
*/
public final class AdlStorageConfiguration {
+ static final String CONTRACT_XML = "adls.xml";
+
private static final String CONTRACT_ENABLE_KEY =
"dfs.adl.test.contract.enable";
+ private static final boolean CONTRACT_ENABLE_DEFAULT = false;
+
+ private static final String FILE_SYSTEM_KEY =
+ String.format("test.fs.%s.name", AdlFileSystem.SCHEME);
- private static final String TEST_CONFIGURATION_FILE_NAME =
- "contract-test-options.xml";
- private static final String TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME =
- "adls.xml";
- private static final String KEY_FILE_SYSTEM_IMPL = "fs.contract.test.fs";
- private static final String KEY_FILE_SYSTEM = "test.fs.adl.name";
+ private static final String FILE_SYSTEM_IMPL_KEY =
+ String.format("fs.%s.impl", AdlFileSystem.SCHEME);
+ private static final Class<?> FILE_SYSTEM_IMPL_DEFAULT =
+ AdlFileSystem.class;
private static boolean isContractTestEnabled = false;
private static Configuration conf = null;
@@ -48,8 +54,7 @@ public final class AdlStorageConfiguration {
public synchronized static Configuration getConfiguration() {
Configuration newConf = new Configuration();
- newConf.addResource(TEST_CONFIGURATION_FILE_NAME);
- newConf.addResource(TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME);
+ newConf.addResource(CONTRACT_XML);
return newConf;
}
@@ -58,7 +63,8 @@ public final class AdlStorageConfiguration {
conf = getConfiguration();
}
- isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, false);
+ isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY,
+ CONTRACT_ENABLE_DEFAULT);
return isContractTestEnabled;
}
@@ -72,23 +78,15 @@ public final class AdlStorageConfiguration {
return null;
}
- String fileSystem = conf.get(KEY_FILE_SYSTEM);
+ String fileSystem = conf.get(FILE_SYSTEM_KEY);
if (fileSystem == null || fileSystem.trim().length() == 0) {
throw new IOException("Default file system not configured.");
}
- String fileSystemImpl = conf.get(KEY_FILE_SYSTEM_IMPL);
- if (fileSystemImpl == null || fileSystemImpl.trim().length() == 0) {
- throw new IOException(
- "Configuration " + KEY_FILE_SYSTEM_IMPL + "does not exist.");
- }
- FileSystem fs = null;
- try {
- fs = (FileSystem) Class.forName(fileSystemImpl).newInstance();
- } catch (Exception e) {
- throw new IOException("Could not instantiate the filesystem.");
- }
- fs.initialize(new URI(conf.get(KEY_FILE_SYSTEM)), conf);
+ Class<?> clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY,
+ FILE_SYSTEM_IMPL_DEFAULT);
+ FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf);
+ fs.initialize(new URI(fileSystem), conf);
return fs;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
index 76a78b0..5bbdd6f 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
@@ -12,6 +12,17 @@
limitations under the License. See accompanying LICENSE file.
-->
<configuration>
+ <!--
+ To run live tests:
+ # Create a file auth-keys.xml - DO NOT ADD TO REVISION CONTROL
+ # Add property test.fs.adl.name to point to an ADLS filesystem URL
+ # Add property dfs.adl.test.contract.enable with value set to true
+ # Add the credentials for the service you are testing against
+ -->
+ <include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml">
+ <fallback/>
+ </include>
+
<property>
<name>fs.contract.test.root-tests-enabled</name>
<value>true</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
deleted file mode 100644
index 646922e..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<configuration>
- <property>
- <name>dfs.adls.oauth2.refresh.url</name>
- <value>
- </value>
- </property>
-
- <property>
- <name>dfs.adls.oauth2.credential</name>
- <value></value>
- </property>
-
- <property>
- <name>dfs.adls.oauth2.client.id</name>
- <value></value>
- </property>
-
- <property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
- <value>ClientCredential</value>
- <description>
- Supported provider type:
-
- "ClientCredential" : Client id and client credentials(Provided
- through configuration file) flow.
-
- "RefreshToken" : Client id and refresh token(Provided
- through configuration file)flow.
-
- "Custom" : Custom AAD token management.
- </description>
- </property>
-
- <property>
- <name>dfs.adl.test.contract.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>test.fs.adl.name</name>
- <value></value>
- </property>
-
- <property>
- <name>fs.contract.test.fs</name>
- <value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
- </property>
-</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org