You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dr...@apache.org on 2016/09/17 11:14:14 UTC
[1/2] hadoop git commit: HADOOP-13610. Clean up AliyunOss integration
tests. Contributed by Genmao Yu
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-12756 dbb28eb1f -> a1940464a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java
deleted file mode 100644
index eb0c5e0..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDispCp.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
-
-import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
-
-/**
- * Contract test suite covering Aliyun OSS integration with DistCp.
- */
-public class TestOSSContractDispCp extends AbstractContractDistCpTest {
-
- private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
-
- @Override
- protected Configuration createConfiguration() {
- Configuration newConf = super.createConfiguration();
- newConf.setLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MULTIPART_SETTING);
- newConf.setLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_SETTING);
- return newConf;
- }
-
- @Override
- protected OSSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java
deleted file mode 100644
index cc21a2e..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractGetFileStatus.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * Test getFileStatus and related listing operations.
- */
-public class TestOSSContractGetFileStatus
- extends AbstractContractGetFileStatusTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java
deleted file mode 100644
index 1dcb7f0..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractMkdir.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * OSS contract directory tests.
- */
-public class TestOSSContractMkdir extends AbstractContractMkdirTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java
deleted file mode 100644
index ee0c055..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractOpen.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * OSS contract opening file tests.
- */
-public class TestOSSContractOpen extends AbstractContractOpenTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java
deleted file mode 100644
index 634fcf1..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRename.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * OSS contract renaming tests.
- */
-public class TestOSSContractRename extends AbstractContractRenameTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java
deleted file mode 100644
index cbc262c..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractRootDir.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- * Root dir operations against an Aliyun OSS bucket.
- */
-public class TestOSSContractRootDir extends
- AbstractContractRootDirectoryTest {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestOSSContractRootDir.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-
- @Override
- public void testListEmptyRootDirectory() throws IOException {
- for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
- try {
- super.testListEmptyRootDirectory();
- break;
- } catch (AssertionError | FileNotFoundException e) {
- if (attempt < maxAttempts) {
- LOG.info("Attempt {} of {} for empty root directory test failed. "
- + "Attempting retry.", attempt, maxAttempts);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e2) {
- Thread.currentThread().interrupt();
- fail("Test interrupted.");
- break;
- }
- } else {
- LOG.error(
- "Empty root directory test failed {} attempts. Failing test.",
- maxAttempts);
- throw e;
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java
deleted file mode 100644
index 40ea772..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractSeek.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * OSS contract seeking tests.
- */
-public class TestOSSContractSeek extends AbstractContractSeekTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
new file mode 100644
index 0000000..2bc34b7
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
@@ -0,0 +1,105 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-remove-dest-if-empty-dir</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek-on-closed-file</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-overwrites-dest</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.download.size</name>
+ <value>102400</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml
deleted file mode 100644
index 2bc34b7..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/oss.xml
+++ /dev/null
@@ -1,105 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<configuration>
- <property>
- <name>fs.contract.test.random-seek-count</name>
- <value>10</value>
- </property>
-
- <property>
- <name>fs.contract.is-blobstore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.is-case-sensitive</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rename-returns-false-if-source-missing</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.rename-remove-dest-if-empty-dir</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-append</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-atomic-directory-delete</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-atomic-rename</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-block-locality</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-concat</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-seek</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-seek-on-closed-file</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rejects-seek-past-eof</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-strict-exceptions</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-unix-permissions</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.rename-overwrites-dest</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.oss.multipart.download.size</name>
- <value>102400</value>
- </property>
-</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HADOOP-13610. Clean up AliyunOss integration
tests. Contributed by Genmao Yu
Posted by dr...@apache.org.
HADOOP-13610. Clean up AliyunOss integration tests. Contributed by Genmao Yu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1940464
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1940464
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1940464
Branch: refs/heads/HADOOP-12756
Commit: a1940464a498d1e662e5c3843f2d31ce63ec726b
Parents: dbb28eb
Author: Kai Zheng <ka...@intel.com>
Authored: Sun Sep 18 19:10:48 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Sun Sep 18 19:10:48 2016 +0800
----------------------------------------------------------------------
.../fs/aliyun/oss/AliyunOSSTestUtils.java | 77 ++++++
.../hadoop/fs/aliyun/oss/OSSTestUtils.java | 80 ------
.../oss/TestAliyunOSSFileSystemContract.java | 239 ++++++++++++++++++
.../oss/TestAliyunOSSFileSystemStore.java | 121 +++++++++
.../fs/aliyun/oss/TestAliyunOSSInputStream.java | 142 +++++++++++
.../aliyun/oss/TestAliyunOSSOutputStream.java | 91 +++++++
.../oss/TestAliyunOSSTemporaryCredentials.java | 65 +++++
.../aliyun/oss/TestOSSFileSystemContract.java | 243 -------------------
.../fs/aliyun/oss/TestOSSFileSystemStore.java | 121 ---------
.../fs/aliyun/oss/TestOSSInputStream.java | 141 -----------
.../fs/aliyun/oss/TestOSSOutputStream.java | 90 -------
.../aliyun/oss/TestOSSTemporaryCredentials.java | 64 -----
.../aliyun/oss/contract/AliyunOSSContract.java | 49 ++++
.../fs/aliyun/oss/contract/OSSContract.java | 53 ----
.../contract/TestAliyunOSSContractCreate.java | 35 +++
.../contract/TestAliyunOSSContractDelete.java | 34 +++
.../contract/TestAliyunOSSContractDispCp.java | 44 ++++
.../TestAliyunOSSContractGetFileStatus.java | 35 +++
.../contract/TestAliyunOSSContractMkdir.java | 34 +++
.../oss/contract/TestAliyunOSSContractOpen.java | 34 +++
.../contract/TestAliyunOSSContractRename.java | 35 +++
.../contract/TestAliyunOSSContractRootDir.java | 69 ++++++
.../oss/contract/TestAliyunOSSContractSeek.java | 34 +++
.../oss/contract/TestOSSContractCreate.java | 35 ---
.../oss/contract/TestOSSContractDelete.java | 34 ---
.../oss/contract/TestOSSContractDispCp.java | 44 ----
.../contract/TestOSSContractGetFileStatus.java | 35 ---
.../oss/contract/TestOSSContractMkdir.java | 34 ---
.../oss/contract/TestOSSContractOpen.java | 34 ---
.../oss/contract/TestOSSContractRename.java | 35 ---
.../oss/contract/TestOSSContractRootDir.java | 69 ------
.../oss/contract/TestOSSContractSeek.java | 34 ---
.../src/test/resources/contract/aliyun-oss.xml | 105 ++++++++
.../src/test/resources/contract/oss.xml | 105 --------
34 files changed, 1243 insertions(+), 1251 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
new file mode 100644
index 0000000..84dba6a
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * Utility class for Aliyun OSS Tests.
+ */
+public final class AliyunOSSTestUtils {
+
+ private AliyunOSSTestUtils() {
+ }
+
+ /**
+ * Create the test filesystem.
+ *
+ * If the test.fs.oss.name property is not set,
+ * tests will fail.
+ *
+ * @param conf configuration
+ * @return the FS
+ * @throws IOException
+ */
+ public static AliyunOSSFileSystem createTestFileSystem(Configuration conf)
+ throws IOException {
+ String fsname = conf.getTrimmed(
+ TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, "");
+
+ boolean liveTest = !StringUtils.isEmpty(fsname);
+ URI testURI = null;
+ if (liveTest) {
+ testURI = URI.create(fsname);
+ liveTest = testURI.getScheme().equals(Constants.FS_OSS);
+ }
+
+ if (!liveTest) {
+ throw new AssumptionViolatedException("No test filesystem in "
+ + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME);
+ }
+ AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
+ ossfs.initialize(testURI, conf);
+ return ossfs;
+ }
+
+ /**
+ * Generate unique test path for multiple user tests.
+ *
+ * @return root test path
+ */
+ public static String generateUniqueTestPath() {
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ return testUniqueForkId == null ? "/test" :
+ "/" + testUniqueForkId + "/test";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java
deleted file mode 100644
index 37ed831..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/OSSTestUtils.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.internal.AssumptionViolatedException;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Date;
-import java.util.Random;
-
-/**
- * Utility class for OSS Tests.
- */
-public final class OSSTestUtils {
-
- private OSSTestUtils() {
- }
-
- /**
- * Create the test filesystem.
- *
- * If the test.fs.oss.name property is not set,
- * tests will fail.
- *
- * @param conf configuration
- * @return the FS
- * @throws IOException
- */
- public static AliyunOSSFileSystem createTestFileSystem(Configuration conf)
- throws IOException {
- String fsname = conf.getTrimmed(
- TestOSSFileSystemContract.TEST_FS_OSS_NAME, "");
-
- boolean liveTest = !StringUtils.isEmpty(fsname);
- URI testURI = null;
- if (liveTest) {
- testURI = URI.create(fsname);
- liveTest = testURI.getScheme().equals(Constants.FS_OSS);
- }
-
- if (!liveTest) {
- throw new AssumptionViolatedException("No test filesystem in "
- + TestOSSFileSystemContract.TEST_FS_OSS_NAME);
- }
- AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
- ossfs.initialize(testURI, conf);
- return ossfs;
- }
-
- /**
- * Generate unique test path for multiple user tests.
- *
- * @return root test path
- */
- public static String generateUniqueTestPath() {
- Long time = new Date().getTime();
- Random rand = new Random();
- return "/test_" + Long.toString(time) + "_"
- + Long.toString(Math.abs(rand.nextLong()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
new file mode 100644
index 0000000..ad8ef6e
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Tests a live Aliyun OSS system.
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ */
+public class TestAliyunOSSFileSystemContract
+ extends FileSystemContractBaseTest {
+ public static final String TEST_FS_OSS_NAME = "test.fs.oss.name";
+ private static String testRootPath =
+ AliyunOSSTestUtils.generateUniqueTestPath();
+
+ @Override
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fs = AliyunOSSTestUtils.createTestFileSystem(conf);
+ super.setUp();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(super.path(testRootPath), true);
+ }
+ super.tearDown();
+ }
+
+ @Override
+ protected Path path(String path) {
+ if (path.startsWith("/")) {
+ return super.path(testRootPath + path);
+ } else {
+ return super.path(testRootPath + "/" + path);
+ }
+ }
+
+ @Override
+ public void testMkdirsWithUmask() throws Exception {
+ // not supported
+ }
+
+ @Override
+ public void testRootDirAlwaysExists() throws Exception {
+ //this will throw an exception if the path is not found
+ fs.getFileStatus(super.path("/"));
+ //this catches overrides of the base exists() method that don't
+ //use getFileStatus() as an existence probe
+ assertTrue("FileSystem.exists() fails for root",
+ fs.exists(super.path("/")));
+ }
+
+ @Override
+ public void testRenameRootDirForbidden() throws Exception {
+ if (!renameSupported()) {
+ return;
+ }
+ rename(super.path("/"),
+ super.path("/test/newRootDir"),
+ false, true, false);
+ }
+
+ public void testDeleteSubdir() throws IOException {
+ Path parentDir = this.path("/test/hadoop");
+ Path file = this.path("/test/hadoop/file");
+ Path subdir = this.path("/test/hadoop/subdir");
+ this.createFile(file);
+
+ assertTrue("Created subdir", this.fs.mkdirs(subdir));
+ assertTrue("File exists", this.fs.exists(file));
+ assertTrue("Parent dir exists", this.fs.exists(parentDir));
+ assertTrue("Subdir exists", this.fs.exists(subdir));
+
+ assertTrue("Deleted subdir", this.fs.delete(subdir, true));
+ assertTrue("Parent should exist", this.fs.exists(parentDir));
+
+ assertTrue("Deleted file", this.fs.delete(file, false));
+ assertTrue("Parent should exist", this.fs.exists(parentDir));
+ }
+
+
+ @Override
+ protected boolean renameSupported() {
+ return true;
+ }
+
+ @Override
+ public void testRenameNonExistentPath() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/path");
+ Path dst = this.path("/test/new/newpath");
+ try {
+ super.rename(src, dst, false, false, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameFileMoveToNonExistentDirectory() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/file");
+ this.createFile(src);
+ Path dst = this.path("/test/new/newfile");
+ try {
+ super.rename(src, dst, false, true, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/dir");
+ this.fs.mkdirs(src);
+ Path dst = this.path("/test/new/newdir");
+ try {
+ super.rename(src, dst, false, true, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameFileMoveToExistingDirectory() throws Exception {
+ super.testRenameFileMoveToExistingDirectory();
+ }
+
+ @Override
+ public void testRenameFileAsExistingFile() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/file");
+ this.createFile(src);
+ Path dst = this.path("/test/new/newfile");
+ this.createFile(dst);
+ try {
+ super.rename(src, dst, false, true, true);
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameDirectoryAsExistingFile() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/dir");
+ this.fs.mkdirs(src);
+ Path dst = this.path("/test/new/newfile");
+ this.createFile(dst);
+ try {
+ super.rename(src, dst, false, true, true);
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+ }
+
+ public void testGetFileStatusFileAndDirectory() throws Exception {
+ Path filePath = this.path("/test/oss/file1");
+ this.createFile(filePath);
+ assertTrue("Should be file", this.fs.getFileStatus(filePath).isFile());
+ assertFalse("Should not be directory",
+ this.fs.getFileStatus(filePath).isDirectory());
+
+ Path dirPath = this.path("/test/oss/dir");
+ this.fs.mkdirs(dirPath);
+ assertTrue("Should be directory",
+ this.fs.getFileStatus(dirPath).isDirectory());
+ assertFalse("Should not be file", this.fs.getFileStatus(dirPath).isFile());
+ }
+
+ public void testMkdirsForExistingFile() throws Exception {
+ Path testFile = this.path("/test/hadoop/file");
+ assertFalse(this.fs.exists(testFile));
+ this.createFile(testFile);
+ assertTrue(this.fs.exists(testFile));
+ try {
+ this.fs.mkdirs(testFile);
+ fail("Should throw FileAlreadyExistsException!");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+
+ public void testWorkingDirectory() throws Exception {
+ Path workDir = super.path(this.getDefaultWorkingDirectory());
+ assertEquals(workDir, this.fs.getWorkingDirectory());
+ this.fs.setWorkingDirectory(super.path("."));
+ assertEquals(workDir, this.fs.getWorkingDirectory());
+ this.fs.setWorkingDirectory(super.path(".."));
+ assertEquals(workDir.getParent(), this.fs.getWorkingDirectory());
+ Path relativeDir = super.path("hadoop");
+ this.fs.setWorkingDirectory(relativeDir);
+ assertEquals(relativeDir, this.fs.getWorkingDirectory());
+ Path absoluteDir = super.path("/test/hadoop");
+ this.fs.setWorkingDirectory(absoluteDir);
+ assertEquals(absoluteDir, this.fs.getWorkingDirectory());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
new file mode 100644
index 0000000..c849804
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.*;
+import java.net.URI;
+import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeNotNull;
+
+/**
+ * Test the bridging logic between Hadoop's abstract filesystem and
+ * Aliyun OSS.
+ */
+public class TestAliyunOSSFileSystemStore {
+ private Configuration conf;
+ private AliyunOSSFileSystemStore store;
+ private AliyunOSSFileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ fs = new AliyunOSSFileSystem();
+ fs.initialize(URI.create(conf.get("test.fs.oss.name")), conf);
+ store = fs.getStore();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ store.purge("test");
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @BeforeClass
+ public static void checkSettings() throws Exception {
+ Configuration conf = new Configuration();
+ assumeNotNull(conf.get(Constants.ACCESS_KEY));
+ assumeNotNull(conf.get(Constants.SECRET_KEY));
+ assumeNotNull(conf.get("test.fs.oss.name"));
+ }
+
+ protected void writeRenameReadCompare(Path path, long len)
+ throws IOException, NoSuchAlgorithmException {
+ // If len > fs.oss.multipart.upload.threshold,
+ // we'll use a multipart upload copy
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ OutputStream out = new BufferedOutputStream(
+ new DigestOutputStream(fs.create(path, false), digest));
+ for (long i = 0; i < len; i++) {
+ out.write('Q');
+ }
+ out.flush();
+ out.close();
+
+ assertTrue("Exists", fs.exists(path));
+
+ Path copyPath = path.suffix(".copy");
+ fs.rename(path, copyPath);
+
+ assertTrue("Copy exists", fs.exists(copyPath));
+
+ // Download file from Aliyun OSS and compare the digest against the original
+ MessageDigest digest2 = MessageDigest.getInstance("MD5");
+ InputStream in = new BufferedInputStream(
+ new DigestInputStream(fs.open(copyPath), digest2));
+ long copyLen = 0;
+ while (in.read() != -1) {
+ copyLen++;
+ }
+ in.close();
+
+ assertEquals("Copy length matches original", len, copyLen);
+ assertArrayEquals("Digests match", digest.digest(), digest2.digest());
+ }
+
+ @Test
+ public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
+ // Regular upload, regular copy
+ writeRenameReadCompare(new Path("/test/small"), 16384);
+ }
+
+ @Test
+ public void testLargeUpload()
+ throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, multipart copy
+ writeRenameReadCompare(new Path("/test/xlarge"), 52428800L); // 50MB byte
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
new file mode 100644
index 0000000..892eda0
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.*;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Random;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests basic functionality for AliyunOSSInputStream, including seeking and
+ * reading files.
+ */
+public class TestAliyunOSSInputStream {
+
+ private FileSystem fs;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestAliyunOSSInputStream.class);
+
+ private static String testRootPath =
+ AliyunOSSTestUtils.generateUniqueTestPath();
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fs = AliyunOSSTestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(new Path(testRootPath), true);
+ }
+ }
+
+ private Path setPath(String path) {
+ if (path.startsWith("/")) {
+ return new Path(testRootPath + path);
+ } else {
+ return new Path(testRootPath + "/" + path);
+ }
+ }
+
+ @Test
+ public void testSeekFile() throws Exception {
+ Path smallSeekFile = setPath("/test/smallSeekFile.txt");
+ long size = 5 * 1024 * 1024;
+
+ ContractTestUtils.generateTestFile(this.fs, smallSeekFile, size, 256, 255);
+ LOG.info("5MB file created: smallSeekFile.txt");
+
+ FSDataInputStream instream = this.fs.open(smallSeekFile);
+ int seekTimes = 5;
+ LOG.info("multiple fold position seeking test...:");
+ for (int i = 0; i < seekTimes; i++) {
+ long pos = size / (seekTimes - i) - 1;
+ LOG.info("begin seeking for pos: " + pos);
+ instream.seek(pos);
+ assertTrue("expected position at:" + pos + ", but got:"
+ + instream.getPos(), instream.getPos() == pos);
+ LOG.info("completed seeking at pos: " + instream.getPos());
+ }
+ LOG.info("random position seeking test...:");
+ Random rand = new Random();
+ for (int i = 0; i < seekTimes; i++) {
+ long pos = Math.abs(rand.nextLong()) % size;
+ LOG.info("begin seeking for pos: " + pos);
+ instream.seek(pos);
+ assertTrue("expected position at:" + pos + ", but got:"
+ + instream.getPos(), instream.getPos() == pos);
+ LOG.info("completed seeking at pos: " + instream.getPos());
+ }
+ IOUtils.closeStream(instream);
+ }
+
+ @Test
+ public void testReadFile() throws Exception {
+ final int bufLen = 256;
+ final int sizeFlag = 5;
+ String filename = "readTestFile_" + sizeFlag + ".txt";
+ Path readTestFile = setPath("/test/" + filename);
+ long size = sizeFlag * 1024 * 1024;
+
+ ContractTestUtils.generateTestFile(this.fs, readTestFile, size, 256, 255);
+ LOG.info(sizeFlag + "MB file created: /test/" + filename);
+
+ FSDataInputStream instream = this.fs.open(readTestFile);
+ byte[] buf = new byte[bufLen];
+ long bytesRead = 0;
+ while (bytesRead < size) {
+ int bytes;
+ if (size - bytesRead < bufLen) {
+ int remaining = (int)(size - bytesRead);
+ bytes = instream.read(buf, 0, remaining);
+ } else {
+ bytes = instream.read(buf, 0, bufLen);
+ }
+ bytesRead += bytes;
+
+ if (bytesRead % (1024 * 1024) == 0) {
+ int available = instream.available();
+ int remaining = (int)(size - bytesRead);
+ assertTrue("expected remaining:" + remaining + ", but got:" + available,
+ remaining == available);
+ LOG.info("Bytes read: " + Math.round((double)bytesRead / (1024 * 1024))
+ + " MB");
+ }
+ }
+ assertTrue(instream.available() == 0);
+ IOUtils.closeStream(instream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
new file mode 100644
index 0000000..6b87d9c
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+
+/**
+ * Tests regular and multi-part upload functionality for AliyunOSSOutputStream.
+ */
+public class TestAliyunOSSOutputStream {
+ private FileSystem fs;
+ private static String testRootPath =
+ AliyunOSSTestUtils.generateUniqueTestPath();
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024);
+ conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024);
+ fs = AliyunOSSTestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(new Path(testRootPath), true);
+ }
+ }
+
+ protected Path getTestPath() {
+ return new Path(testRootPath + "/test-aliyun-oss");
+ }
+
+ @Test
+ public void testRegularUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
+ }
+
+ @Test
+ public void testMultiPartUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024);
+ }
+
+ @Test
+ public void testMultiPartUploadLimit() throws IOException {
+ long partSize1 = AliyunOSSUtils.calculatePartSize(10 * 1024, 100 * 1024);
+ assert(10 * 1024 / partSize1 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize2 = AliyunOSSUtils.calculatePartSize(200 * 1024, 100 * 1024);
+ assert(200 * 1024 / partSize2 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize3 = AliyunOSSUtils.calculatePartSize(10000 * 100 * 1024,
+ 100 * 1024);
+ assert(10000 * 100 * 1024 / partSize3
+ < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize4 = AliyunOSSUtils.calculatePartSize(10001 * 100 * 1024,
+ 100 * 1024);
+ assert(10001 * 100 * 1024 / partSize4
+ < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java
new file mode 100644
index 0000000..e106aae
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSTemporaryCredentials.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.common.auth.Credentials;
+import com.aliyun.oss.common.auth.InvalidCredentialsException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.junit.Test;
+
+import java.net.URI;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SECRET_KEY;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
+
+/**
+ * Tests use of temporary credentials (for example, Aliyun STS & Aliyun OSS).
+ * This test extends a class that "does things to the root directory", and
+ * should only be used against transient filesystems where you don't care about
+ * the data.
+ */
+public class TestAliyunOSSTemporaryCredentials
+ extends AbstractFSContractTestBase {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+ @Test
+ public void testTemporaryCredentialValidation() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ACCESS_KEY, "accessKeyId");
+ conf.set(SECRET_KEY, "accessKeySecret");
+ conf.set(SECURITY_TOKEN, "");
+ URI uri = getFileSystem().getUri();
+ TemporaryAliyunCredentialsProvider provider
+ = new TemporaryAliyunCredentialsProvider(uri, conf);
+ try {
+ Credentials credentials = provider.getCredentials();
+ fail("Expected a CredentialInitializationException, got " + credentials);
+ } catch (InvalidCredentialsException expected) {
+ // expected
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
deleted file mode 100644
index f234d50..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemContract.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileSystemContractBaseTest;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- * Tests a live OSS system.
- *
- * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
- * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
- * properly making it impossible to skip the tests if we don't have a valid
- * bucket.
- */
-public class TestOSSFileSystemContract extends FileSystemContractBaseTest {
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestOSSFileSystemContract.class);
-
- public static final String TEST_FS_OSS_NAME = "test.fs.oss.name";
- private static String testRootPath = OSSTestUtils.generateUniqueTestPath();
-
- @Override
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- fs = OSSTestUtils.createTestFileSystem(conf);
- super.setUp();
- }
-
- @Override
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(super.path(testRootPath), true);
- }
- super.tearDown();
- }
-
- @Override
- protected Path path(String path) {
- if (path.startsWith("/")) {
- return super.path(testRootPath + path);
- } else {
- return super.path(testRootPath + "/" + path);
- }
- }
-
- @Override
- public void testMkdirsWithUmask() throws Exception {
- // not supported
- }
-
- @Override
- public void testRootDirAlwaysExists() throws Exception {
- //this will throw an exception if the path is not found
- fs.getFileStatus(super.path("/"));
- //this catches overrides of the base exists() method that don't
- //use getFileStatus() as an existence probe
- assertTrue("FileSystem.exists() fails for root",
- fs.exists(super.path("/")));
- }
-
- @Override
- public void testRenameRootDirForbidden() throws Exception {
- if (!renameSupported()) {
- return;
- }
- rename(super.path("/"),
- super.path("/test/newRootDir"),
- false, true, false);
- }
-
- public void testDeleteSubdir() throws IOException {
- Path parentDir = this.path("/test/hadoop");
- Path file = this.path("/test/hadoop/file");
- Path subdir = this.path("/test/hadoop/subdir");
- this.createFile(file);
-
- assertTrue("Created subdir", this.fs.mkdirs(subdir));
- assertTrue("File exists", this.fs.exists(file));
- assertTrue("Parent dir exists", this.fs.exists(parentDir));
- assertTrue("Subdir exists", this.fs.exists(subdir));
-
- assertTrue("Deleted subdir", this.fs.delete(subdir, true));
- assertTrue("Parent should exist", this.fs.exists(parentDir));
-
- assertTrue("Deleted file", this.fs.delete(file, false));
- assertTrue("Parent should exist", this.fs.exists(parentDir));
- }
-
-
- @Override
- protected boolean renameSupported() {
- return true;
- }
-
- @Override
- public void testRenameNonExistentPath() throws Exception {
- if (this.renameSupported()) {
- Path src = this.path("/test/hadoop/path");
- Path dst = this.path("/test/new/newpath");
- try {
- super.rename(src, dst, false, false, false);
- fail("Should throw FileNotFoundException!");
- } catch (FileNotFoundException e) {
- // expected
- }
- }
- }
-
- @Override
- public void testRenameFileMoveToNonExistentDirectory() throws Exception {
- if (this.renameSupported()) {
- Path src = this.path("/test/hadoop/file");
- this.createFile(src);
- Path dst = this.path("/test/new/newfile");
- try {
- super.rename(src, dst, false, true, false);
- fail("Should throw FileNotFoundException!");
- } catch (FileNotFoundException e) {
- // expected
- }
- }
- }
-
- @Override
- public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception {
- if (this.renameSupported()) {
- Path src = this.path("/test/hadoop/dir");
- this.fs.mkdirs(src);
- Path dst = this.path("/test/new/newdir");
- try {
- super.rename(src, dst, false, true, false);
- fail("Should throw FileNotFoundException!");
- } catch (FileNotFoundException e) {
- // expected
- }
- }
- }
-
- @Override
- public void testRenameFileMoveToExistingDirectory() throws Exception {
- super.testRenameFileMoveToExistingDirectory();
- }
-
- @Override
- public void testRenameFileAsExistingFile() throws Exception {
- if (this.renameSupported()) {
- Path src = this.path("/test/hadoop/file");
- this.createFile(src);
- Path dst = this.path("/test/new/newfile");
- this.createFile(dst);
- try {
- super.rename(src, dst, false, true, true);
- fail("Should throw FileAlreadyExistsException");
- } catch (FileAlreadyExistsException e) {
- // expected
- }
- }
- }
-
- @Override
- public void testRenameDirectoryAsExistingFile() throws Exception {
- if (this.renameSupported()) {
- Path src = this.path("/test/hadoop/dir");
- this.fs.mkdirs(src);
- Path dst = this.path("/test/new/newfile");
- this.createFile(dst);
- try {
- super.rename(src, dst, false, true, true);
- fail("Should throw FileAlreadyExistsException");
- } catch (FileAlreadyExistsException e) {
- // expected
- }
- }
- }
-
- public void testGetFileStatusFileAndDirectory() throws Exception {
- Path filePath = this.path("/test/oss/file1");
- this.createFile(filePath);
- assertTrue("Should be file", this.fs.getFileStatus(filePath).isFile());
- assertFalse("Should not be directory",
- this.fs.getFileStatus(filePath).isDirectory());
-
- Path dirPath = this.path("/test/oss/dir");
- this.fs.mkdirs(dirPath);
- assertTrue("Should be directory",
- this.fs.getFileStatus(dirPath).isDirectory());
- assertFalse("Should not be file", this.fs.getFileStatus(dirPath).isFile());
- }
-
- public void testMkdirsForExistingFile() throws Exception {
- Path testFile = this.path("/test/hadoop/file");
- assertFalse(this.fs.exists(testFile));
- this.createFile(testFile);
- assertTrue(this.fs.exists(testFile));
- try {
- this.fs.mkdirs(testFile);
- fail("Should throw FileAlreadyExistsException!");
- } catch (FileAlreadyExistsException e) {
- // expected
- }
- }
-
- public void testWorkingDirectory() throws Exception {
- Path workDir = super.path(this.getDefaultWorkingDirectory());
- assertEquals(workDir, this.fs.getWorkingDirectory());
- this.fs.setWorkingDirectory(super.path("."));
- assertEquals(workDir, this.fs.getWorkingDirectory());
- this.fs.setWorkingDirectory(super.path(".."));
- assertEquals(workDir.getParent(), this.fs.getWorkingDirectory());
- Path relativeDir = super.path("hadoop");
- this.fs.setWorkingDirectory(relativeDir);
- assertEquals(relativeDir, this.fs.getWorkingDirectory());
- Path absoluteDir = super.path("/test/hadoop");
- this.fs.setWorkingDirectory(absoluteDir);
- assertEquals(absoluteDir, this.fs.getWorkingDirectory());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java
deleted file mode 100644
index 6331ed8..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSFileSystemStore.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.*;
-import java.net.URI;
-import java.security.DigestInputStream;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeNotNull;
-
-/**
- * Test the bridging logic between Hadoop's abstract filesystem and
- * Aliyun OSS.
- */
-public class TestOSSFileSystemStore {
- private Configuration conf;
- private AliyunOSSFileSystemStore store;
- private AliyunOSSFileSystem fs;
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- fs = new AliyunOSSFileSystem();
- fs.initialize(URI.create(conf.get("test.fs.oss.name")), conf);
- store = fs.getStore();
- }
-
- @After
- public void tearDown() throws Exception {
- try {
- store.purge("test");
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @BeforeClass
- public static void checkSettings() throws Exception {
- Configuration conf = new Configuration();
- assumeNotNull(conf.get("fs.oss.accessKeyId"));
- assumeNotNull(conf.get("fs.oss.accessKeySecret"));
- assumeNotNull(conf.get("test.fs.oss.name"));
- }
-
- protected void writeRenameReadCompare(Path path, long len)
- throws IOException, NoSuchAlgorithmException {
- // If len > fs.oss.multipart.upload.threshold,
- // we'll use a multipart upload copy
- MessageDigest digest = MessageDigest.getInstance("MD5");
- OutputStream out = new BufferedOutputStream(
- new DigestOutputStream(fs.create(path, false), digest));
- for (long i = 0; i < len; i++) {
- out.write('Q');
- }
- out.flush();
- out.close();
-
- assertTrue("Exists", fs.exists(path));
-
- Path copyPath = path.suffix(".copy");
- fs.rename(path, copyPath);
-
- assertTrue("Copy exists", fs.exists(copyPath));
-
- // Download file from Aliyun OSS and compare the digest against the original
- MessageDigest digest2 = MessageDigest.getInstance("MD5");
- InputStream in = new BufferedInputStream(
- new DigestInputStream(fs.open(copyPath), digest2));
- long copyLen = 0;
- while (in.read() != -1) {
- copyLen++;
- }
- in.close();
-
- assertEquals("Copy length matches original", len, copyLen);
- assertArrayEquals("Digests match", digest.digest(), digest2.digest());
- }
-
- @Test
- public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
- // Regular upload, regular copy
- writeRenameReadCompare(new Path("/test/small"), 16384);
- }
-
- @Test
- public void testLargeUpload()
- throws IOException, NoSuchAlgorithmException {
- // Multipart upload, multipart copy
- writeRenameReadCompare(new Path("/test/xlarge"), 52428800L); // 50MB byte
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java
deleted file mode 100644
index 411cd57..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSInputStream.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.*;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Random;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests basic functionality for AliyunOSSInputStream, including seeking and
- * reading files.
- */
-public class TestOSSInputStream {
-
- private FileSystem fs;
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestOSSInputStream.class);
-
- private static String testRootPath = OSSTestUtils.generateUniqueTestPath();
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- fs = OSSTestUtils.createTestFileSystem(conf);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(new Path(testRootPath), true);
- }
- }
-
- private Path setPath(String path) {
- if (path.startsWith("/")) {
- return new Path(testRootPath + path);
- } else {
- return new Path(testRootPath + "/" + path);
- }
- }
-
- @Test
- public void testSeekFile() throws Exception {
- Path smallSeekFile = setPath("/test/smallSeekFile.txt");
- long size = 5 * 1024 * 1024;
-
- ContractTestUtils.generateTestFile(this.fs, smallSeekFile, size, 256, 255);
- LOG.info("5MB file created: smallSeekFile.txt");
-
- FSDataInputStream instream = this.fs.open(smallSeekFile);
- int seekTimes = 5;
- LOG.info("multiple fold position seeking test...:");
- for (int i = 0; i < seekTimes; i++) {
- long pos = size / (seekTimes - i) - 1;
- LOG.info("begin seeking for pos: " + pos);
- instream.seek(pos);
- assertTrue("expected position at:" + pos + ", but got:"
- + instream.getPos(), instream.getPos() == pos);
- LOG.info("completed seeking at pos: " + instream.getPos());
- }
- LOG.info("random position seeking test...:");
- Random rand = new Random();
- for (int i = 0; i < seekTimes; i++) {
- long pos = Math.abs(rand.nextLong()) % size;
- LOG.info("begin seeking for pos: " + pos);
- instream.seek(pos);
- assertTrue("expected position at:" + pos + ", but got:"
- + instream.getPos(), instream.getPos() == pos);
- LOG.info("completed seeking at pos: " + instream.getPos());
- }
- IOUtils.closeStream(instream);
- }
-
- @Test
- public void testReadFile() throws Exception {
- final int bufLen = 256;
- final int sizeFlag = 5;
- String filename = "readTestFile_" + sizeFlag + ".txt";
- Path readTestFile = setPath("/test/" + filename);
- long size = sizeFlag * 1024 * 1024;
-
- ContractTestUtils.generateTestFile(this.fs, readTestFile, size, 256, 255);
- LOG.info(sizeFlag + "MB file created: /test/" + filename);
-
- FSDataInputStream instream = this.fs.open(readTestFile);
- byte[] buf = new byte[bufLen];
- long bytesRead = 0;
- while (bytesRead < size) {
- int bytes;
- if (size - bytesRead < bufLen) {
- int remaining = (int)(size - bytesRead);
- bytes = instream.read(buf, 0, remaining);
- } else {
- bytes = instream.read(buf, 0, bufLen);
- }
- bytesRead += bytes;
-
- if (bytesRead % (1024 * 1024) == 0) {
- int available = instream.available();
- int remaining = (int)(size - bytesRead);
- assertTrue("expected remaining:" + remaining + ", but got:" + available,
- remaining == available);
- LOG.info("Bytes read: " + Math.round((double)bytesRead / (1024 * 1024))
- + " MB");
- }
- }
- assertTrue(instream.available() == 0);
- IOUtils.closeStream(instream);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
deleted file mode 100644
index b33ab99..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSOutputStream.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import java.io.IOException;
-
-/**
- * Tests regular and multi-part upload functionality for AliyunOSSOutputStream.
- */
-public class TestOSSOutputStream {
- private FileSystem fs;
- private static String testRootPath = OSSTestUtils.generateUniqueTestPath();
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024);
- conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024);
- fs = OSSTestUtils.createTestFileSystem(conf);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(new Path(testRootPath), true);
- }
- }
-
- protected Path getTestPath() {
- return new Path(testRootPath + "/testoss");
- }
-
- @Test
- public void testRegularUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
- }
-
- @Test
- public void testMultiPartUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024);
- }
-
- @Test
- public void testMultiPartUploadLimit() throws IOException {
- long partSize1 = AliyunOSSUtils.calculatePartSize(10 * 1024, 100 * 1024);
- assert(10 * 1024 / partSize1 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
-
- long partSize2 = AliyunOSSUtils.calculatePartSize(200 * 1024, 100 * 1024);
- assert(200 * 1024 / partSize2 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
-
- long partSize3 = AliyunOSSUtils.calculatePartSize(10000 * 100 * 1024,
- 100 * 1024);
- assert(10000 * 100 * 1024 / partSize3
- < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
-
- long partSize4 = AliyunOSSUtils.calculatePartSize(10001 * 100 * 1024,
- 100 * 1024);
- assert(10001 * 100 * 1024 / partSize4
- < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java
deleted file mode 100644
index ca2c0bc..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestOSSTemporaryCredentials.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss;
-
-import com.aliyun.oss.common.auth.Credentials;
-import com.aliyun.oss.common.auth.InvalidCredentialsException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.aliyun.oss.contract.OSSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.junit.Test;
-
-import java.net.URI;
-
-import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY;
-import static org.apache.hadoop.fs.aliyun.oss.Constants.SECRET_KEY;
-import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
-
-/**
- * Tests use of temporary credentials (for example, Aliyun STS & Aliyun OSS).
- * This test extends a class that "does things to the root directory", and
- * should only be used against transient filesystems where you don't care about
- * the data.
- */
-public class TestOSSTemporaryCredentials extends AbstractFSContractTestBase {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-
- @Test
- public void testTemporaryCredentialValidation() throws Throwable {
- Configuration conf = new Configuration();
- conf.set(ACCESS_KEY, "accessKeyId");
- conf.set(SECRET_KEY, "accessKeySecret");
- conf.set(SECURITY_TOKEN, "");
- URI uri = getFileSystem().getUri();
- TemporaryAliyunCredentialsProvider provider
- = new TemporaryAliyunCredentialsProvider(uri, conf);
- try {
- Credentials credentials = provider.getCredentials();
- fail("Expected a CredentialInitializationException, got " + credentials);
- } catch (InvalidCredentialsException expected) {
- // expected
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
new file mode 100644
index 0000000..624c606
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+
+/**
+ * The contract of Aliyun OSS: only enabled if the test bucket is provided.
+ */
+public class AliyunOSSContract extends AbstractBondedFSContract {
+
+ public static final String CONTRACT_XML = "contract/aliyun-oss.xml";
+
+ public AliyunOSSContract(Configuration conf) {
+ super(conf);
+ //insert the base features
+ addConfResource(CONTRACT_XML);
+ }
+
+ @Override
+ public String getScheme() {
+ return "oss";
+ }
+
+ @Override
+ public Path getTestPath() {
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ return testUniqueForkId == null ? super.getTestPath() :
+ new Path("/" + testUniqueForkId, "test");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
deleted file mode 100644
index f90a8bb..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/OSSContract.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.aliyun.oss.OSSTestUtils;
-import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
-
-/**
- * The contract of OSS: only enabled if the test bucket is provided.
- */
-public class OSSContract extends AbstractBondedFSContract {
-
- public static final String CONTRACT_XML = "contract/oss.xml";
- public static final String CONTRACT_TEST_OSS_FS_NAME =
- "fs.contract.test.fs.oss";
-
- private static String testPath = OSSTestUtils.generateUniqueTestPath();
-
- public OSSContract(Configuration conf) {
- super(conf);
- //insert the base features
- addConfResource(CONTRACT_XML);
- }
-
- @Override
- public String getScheme() {
- return "oss";
- }
-
- @Override
- public Path getTestPath() {
- Path path = new Path(testPath);
- return path;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java
new file mode 100644
index 0000000..88dd8cd
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract creating tests.
+ */
+public class TestAliyunOSSContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java
new file mode 100644
index 0000000..1658d80
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract deleting tests.
+ */
+public class TestAliyunOSSContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java
new file mode 100644
index 0000000..4b482fc
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDispCp.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Contract test suite covering Aliyun OSS integration with DistCp.
+ */
+public class TestAliyunOSSContractDispCp extends AbstractContractDistCpTest {
+
+ private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ newConf.setLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MULTIPART_SETTING);
+ newConf.setLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_SETTING);
+ return newConf;
+ }
+
+ @Override
+ protected AliyunOSSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java
new file mode 100644
index 0000000..c69124d
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test getFileStatus and related listing operations.
+ */
+public class TestAliyunOSSContractGetFileStatus
+ extends AbstractContractGetFileStatusTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java
new file mode 100644
index 0000000..6cb7549
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract directory tests.
+ */
+public class TestAliyunOSSContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java
new file mode 100644
index 0000000..099aba6
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract opening file tests.
+ */
+public class TestAliyunOSSContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java
new file mode 100644
index 0000000..e15b3ba
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract renaming tests.
+ */
+public class TestAliyunOSSContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java
new file mode 100644
index 0000000..9faae37
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Root dir operations against an Aliyun OSS bucket.
+ */
+public class TestAliyunOSSContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestAliyunOSSContractRootDir.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+ @Override
+ public void testListEmptyRootDirectory() throws IOException {
+ for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
+ try {
+ super.testListEmptyRootDirectory();
+ break;
+ } catch (AssertionError | FileNotFoundException e) {
+ if (attempt < maxAttempts) {
+ LOG.info("Attempt {} of {} for empty root directory test failed. "
+ + "Attempting retry.", attempt, maxAttempts);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e2) {
+ Thread.currentThread().interrupt();
+ fail("Test interrupted.");
+ break;
+ }
+ } else {
+ LOG.error(
+ "Empty root directory test failed {} attempts. Failing test.",
+ maxAttempts);
+ throw e;
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java
new file mode 100644
index 0000000..b247ab1
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract seeking tests.
+ */
+public class TestAliyunOSSContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
deleted file mode 100644
index ce927a9..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractCreate.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * OSS contract creating tests.
- */
-public class TestOSSContractCreate extends AbstractContractCreateTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1940464/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java
deleted file mode 100644
index 6a1eb40..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestOSSContractDelete.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * OSS contract deleting tests.
- */
-public class TestOSSContractDelete extends AbstractContractDeleteTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OSSContract(conf);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org