You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2019/10/02 17:06:13 UTC
[hbase-filesystem] branch master updated: HBASE-23099 - [HBOSS]
Raise hadoop dependency version to 3.2.1 (#9)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-filesystem.git
The following commit(s) were added to refs/heads/master by this push:
new 2e2628e HBASE-23099 - [HBOSS] Raise hadoop dependency version to 3.2.1 (#9)
2e2628e is described below
commit 2e2628e9c54c7b77e8dde34395ba52d177051033
Author: Wellington Ramos Chevreuil <wc...@apache.org>
AuthorDate: Wed Oct 2 18:06:08 2019 +0100
HBASE-23099 - [HBOSS] Raise hadoop dependency version to 3.2.1 (#9)
HBASE-23099 - [HBOSS] Raise hadoop dependency version to 3.2.1
There's been some changes on S3Guard that comes with hadoop 3.2.1 and upcoming versions, such as HADOOP-16085, HADOOP-16233 and HADOOP-16380, among other recent commits. Some of these required additional properties or modified S3A expected behaviour for a few FS contract tests.
Signed-off-by: Josh Elser <el...@apache.org>
---
.../hbase/oss/HBaseObjectStoreSemanticsTest.java | 1 +
.../org/apache/hadoop/hbase/oss/TestUtils.java | 14 +++++++++-
.../hbase/oss/contract/TestHBOSSContract.java | 1 +
.../hbase/oss/contract/TestHBOSSContractOpen.java | 10 +++++++-
...ractOpen.java => TestHBOSSContractOpenS3A.java} | 21 +++++++++++++--
.../oss/contract/TestHBOSSContractRename.java | 12 +--------
.../oss/contract/TestHBOSSContractRenameS3A.java | 12 +--------
.../contract/TestHBOSSContractRootDirectory.java | 8 +++---
...java => TestHBOSSContractRootDirectoryS3A.java} | 30 ++++++++++------------
hbase-oss/src/test/resources/contract/s3a.xml | 4 +++
pom.xml | 2 +-
11 files changed, 68 insertions(+), 47 deletions(-)
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemanticsTest.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemanticsTest.java
index 1bb3710..21299db 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemanticsTest.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemanticsTest.java
@@ -52,6 +52,7 @@ public class HBaseObjectStoreSemanticsTest {
@Before
public void setup() throws Exception {
Configuration conf = new Configuration();
+ conf.addResource("contract/s3a.xml");
hboss = TestUtils.getFileSystem(conf);
sync = hboss.getLockManager();
hboss.mkdirs(testPathRoot());
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
index 30f116f..9e4f5d9 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
@@ -102,7 +102,6 @@ public class TestUtils {
return hboss;
} catch (Exception e) {
LOG.error(e.getMessage());
- e.printStackTrace();
throw e;
}
}
@@ -117,4 +116,17 @@ public class TestUtils {
}
}
}
+
+ public static void runIfS3(boolean isS3, Configuration conf) {
+ try {
+ TestUtils.getFileSystem(conf);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ if(isS3) {
+ Assume.assumeTrue(TestUtils.fsIs(TestUtils.S3A, conf));
+ } else {
+ Assume.assumeFalse(TestUtils.fsIs(TestUtils.S3A, conf));
+ }
+ }
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
index 6ad69ca..4ed4be0 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
@@ -52,6 +52,7 @@ public class TestHBOSSContract extends FileSystemContractBaseTest {
public void setUp() throws Exception {
nameThread();
conf = new Configuration();
+ conf.addResource("contract/s3a.xml");
fs = TestUtils.getFileSystem(conf);
Assume.assumeNotNull(fs);
HBaseObjectStoreSemantics hboss = (HBaseObjectStoreSemantics)fs;
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java
index 36adfcf..33d684d 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java
@@ -21,7 +21,13 @@ package org.apache.hadoop.hbase.oss.contract;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.hbase.oss.TestUtils;
+/**
+ * There is an S3A-specific extension of AbstractContractOpenTest
+ * that is similarly extended for HBOSS-on-S3A. This class remains to be run in
+ * the general case.
+ */
public class TestHBOSSContractOpen extends AbstractContractOpenTest {
@Override
@@ -31,6 +37,8 @@ public class TestHBOSSContractOpen extends AbstractContractOpenTest {
@Override
protected AbstractFSContract createContract(Configuration conf) {
- return new HBOSSContract(conf);
+ AbstractFSContract contract = new HBOSSContract(conf);
+ TestUtils.runIfS3(false, conf);
+ return contract;
}
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpenS3A.java
similarity index 66%
copy from hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java
copy to hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpenS3A.java
index 36adfcf..cbf5b65 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpen.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractOpenS3A.java
@@ -21,8 +21,15 @@ package org.apache.hadoop.hbase.oss.contract;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.hbase.oss.TestUtils;
-public class TestHBOSSContractOpen extends AbstractContractOpenTest {
+
+/**
+ * For S3A-specific extension of AbstractContractOpenTest, we needed to override
+ * areZeroByteFilesEncrypted() method to always return true.
+ * TestHBOSSContractOpen remains to be run in the general case.
+ */
+public class TestHBOSSContractOpenS3A extends AbstractContractOpenTest {
@Override
protected Configuration createConfiguration() {
@@ -31,6 +38,16 @@ public class TestHBOSSContractOpen extends AbstractContractOpenTest {
@Override
protected AbstractFSContract createContract(Configuration conf) {
- return new HBOSSContract(conf);
+ AbstractFSContract contract = new HBOSSContract(conf);
+ TestUtils.runIfS3(true, conf);
+ return contract;
+ }
+
+ /**
+ * S3A always declares zero byte files as encrypted.
+ * @return true, always.
+ */
+ protected boolean areZeroByteFilesEncrypted() {
+ return true;
}
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java
index f1047c4..2ee68af 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java
@@ -18,14 +18,10 @@
package org.apache.hadoop.hbase.oss.contract;
-import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.hbase.oss.EmbeddedS3;
import org.apache.hadoop.hbase.oss.TestUtils;
-import org.junit.Assume;
-import org.junit.Test;
/**
* There is an S3A-specific extension of AbstractContractRenameTest
@@ -42,13 +38,7 @@ public class TestHBOSSContractRename extends AbstractContractRenameTest {
@Override
protected AbstractFSContract createContract(Configuration conf) {
HBOSSContract contract = new HBOSSContract(conf);
- try {
- TestUtils.getFileSystem(conf);
- } catch (Exception e) {
- e.printStackTrace();
- fail("Exception configuring FS: " + e);
- }
- Assume.assumeFalse(TestUtils.fsIs(TestUtils.S3A, conf));
+ TestUtils.runIfS3(false, conf);
return contract;
}
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRenameS3A.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRenameS3A.java
index a441dbf..cb991fb 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRenameS3A.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRenameS3A.java
@@ -18,17 +18,13 @@
package org.apache.hadoop.hbase.oss.contract;
-import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.hbase.oss.EmbeddedS3;
import org.apache.hadoop.hbase.oss.TestUtils;
-import org.junit.Assume;
-import org.junit.Test;
/**
* There is an S3A-specific extension of AbstractContractRenameTest, and this
@@ -45,13 +41,7 @@ public class TestHBOSSContractRenameS3A extends AbstractContractRenameTest {
@Override
protected AbstractFSContract createContract(Configuration conf) {
HBOSSContract contract = new HBOSSContract(conf);
- try {
- TestUtils.getFileSystem(conf);
- } catch (Exception e) {
- e.printStackTrace();
- fail("Exception configuring FS: " + e);
- }
- Assume.assumeTrue(TestUtils.fsIs(TestUtils.S3A, conf));
+ TestUtils.runIfS3(true, conf);
return contract;
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectory.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectory.java
index 189693a..f14036c 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectory.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectory.java
@@ -18,12 +18,10 @@
package org.apache.hadoop.hbase.oss.contract;
-import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.oss.TestUtils;
public class TestHBOSSContractRootDirectory extends AbstractContractRootDirectoryTest {
@@ -34,6 +32,8 @@ public class TestHBOSSContractRootDirectory extends AbstractContractRootDirector
@Override
protected AbstractFSContract createContract(Configuration conf) {
- return new HBOSSContract(conf);
+ HBOSSContract contract = new HBOSSContract(conf);
+ TestUtils.runIfS3(false, conf);
+ return contract;
}
}
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectoryS3A.java
similarity index 62%
copy from hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java
copy to hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectoryS3A.java
index f1047c4..7fa151a 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRename.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContractRootDirectoryS3A.java
@@ -18,21 +18,19 @@
package org.apache.hadoop.hbase.oss.contract;
-import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.hbase.oss.EmbeddedS3;
import org.apache.hadoop.hbase.oss.TestUtils;
-import org.junit.Assume;
-import org.junit.Test;
+import org.junit.Ignore;
/**
- * There is an S3A-specific extension of AbstractContractRenameTest
- * that is similarly extended for HBOSS-on-S3A. This class remains to be run in
- * the general case.
+ * For S3A-specific extension of AbstractContractRootDirectoryTest, this overrides
+ * testRmNonEmptyRootDirNonRecursive(), marking it as ignored, since
+ * HADOOP-16380 introduced changes on S3AFileSystem for delete not throw any exception.
+ * TestHBOSSContractRootDirectory remains to be run in the general case.
*/
-public class TestHBOSSContractRename extends AbstractContractRenameTest {
+public class TestHBOSSContractRootDirectoryS3A extends AbstractContractRootDirectoryTest {
@Override
protected Configuration createConfiguration() {
@@ -42,13 +40,13 @@ public class TestHBOSSContractRename extends AbstractContractRenameTest {
@Override
protected AbstractFSContract createContract(Configuration conf) {
HBOSSContract contract = new HBOSSContract(conf);
- try {
- TestUtils.getFileSystem(conf);
- } catch (Exception e) {
- e.printStackTrace();
- fail("Exception configuring FS: " + e);
- }
- Assume.assumeFalse(TestUtils.fsIs(TestUtils.S3A, conf));
+ TestUtils.runIfS3(true, conf);
return contract;
}
+
+ //HADOOP-16380 introduced changes on S3AFileSystem for delete not throw any exception
+ @Override
+ @Ignore("S3 always return false when non-recursively remove root dir")
+ public void testRmNonEmptyRootDirNonRecursive() throws Throwable {
+ }
}
diff --git a/hbase-oss/src/test/resources/contract/s3a.xml b/hbase-oss/src/test/resources/contract/s3a.xml
index 72aac03..98389a1 100644
--- a/hbase-oss/src/test/resources/contract/s3a.xml
+++ b/hbase-oss/src/test/resources/contract/s3a.xml
@@ -125,4 +125,8 @@
<value>false</value>
</property>
+ <property>
+ <name>fs.s3a.change.detection.version.required</name>
+ <value>false</value>
+ </property>
</configuration>
diff --git a/pom.xml b/pom.xml
index 1453038..1e822ff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,7 +51,7 @@
<curator.version>4.2.0</curator.version>
<guava.version>23.0</guava.version>
<hadoop2.version>2.9.2</hadoop2.version>
- <hadoop3.version>3.2.0</hadoop3.version>
+ <hadoop3.version>3.2.1</hadoop3.version>
<hbase1.version>1.4.10</hbase1.version>
<hbase2.version>2.1.4</hbase2.version>
<hbase-thirdparty.version>2.2.1</hbase-thirdparty.version>