You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2019/05/22 18:58:41 UTC
[hbase-filesystem] branch master updated: HBASE-22437 HBOSS: Add
Hadoop 2 / 3 profiles.
This is an automated email from the ASF dual-hosted git repository.
elserj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-filesystem.git
The following commit(s) were added to refs/heads/master by this push:
new 9045272 HBASE-22437 HBOSS: Add Hadoop 2 / 3 profiles.
9045272 is described below
commit 9045272c61b902ef68d472383c1d07630a51f154
Author: Sean Mackrory <ma...@apache.org>
AuthorDate: Wed May 22 12:30:28 2019 -0400
HBASE-22437 HBOSS: Add Hadoop 2 / 3 profiles.
Signed-off-by: wellington <wc...@cloudera.com>
Amending-Author: Josh Elser <el...@apache.org>
---
hbase-oss/README.md | 28 ++++++++++--
.../org/apache/hadoop/hbase/oss/sync/AutoLock.java | 13 +++++-
.../hbase/oss/contract/TestHBOSSContract.java | 31 ++++++++++++--
pom.xml | 50 +++++++++++++++++++++-
4 files changed, 111 insertions(+), 11 deletions(-)
diff --git a/hbase-oss/README.md b/hbase-oss/README.md
index 1d100af..3b71ee9 100644
--- a/hbase-oss/README.md
+++ b/hbase-oss/README.md
@@ -76,12 +76,20 @@ use and as such there's a dependency on DynamoDB anyway.
## Storage Implementations
Currently HBOSS is primarily designed for and exclusively tested with Hadoop's
-s3a client against Amazon S3. S3Guard must be enabled. Both this requirement and
-the use of an external data store for locking have serious implications if any
-other client accesses the same data.
+s3a client against Amazon S3. *S3Guard must be enabled, which is available in
+Hadoop 2.9.0, 3.0.0, and higher*.
+
+Both the use of S3Guard and Zookeeper for locking (i.e. Zookeeper) have
+implications for other clients that are not configured to share the same
+metadata store and Zookeeper ensemble. Ideally, all clients should be have the
+same configuration in these respects. Read-only clients may not share these
+resources with the HBase processes, but they will not have the added safety
+provided by these features. Clients that do not share these resources and modify
+data can compromise the correctness of HBase.
+
In theory, HBOSS could also work well with Google's cloud storage client (gs)
-or other object storage clients.
+or other object storage clients, but this has not been tested.
## FileSystem Instantiation
@@ -121,3 +129,15 @@ other storage in src/test/resources/core-site.xml.
Any required credentials or other individal configuration should be set in
src/test/resources/auth-keys.xml, which should be ignored by source control.
+
+### Hadoop Versions
+
+There are Maven profiles defined for Hadoop 2 and Hadoop 3 major versions.
+These are activated via the property `hadoop.profile`. These profiles choose
+a specific Hadoop release in that major line, defaulting to versions as defined
+in `hadoop2.version` and `hadoop3.version`. By default, Hadoop 3 is used by
+the build.
+
+ mvn verify # Defaults to Hadoop 3
+ mvn verify -Dhadoop.profile=3 # Activate Hadoop 3
+ mvn verify -Dhadoop.profile=2 # Activate Hadoop 2
diff --git a/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/sync/AutoLock.java b/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/sync/AutoLock.java
index 3b57d20..18eb8cf 100644
--- a/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/sync/AutoLock.java
+++ b/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/sync/AutoLock.java
@@ -131,10 +131,19 @@ public interface AutoLock extends AutoCloseable {
* Returns the position in the wrapped stream. This should not be accessed
* after the stream has been closed. Unlike most other functions in this
* class, this is not enforced because this function shouldn't throw
- * IOExceptions.
+ * IOExceptions in Hadoop 3
+ *
+ * FSDataOutputStream.getPos() declares that it can throw IOExceptions in Hadoop
+ * 2, but the implementation never does. So it could, in theory, but no
+ * situation in which it actually would is know.
*/
public long getPos() {
- return stream.getPos();
+ try {
+ return stream.getPos();
+ } catch (Exception e) {
+ // We can't specify IOException and still compile against Hadoop 3
+ throw new RuntimeException(e);
+ }
}
@Override
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
index 1ba31f9..6ad69ca 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/contract/TestHBOSSContract.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.hbase.oss.contract;
+import java.lang.reflect.Method;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.oss.HBaseObjectStoreSemantics;
import org.apache.hadoop.hbase.oss.TestUtils;
-import org.junit.Assume;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -65,7 +66,11 @@ public class TestHBOSSContract extends FileSystemContractBaseTest {
@Test
public void testMkdirsWithUmask() throws Exception {
// Skipped in the hadoop-aws tests
- Assume.assumeFalse(TestUtils.fsIs(TestUtils.S3A, conf));
+ if (TestUtils.fsIs(TestUtils.S3A, conf)) {
+ // It would be nice to use Assume.assumeFalse instead of if, but Hadoop 2
+ // builds pull in JUnit 3, and this is the only way to skip the test.
+ return;
+ }
super.testMkdirsWithUmask();
}
@@ -100,7 +105,25 @@ public class TestHBOSSContract extends FileSystemContractBaseTest {
@Test
public void testMoveDirUnderParent() throws Throwable {
// Skipped in the hadoop-aws tests
- Assume.assumeFalse(TestUtils.fsIs(TestUtils.S3A, conf));
- super.testMoveDirUnderParent();
+ if (TestUtils.fsIs(TestUtils.S3A, conf)) {
+ // It would be nice to use Assume.assumeFalse instead of if, but Hadoop 2
+ // builds pull in JUnit 3, and this is the only way to skip the test.
+ return;
+ }
+
+ // Can't just call super.testMoveDirUnderParent() because it doesn't
+ // exist in older Hadoop versions
+ String methodName = "testMoveDirUnderParent";
+ Method method = null;
+ boolean skip = false;
+ try {
+ method = super.getClass().getMethod(methodName, (Class<?>[]) null);
+ } catch (NoSuchMethodException e) {
+ skip = true;
+ }
+ Assume.assumeFalse("Unable to find method " + methodName, skip);
+ if (!skip) {
+ method.invoke(this, (Object[]) null);
+ }
}
}
diff --git a/pom.xml b/pom.xml
index ec62ba5..c14a6b5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,7 +39,8 @@
<commons-io.version>2.5</commons-io.version>
<commons-lang3.version>3.6</commons-lang3.version>
<curator.version>4.0.0</curator.version>
- <hadoop.version>3.2.0</hadoop.version>
+ <hadoop2.version>2.9.2</hadoop2.version>
+ <hadoop3.version>3.2.0</hadoop3.version>
<hbase.version>2.1.4</hbase.version>
<hbase-thirdparty.version>2.2.0</hbase-thirdparty.version>
<junit.version>4.12</junit.version>
@@ -68,4 +69,51 @@
</plugins>
</pluginManagement>
</build>
+
+ <profiles>
+ <profile>
+ <id>hadoop2</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>${hadoop2.version}</hadoop.version>
+ </properties>
+ <dependencies>
+ <!-- TestHBOSSContractDistCp hard-codes some HDFS classes in Hadoop 2 -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>hadoop3</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>3</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>${hadoop3.version}</hadoop.version>
+ </properties>
+ </profile>
+ <profile>
+ <id>hadoop-default</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>${hadoop3.version}</hadoop.version>
+ </properties>
+ </profile>
+ </profiles>
</project>