You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by el...@apache.org on 2017/11/27 23:16:56 UTC
phoenix git commit: PHOENIX-4405 Switch to using Hadoop3
Repository: phoenix
Updated Branches:
refs/heads/5.x-HBase-2.0 309ef0b1a -> 8e5b4131b
PHOENIX-4405 Switch to using Hadoop3
Includes a temporary fix to work around a jetty dependency conflict
between Hadoop and HBase
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8e5b4131
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8e5b4131
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8e5b4131
Branch: refs/heads/5.x-HBase-2.0
Commit: 8e5b4131b4b9bbbf9aab43bf304504e8238027f0
Parents: 309ef0b
Author: Josh Elser <el...@apache.org>
Authored: Mon Nov 27 15:25:42 2017 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Nov 27 15:55:40 2017 -0500
----------------------------------------------------------------------
phoenix-core/pom.xml | 21 ++++++++
.../phoenix/trace/PhoenixMetricsSink.java | 2 +-
.../recovery/TestPerRegionIndexWriteCache.java | 31 ++++++++++--
.../org/apache/phoenix/metrics/LoggingSink.java | 2 +-
phoenix-spark/pom.xml | 8 +--
phoenix-tracing-webapp/pom.xml | 16 ++++--
pom.xml | 52 ++++++++++++++------
7 files changed, 104 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 3124d9b..0d5affe 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -418,6 +418,27 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- TODO remove after HBASE-19256 -->
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
index fea6d61..65071f0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
@@ -34,7 +34,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
-import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.configuration2.SubsetConfiguration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseConfiguration;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
index 2bfbcbf..c51ac99 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
@@ -43,11 +43,16 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
@@ -60,6 +65,8 @@ public class TestPerRegionIndexWriteCache {
private static final byte[] qual = Bytes.toBytes("qual");
private static final byte[] val = Bytes.toBytes("val");
+ private static MiniDFSCluster miniDfs = null;
+
Put p = new Put(row);
Put p2 = new Put(Bytes.toBytes("other row"));
{
@@ -71,18 +78,34 @@ public class TestPerRegionIndexWriteCache {
HRegion r2; // FIXME: Uses private type
WAL wal;
+ @Rule
+ public TestName testName = new TestName();
+
+ @BeforeClass
+ public static void startDfs() throws Exception {
+ miniDfs = TEST_UTIL.startMiniDFSCluster(1);
+ }
+
+ @AfterClass
+ public static void stopDfs() throws Exception {
+ if (miniDfs != null) {
+ miniDfs.shutdown();
+ miniDfs = null;
+ }
+ }
+
@SuppressWarnings("deprecation")
-@Before
+ @Before
public void setUp() throws Exception {
- Path hbaseRootDir = TEST_UTIL.getDataTestDir();
+ Path hbaseRootDir = new Path(getClass().getSimpleName() + "_" + testName.getMethodName());
TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString());
- FileSystem newFS = FileSystem.newInstance(TEST_UTIL.getConfiguration());
+ FileSystem newFS = miniDfs.getFileSystem();
RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(null).setEndKey(null).setSplit(false).build();
Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
Random rn = new Random();
tableName = TableName.valueOf("TestPerRegion" + rn.nextInt());
- WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestPerRegionIndexWriteCache");
+ WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, getClass().getSimpleName());
wal = walFactory.getWAL(Bytes.toBytes("logs"), null);
TableDescriptor htd =
TableDescriptorBuilder
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java b/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java
index 2cea684..1af01ab 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java
@@ -17,7 +17,7 @@
*/
package org.apache.phoenix.metrics;
-import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.configuration2.SubsetConfiguration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.metrics2.AbstractMetric;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index f3bd9c2..d6ad4d5 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -102,7 +102,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
@@ -134,7 +134,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
@@ -166,7 +166,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
@@ -200,7 +200,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-tracing-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-tracing-webapp/pom.xml b/phoenix-tracing-webapp/pom.xml
index 6cc9960..1cb047f 100755
--- a/phoenix-tracing-webapp/pom.xml
+++ b/phoenix-tracing-webapp/pom.xml
@@ -41,15 +41,23 @@
<dependencies>
<dependency>
<groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
- <!-- TODO : pick the same jetty version throughout the project (PHOENIX-2211)-->
- <version>${jettyVersion}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
- <!-- TODO : pick the same jetty version throughout the project (PHOENIX-2211)-->
- <version>${jettyVersion}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bef97db..ac5cd08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,12 +67,11 @@
<!-- Hadoop Versions -->
<hbase.version>2.0.0-beta-1-SNAPSHOT</hbase.version>
- <hadoop-two.version>2.7.1</hadoop-two.version>
+ <hadoop.version>3.0.0-beta1</hadoop.version>
<!-- Dependency versions -->
<commons-cli.version>1.2</commons-cli.version>
<hive.version>1.2.1</hive.version>
- <hadoop.version>2.7.1</hadoop.version>
<pig.version>0.13.0</pig.version>
<jackson.version>1.9.2</jackson.version>
<antlr.version>3.5.2</antlr.version>
@@ -99,7 +98,7 @@
<jodatime.version>1.6</jodatime.version>
<joni.version>2.1.2</joni.version>
<avatica.version>1.10.0</avatica.version>
- <jettyVersion>8.1.7.v20120910</jettyVersion>
+ <jetty.version>9.3.19.v20170502</jetty.version>
<tephra.version>0.13.0-incubating</tephra.version>
<spark.version>2.0.2</spark.version>
<scala.version>2.11.8</scala.version>
@@ -655,7 +654,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>org.xerial.snappy</groupId>
@@ -666,17 +665,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
<optional>true</optional>
<scope>test</scope>
</dependency>
@@ -685,39 +684,39 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
<type>test-jar</type> <!-- this does not work which is typical for maven.-->
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
- <version>${hadoop-two.version}</version>
+ <version>${hadoop.version}</version>
</dependency>
<!-- General Dependencies -->
@@ -945,6 +944,31 @@
<artifactId>i18n-util</artifactId>
<version>${i18n-util.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>