You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2023/11/30 11:26:53 UTC
(phoenix-connectors) branch master updated (26c1aba -> d032bd5)
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
from 26c1aba PHOENIX-7114 Use Phoenix 5.1.3 for building Connectiors
new b2a283a PHOENIX-7122 Build connectors with Hbase 2.4
new e7ad177 PHOENIX-7065 Spark3 connector tests fail with Spark 3.4.1
new 592803e PHOENIX-7119 Java ITs are not run for either Spark Connector
new 17f408c PHOENIX-6923 Always run Scala tests for Phoenix-Spark connector
new bfbfb8d PHOENIX-7118 Fix Shading Regressions in Spark Connector
new af10459 PHOENIX-6939 Change phoenix-hive connector shading to work with hbase-shaded-mapreduce
new d032bd5 PHOENIX-7124 Build connectors with Spark 2.4.8, Spark 3.2.4, and Hive 3.1.3
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
phoenix5-hive-shaded/pom.xml | 248 ++++++++++++++++---
phoenix5-hive/pom.xml | 22 +-
phoenix5-spark-shaded/pom.xml | 271 +++++++++++++++++++--
phoenix5-spark/README.md | 18 +-
phoenix5-spark/pom.xml | 37 +--
.../java/org/apache/phoenix/spark/AggregateIT.java | 3 +
.../org/apache/phoenix/spark/DataSourceApiIT.java | 9 +-
.../java/org/apache/phoenix/spark/OrderByIT.java | 3 +
.../org/apache/phoenix/spark/SaltedTableIT.java | 3 +
.../java/org/apache/phoenix/spark/SparkUtil.java | 3 +-
.../phoenix/spark/AbstractPhoenixSparkIT.scala | 2 +
phoenix5-spark3-it/pom.xml | 38 +--
.../java/org/apache/phoenix/spark/AggregateIT.java | 3 +
.../org/apache/phoenix/spark/DataSourceApiIT.java | 3 +
.../java/org/apache/phoenix/spark/OrderByIT.java | 3 +
.../org/apache/phoenix/spark/SaltedTableIT.java | 3 +
.../phoenix/spark/AbstractPhoenixSparkIT.scala | 1 +
phoenix5-spark3-shaded/pom.xml | 38 +--
phoenix5-spark3/README.md | 21 +-
phoenix5-spark3/pom.xml | 4 +-
pom.xml | 49 ++--
21 files changed, 638 insertions(+), 144 deletions(-)
(phoenix-connectors) 02/07: PHOENIX-7065 Spark3 connector tests fail with Spark 3.4.1
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
commit e7ad177c40f0c34e9120ddb13ac337486c282800
Author: Istvan Toth <st...@apache.org>
AuthorDate: Thu Nov 23 21:06:13 2023 +0100
PHOENIX-7065 Spark3 connector tests fail with Spark 3.4.1
Fix test classpath issue
use spark.hadoopRDD.ignoreEmptySplits=false for Spark3 tests
---
phoenix5-spark/README.md | 18 +++++++++++++-----
phoenix5-spark/pom.xml | 13 +++++++++++++
.../org/apache/phoenix/spark/DataSourceApiIT.java | 6 ++++--
.../it/java/org/apache/phoenix/spark/SparkUtil.java | 3 ++-
.../phoenix/spark/AbstractPhoenixSparkIT.scala | 2 ++
phoenix5-spark3-it/pom.xml | 15 +++++++++++++--
.../phoenix/spark/AbstractPhoenixSparkIT.scala | 1 +
phoenix5-spark3/README.md | 21 ++++++++++++++++-----
phoenix5-spark3/pom.xml | 4 ++--
pom.xml | 6 ++++--
10 files changed, 70 insertions(+), 19 deletions(-)
diff --git a/phoenix5-spark/README.md b/phoenix5-spark/README.md
index f443cb0..73d68c2 100644
--- a/phoenix5-spark/README.md
+++ b/phoenix5-spark/README.md
@@ -38,6 +38,7 @@ val spark = SparkSession
.builder()
.appName("phoenix-test")
.master("local")
+ .config("spark.hadoopRDD.ignoreEmptySplits", "false")
.getOrCreate()
// Load data from TABLE1
@@ -62,7 +63,8 @@ import org.apache.spark.sql.SQLContext;
public class PhoenixSparkRead {
public static void main() throws Exception {
- SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test");
+ SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test")
+ .set("spark.hadoopRDD.ignoreEmptySplits", "false");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(jsc);
@@ -109,6 +111,7 @@ val spark = SparkSession
.builder()
.appName("phoenix-test")
.master("local")
+ .config("spark.hadoopRDD.ignoreEmptySplits", "false")
.getOrCreate()
// Load INPUT_TABLE
@@ -137,7 +140,8 @@ import org.apache.spark.sql.SQLContext;
public class PhoenixSparkWriteFromInputTable {
public static void main() throws Exception {
- SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test");
+ SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test")
+ .set("spark.hadoopRDD.ignoreEmptySplits", "false");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(jsc);
@@ -183,6 +187,7 @@ val spark = SparkSession
.builder()
.appName("phoenix-test")
.master("local")
+ .config("spark.hadoopRDD.ignoreEmptySplits", "false")
.getOrCreate()
val dataSet = List(Row(1L, "1", 1), Row(2L, "2", 2), Row(3L, "3", 3))
@@ -223,7 +228,8 @@ import java.util.List;
public class PhoenixSparkWriteFromRDDWithSchema {
public static void main() throws Exception {
- SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test");
+ SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test")
+ .set("spark.hadoopRDD.ignoreEmptySplits", "false");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(jsc);
SparkSession spark = sqlContext.sparkSession();
@@ -306,7 +312,8 @@ import org.apache.phoenix.spark._
val configuration = new Configuration()
// Can set Phoenix-specific settings, requires 'hbase.zookeeper.quorum'
-val sc = new SparkContext("local", "phoenix-test")
+val sparkConf = new SparkConf().set("spark.ui.showConsoleProgress", "false")
+val sc = new SparkContext("local", "phoenix-test", sparkConf)
val sqlContext = new SQLContext(sc)
// Load the columns 'ID' and 'COL1' from TABLE1 as a DataFrame
@@ -324,7 +331,8 @@ import org.apache.spark.sql.SQLContext
import org.apache.phoenix.spark._
import org.apache.spark.rdd.RDD
-val sc = new SparkContext("local", "phoenix-test")
+val sparkConf = new SparkConf().set("spark.ui.showConsoleProgress", "false")
+val sc = new SparkContext("local", "phoenix-test", sparkConf)
// Load the columns 'ID' and 'COL1' from TABLE1 as an RDD
val rdd: RDD[Map[String, AnyRef]] = sc.phoenixTableAsRDD(
diff --git a/phoenix5-spark/pom.xml b/phoenix5-spark/pom.xml
index cd33ca3..78eedfa 100644
--- a/phoenix5-spark/pom.xml
+++ b/phoenix5-spark/pom.xml
@@ -36,6 +36,8 @@
<properties>
<top.dir>${project.basedir}/..</top.dir>
<skip-scala-tests>true</skip-scala-tests>
+ <scala.version>${scala.version.for.spark2}</scala.version>
+ <scala.binary.version>${scala.binary.version.for.spark2}</scala.binary.version>
</properties>
<dependencies>
@@ -58,6 +60,17 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
+ <exclusions>
+ <!-- The shaded hadoop-client libraries conflict with hbase-shaded-mapreduce -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ </exclusion>
+ </exclusions>
<scope>provided</scope>
</dependency>
<dependency>
diff --git a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
index c6a4465..bc2637d 100644
--- a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
+++ b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
@@ -70,7 +70,8 @@ public class DataSourceApiIT extends ParallelStatsDisabledIT {
@Test
public void basicWriteAndReadBackTest() throws SQLException {
- SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test");
+ SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test")
+ .set("spark.hadoopRDD.ignoreEmptySplits", "false");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(jsc);
String tableName = generateUniqueName();
@@ -165,7 +166,8 @@ public class DataSourceApiIT extends ParallelStatsDisabledIT {
@Test
@Ignore // Spark3 seems to be unable to handle mixed case colum names
public void lowerCaseWriteTest() throws SQLException {
- SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test");
+ SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test")
+ .set("spark.hadoopRDD.ignoreEmptySplits", "false");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(jsc);
String tableName = generateUniqueName();
diff --git a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
index 1c36f01..f00dadb 100644
--- a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
+++ b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
@@ -45,7 +45,8 @@ public class SparkUtil {
public static SparkSession getSparkSession() {
return SparkSession.builder().appName(APP_NAME).master(NUM_EXECUTORS)
- .config(UI_SHOW_CONSOLE_PROGRESS, false).getOrCreate();
+ .config(UI_SHOW_CONSOLE_PROGRESS, false)
+ .config("spark.hadoopRDD.ignoreEmptySplits", false).getOrCreate();
}
public static ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder, String url, Configuration config)
diff --git a/phoenix5-spark/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala b/phoenix5-spark/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala
index a10d303..1b24403 100644
--- a/phoenix5-spark/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala
+++ b/phoenix5-spark/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala
@@ -103,6 +103,7 @@ class AbstractPhoenixSparkIT extends FunSuite with Matchers with BeforeAndAfter
// We pass in a TenantId to allow the DDL to create tenant-specific tables/views
setupTables("tenantSetup.sql", Some(TenantId))
+ //FIXME is this ever used ?
val conf = new SparkConf()
.setAppName("PhoenixSparkIT")
.setMaster("local[2]") // 2 threads, some parallelism
@@ -113,6 +114,7 @@ class AbstractPhoenixSparkIT extends FunSuite with Matchers with BeforeAndAfter
.appName("PhoenixSparkIT")
.master("local[2]") // 2 threads, some parallelism
.config("spark.ui.showConsoleProgress", "false")
+ .config("spark.hadoopRDD.ignoreEmptySplits", "false")
.getOrCreate()
}
diff --git a/phoenix5-spark3-it/pom.xml b/phoenix5-spark3-it/pom.xml
index 3d8c25b..5ce08df 100644
--- a/phoenix5-spark3-it/pom.xml
+++ b/phoenix5-spark3-it/pom.xml
@@ -35,8 +35,8 @@
<properties>
<top.dir>${project.basedir}/..</top.dir>
<spark.version>${spark3.version}</spark.version>
- <scala.version>2.12.10</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
+ <scala.version>${scala.version.for.spark3}</scala.version>
+ <scala.binary.version>${scala.binary.version.for.spark3}</scala.binary.version>
</properties>
<dependencies>
@@ -47,6 +47,17 @@
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <!-- The shaded hadoop-client libraries conflict with the minicluster -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/phoenix5-spark3-it/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala b/phoenix5-spark3-it/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala
index 3308b82..12e679b 100644
--- a/phoenix5-spark3-it/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala
+++ b/phoenix5-spark3-it/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala
@@ -108,6 +108,7 @@ class AbstractPhoenixSparkIT extends FunSuite with Matchers with BeforeAndAfter
.appName("PhoenixSparkIT")
.master("local[2]") // 2 threads, some parallelism
.config("spark.ui.showConsoleProgress", "false")
+ .config("spark.hadoopRDD.ignoreEmptySplits", "false")
.getOrCreate()
}
diff --git a/phoenix5-spark3/README.md b/phoenix5-spark3/README.md
index 824886d..ec7684a 100644
--- a/phoenix5-spark3/README.md
+++ b/phoenix5-spark3/README.md
@@ -48,6 +48,7 @@ val spark = SparkSession
.builder()
.appName("phoenix-test")
.master("local")
+ .config("spark.hadoopRDD.ignoreEmptySplits", "false")
.getOrCreate()
// Load data from TABLE1
@@ -72,7 +73,8 @@ import org.apache.spark.sql.SQLContext;
public class PhoenixSparkRead {
public static void main() throws Exception {
- SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test");
+ SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test")
+ .set("spark.hadoopRDD.ignoreEmptySplits", "false");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(jsc);
@@ -119,6 +121,7 @@ val spark = SparkSession
.builder()
.appName("phoenix-test")
.master("local")
+ .config("spark.hadoopRDD.ignoreEmptySplits", "false")
.getOrCreate()
// Load INPUT_TABLE
@@ -147,7 +150,8 @@ import org.apache.spark.sql.SQLContext;
public class PhoenixSparkWriteFromInputTable {
public static void main() throws Exception {
- SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test");
+ SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test")
+ .set("spark.hadoopRDD.ignoreEmptySplits", "false");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(jsc);
@@ -193,6 +197,7 @@ val spark = SparkSession
.builder()
.appName("phoenix-test")
.master("local")
+ .config("spark.hadoopRDD.ignoreEmptySplits", "false")
.getOrCreate()
val dataSet = List(Row(1L, "1", 1), Row(2L, "2", 2), Row(3L, "3", 3))
@@ -233,7 +238,8 @@ import java.util.List;
public class PhoenixSparkWriteFromRDDWithSchema {
public static void main() throws Exception {
- SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test");
+ SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("phoenix-test")
+ .set("spark.hadoopRDD.ignoreEmptySplits", "false");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
SQLContext sqlContext = new SQLContext(jsc);
SparkSession spark = sqlContext.sparkSession();
@@ -308,6 +314,7 @@ create the DataFrame or RDD directly if you need fine-grained configuration.
### Load as a DataFrame directly using a Configuration object
```scala
import org.apache.hadoop.conf.Configuration
+import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.phoenix.spark._
@@ -315,7 +322,8 @@ import org.apache.phoenix.spark._
val configuration = new Configuration()
// Can set Phoenix-specific settings, requires 'hbase.zookeeper.quorum'
-val sc = new SparkContext("local", "phoenix-test")
+val sparkConf = new SparkConf().set("spark.ui.showConsoleProgress", "false")
+val sc = new SparkContext("local", "phoenix-test", sparkConf)
val sqlContext = new SQLContext(sc)
// Load the columns 'ID' and 'COL1' from TABLE1 as a DataFrame
@@ -328,6 +336,7 @@ df.show
### Load as an RDD, using a Zookeeper URL
```scala
+import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.phoenix.spark._
@@ -358,10 +367,12 @@ CREATE TABLE OUTPUT_TEST_TABLE (id BIGINT NOT NULL PRIMARY KEY, col1 VARCHAR, co
```
```scala
+import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.phoenix.spark._
-val sc = new SparkContext("local", "phoenix-test")
+val sparkConf = new SparkConf().set("spark.ui.showConsoleProgress", "false")
+val sc = new SparkContext("local", "phoenix-test", sparkConf)
val dataSet = List((1L, "1", 1), (2L, "2", 2), (3L, "3", 3))
sc
diff --git a/phoenix5-spark3/pom.xml b/phoenix5-spark3/pom.xml
index 1b583e6..f2cec4a 100644
--- a/phoenix5-spark3/pom.xml
+++ b/phoenix5-spark3/pom.xml
@@ -35,8 +35,8 @@
<properties>
<top.dir>${project.basedir}/..</top.dir>
<spark.version>${spark3.version}</spark.version>
- <scala.version>2.12.10</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
+ <scala.version>${scala.version.for.spark3}</scala.version>
+ <scala.binary.version>${scala.binary.version.for.spark3}</scala.binary.version>
<jodatime.version>2.10.5</jodatime.version>
</properties>
diff --git a/pom.xml b/pom.xml
index b795ff9..5f796e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,9 +98,11 @@
<hive3-storage.version>2.7.0</hive3-storage.version>
<hive-storage.version>${hive3-storage.version}</hive-storage.version>
<spark.version>2.4.0</spark.version>
+ <scala.version.for.spark2>2.11.12</scala.version.for.spark2>
+ <scala.binary.version.for.spark2>2.11</scala.binary.version.for.spark2>
<spark3.version>3.0.3</spark3.version>
- <scala.version>2.11.12</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
+ <scala.version.for.spark3>2.12.18</scala.version.for.spark3>
+ <scala.binary.version.for.spark3>2.12</scala.binary.version.for.spark3>
<log4j.version>1.2.17</log4j.version>
<log4j2.version>2.18.0</log4j2.version>
(phoenix-connectors) 04/07: PHOENIX-6923 Always run Scala tests for Phoenix-Spark connector
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
commit 17f408ce6cac4a66677c6d1502a45fa06821016a
Author: Istvan Toth <st...@apache.org>
AuthorDate: Tue Nov 21 14:53:42 2023 +0100
PHOENIX-6923 Always run Scala tests for Phoenix-Spark connector
---
phoenix5-spark/pom.xml | 16 +---------------
phoenix5-spark3-it/pom.xml | 15 +--------------
pom.xml | 13 +++----------
3 files changed, 5 insertions(+), 39 deletions(-)
diff --git a/phoenix5-spark/pom.xml b/phoenix5-spark/pom.xml
index 78eedfa..e6be35c 100644
--- a/phoenix5-spark/pom.xml
+++ b/phoenix5-spark/pom.xml
@@ -35,7 +35,6 @@
<properties>
<top.dir>${project.basedir}/..</top.dir>
- <skip-scala-tests>true</skip-scala-tests>
<scala.version>${scala.version.for.spark2}</scala.version>
<scala.binary.version>${scala.binary.version.for.spark2}</scala.binary.version>
</properties>
@@ -551,7 +550,7 @@
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>WDF TestSuite.txt</filereports>
- <skipTests>${skip-scala-tests}</skipTests>
+ <skipTests>${skip.scalatest}</skipTests>
</configuration>
<executions>
<execution>
@@ -610,17 +609,4 @@
this should be kept in sync with Phoenix as much as possible -->
</plugins>
</build>
- <profiles>
- <profile>
- <id>scala-tests-enabled</id>
- <activation>
- <property>
- <name>scala-tests-enabled</name>
- </property>
- </activation>
- <properties>
- <skip-scala-tests>false</skip-scala-tests>
- </properties>
- </profile>
- </profiles>
</project>
diff --git a/phoenix5-spark3-it/pom.xml b/phoenix5-spark3-it/pom.xml
index 5ce08df..5a8aaf1 100644
--- a/phoenix5-spark3-it/pom.xml
+++ b/phoenix5-spark3-it/pom.xml
@@ -219,7 +219,7 @@
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<junitxml>.</junitxml>
<filereports>WDF TestSuite.txt</filereports>
- <skipTests>${skip-scala-tests}</skipTests>
+ <skipTests>${skip.scalatest}</skipTests>
</configuration>
<executions>
<execution>
@@ -257,17 +257,4 @@
</plugin>
</plugins>
</build>
- <profiles>
- <profile>
- <id>scala-tests-enabled</id>
- <activation>
- <property>
- <name>scala-tests-enabled</name>
- </property>
- </activation>
- <properties>
- <skip-scala-tests>false</skip-scala-tests>
- </properties>
- </profile>
- </profiles>
</project>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 5f796e4..5dbfdec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -146,7 +146,7 @@
<hbase.shaded.package>org.apache.hadoop.hbase.shaded</hbase.shaded.package>
<phoenix.main.version>5</phoenix.main.version>
- <skip-scala-tests>true</skip-scala-tests>
+ <skip.scalatest>false</skip.scalatest>
<skip.spark.javadoc>false</skip.spark.javadoc>
</properties>
@@ -858,10 +858,7 @@
<jdk>[11,17)</jdk>
</activation>
<properties>
- <argLine>${phoenix-surefire.jdk11.flags}
- ${phoenix-surefire.jdk11.tuning.flags}
- ${phoenix-surefire.argLine}
- @{jacocoArgLine}</argLine>
+ <argLine>${phoenix-surefire.jdk11.flags} ${phoenix-surefire.jdk11.tuning.flags} ${phoenix-surefire.argLine}</argLine>
</properties>
</profile>
<profile>
@@ -870,11 +867,7 @@
<jdk>[17,)</jdk>
</activation>
<properties>
- <argLine>${phoenix-surefire.jdk11.flags}
- ${phoenix-surefire.jdk17.flags}
- ${phoenix-surefire.jdk17.tuning.flags}
- ${phoenix-surefire.argLine}
- @{jacocoArgLine}</argLine>
+ <argLine>${phoenix-surefire.jdk11.flags} ${phoenix-surefire.jdk17.flags} ${phoenix-surefire.jdk17.tuning.flags} ${phoenix-surefire.argLine}</argLine>
</properties>
</profile>
<!-- disable doclint with 1.8+ JDKs-->
(phoenix-connectors) 01/07: PHOENIX-7122 Build connectors with Hbase 2.4
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
commit b2a283a3b2e2b1d2d11dcecf594f9d8c10e3cec2
Author: Istvan Toth <st...@apache.org>
AuthorDate: Thu Nov 23 08:13:54 2023 +0100
PHOENIX-7122 Build connectors with Hbase 2.4
also update apache parent to 31, and fix some dependency issues found by the newer maven-dependency-plugin
---
phoenix5-hive/pom.xml | 21 +++++++++++++++------
phoenix5-spark/pom.xml | 8 ++++++++
phoenix5-spark3-it/pom.xml | 8 ++++++++
pom.xml | 17 ++++++++---------
4 files changed, 39 insertions(+), 15 deletions(-)
diff --git a/phoenix5-hive/pom.xml b/phoenix5-hive/pom.xml
index 25f90ae..bd23b41 100644
--- a/phoenix5-hive/pom.xml
+++ b/phoenix5-hive/pom.xml
@@ -197,19 +197,25 @@
<scope>provided</scope>
</dependency>
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>3.0.0</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>${commons-compress.version}</version>
+ <scope>test</scope>
</dependency>
-
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
+ <scope>test</scope>
</dependency>
-
- <!-- Test dependencies -->
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>
@@ -385,13 +391,16 @@
</ignoredUnusedDeclaredDependencies>
<ignoredUsedUndeclaredDependencies>
<!-- I couldn't find it referenced anywhere in the phoenix-hive codebase -->
- <ignoredUsedUndeclaredDependency>
- com.google.code.findbugs:jsr305
- </ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>
org.apache.calcite.avatica:avatica
</ignoredUsedUndeclaredDependency>
+ <ignoredUnusedDeclaredDependency>
+ org.antlr:antlr-runtime
+ </ignoredUnusedDeclaredDependency>
</ignoredUsedUndeclaredDependencies>
+ <ignoredNonTestScopedDependencies>
+ org.antlr:antlr-runtime
+ </ignoredNonTestScopedDependencies>
</configuration>
</plugin>
<plugin>
diff --git a/phoenix5-spark/pom.xml b/phoenix5-spark/pom.xml
index 7b26396..cd33ca3 100644
--- a/phoenix5-spark/pom.xml
+++ b/phoenix5-spark/pom.xml
@@ -401,6 +401,14 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <!-- Why is this not provided transitively via Phoenix ? -->
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<!-- Mark every Hadoop jar as provided -->
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/phoenix5-spark3-it/pom.xml b/phoenix5-spark3-it/pom.xml
index 37f9ca8..3d8c25b 100644
--- a/phoenix5-spark3-it/pom.xml
+++ b/phoenix5-spark3-it/pom.xml
@@ -130,6 +130,13 @@
</dependency>
<!-- Minicluster dependencies -->
+ <dependency>
+ <!-- Why is this not provided transitively via Phoenix ? -->
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-it</artifactId>
@@ -233,6 +240,7 @@
<ignoredDependency>org.apache.phoenix:phoenix-hbase-compat-${hbase.compat.version}</ignoredDependency>
<ignoredDependency>org.apache.hbase:hbase-it</ignoredDependency>
<ignoredDependency>org.apache.omid:*</ignoredDependency>
+ <ignoredDependency>org.apache.zookeeper:zookeeper</ignoredDependency>
</ignoredDependencies>
</configuration>
</plugin>
diff --git a/pom.xml b/pom.xml
index a415667..b795ff9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
- <version>23</version>
+ <version>31</version>
</parent>
<scm>
@@ -58,14 +58,13 @@
<phoenix.version>5.1.3</phoenix.version>
<omid.version>1.0.2</omid.version>
<phoenix.thirdparty.version>2.0.0</phoenix.thirdparty.version>
- <!-- Hadoop Versions -->
- <hbase.version>2.1.10</hbase.version>
- <hbase.compat.version>2.1.6</hbase.compat.version>
- <hadoop.version>3.0.3</hadoop.version>
- <zookeeper.version>3.5.9</zookeeper.version>
-
- <hbase-thirdparty-version>2.1.0</hbase-thirdparty-version>
-
+ <!-- The should match the versions used to build HBase and Hadoop -->
+ <hbase.version>2.4.16</hbase.version>
+ <hbase.compat.version>2.4.1</hbase.compat.version>
+ <hadoop.version>3.1.2</hadoop.version>
+ <zookeeper.version>3.5.7</zookeeper.version>
+ <hbase-thirdparty-version>4.1.4</hbase-thirdparty-version>
+ <!-- Hbase version dependent versions end -->
<!-- General Properties -->
<test.output.tofile>true</test.output.tofile>
(phoenix-connectors) 05/07: PHOENIX-7118 Fix Shading Regressions in Spark Connector
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
commit bfbfb8d14e63f39adb2bce3e9ed281dccf4dd875
Author: Istvan Toth <st...@apache.org>
AuthorDate: Wed Nov 22 15:54:46 2023 +0100
PHOENIX-7118 Fix Shading Regressions in Spark Connector
---
phoenix5-spark-shaded/pom.xml | 271 ++++++++++++++++++++++++++++++++++++++---
phoenix5-spark3-shaded/pom.xml | 38 +++---
pom.xml | 7 +-
3 files changed, 282 insertions(+), 34 deletions(-)
diff --git a/phoenix5-spark-shaded/pom.xml b/phoenix5-spark-shaded/pom.xml
index 7cab58c..1b9ff72 100644
--- a/phoenix5-spark-shaded/pom.xml
+++ b/phoenix5-spark-shaded/pom.xml
@@ -30,13 +30,15 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>phoenix5-spark-shaded</artifactId>
- <name>Shaded Phoenix Spark Connector for Phoenix 5</name>
+ <name>Shaded Phoenix Spark 2 Connector for Phoenix 5</name>
<properties>
<top.dir>${project.basedir}/..</top.dir>
</properties>
<dependencies>
+
+ <!-- Phoenix comes first, as we shade most dependencies anyway -->
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix5-spark</artifactId>
@@ -44,18 +46,199 @@
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId>
- <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Phoenix excludes commons-beanutils from the Omid dependency, but that's basically a bug
+ We need to add it back, so that we don't depend on hadoop's common-beanutils, which may or
+ may not be shaded.
+ This can be removed once we use a Phoenix version that doesn't have this problem -->
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <!-- Mark every Hadoop jar as provided -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <scope>provided</scope>
</dependency>
<!-- We want to take the implementation from Spark -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>provided</scope>
</dependency>
+
+ <!-- Mark HBase as provided, too -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-mapreduce</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-replication</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-endpoint</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-endpoint</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-metrics-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-metrics</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-protocol</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-protocol-shaded</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-zookeeper</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase.thirdparty</groupId>
+ <artifactId>hbase-shaded-netty</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase.thirdparty</groupId>
+ <artifactId>hbase-shaded-miscellaneous</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase.thirdparty</groupId>
+ <artifactId>hbase-shaded-protobuf</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!-- Other dependencies we don't want to shade in, but are not transitively excluded by the
+ above for some reason -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <!-- random version, for exclusion only -->
+ <version>11.0.2</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.stephenc.findbugs</groupId>
+ <artifactId>findbugs-annotations</artifactId>
+ <!-- random version, for exclusion only -->
+ <version>1.3.9-1</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
+ <!-- Taken from phoenix-client-parent this should be kept in sync with
+ Phoenix as much as possible -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -75,6 +258,7 @@
<exclude>NOTICE.*</exclude>
<exclude>NOTICE</exclude>
<exclude>README*</exclude>
+ <exclude>META-INF/versions/11/org/glassfish/jersey/internal/jsr166/*.class</exclude>
<!-- Coming from Omid, should be fixed there -->
<exclude>log4j.properties</exclude>
</excludes>
@@ -107,31 +291,32 @@
</filters>
<transformers>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer
- implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>csv-bulk-load-config.properties</resource>
<file>
${project.basedir}/../config/csv-bulk-load-config.properties
</file>
</transformer>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>README.md</resource>
<file>${project.basedir}/../README.md</file>
</transformer>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>LICENSE.txt</resource>
<file>${project.basedir}/../LICENSE</file>
</transformer>
<transformer
- implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>NOTICE</resource>
<file>${project.basedir}/../NOTICE</file>
</transformer>
</transformers>
<relocations>
+ <!-- Keep phoenix-client-byo-shaded-hadoop in sync with this -->
<relocation>
<pattern>org/</pattern>
<shadedPattern>${shaded.package}.org.</shadedPattern>
@@ -156,20 +341,68 @@
<exclude>org/apache/phoenix/**</exclude>
<exclude>org/apache/omid/**</exclude>
<!-- Do want/need to expose Tephra as well ? -->
+ <!-- See PHOENIX-7118
+ Depending on the Spark classpath we may need to leave this unshaded, relocate
+ it under org/apache/hadoop/shaded/ or under org/apache/hadoop/hbase/shaded/.
+ The only thing that is guaranteed not to work is relocating it under
+ ${shaded.package} -->
+ <exclude>org/apache/commons/configuration2/**</exclude>
</excludes>
</relocation>
+ <!-- We cannot use the more elegant shading rules in -client
+ and -server for com packages, but it SHOULD be equivalent, except for the
+ protobuf change for hbase-shaded-client compatibility -->
<relocation>
- <pattern>com/</pattern>
- <shadedPattern>${shaded.package}.com.</shadedPattern>
- <excludes>
- <!-- Not the com/ packages that are a part of particular jdk implementations -->
- <exclude>com/sun/tools/**</exclude>
- <exclude>com/sun/javadoc/**</exclude>
- <exclude>com/sun/security/**</exclude>
- <exclude>com/sun/jndi/**</exclude>
- <exclude>com/sun/management/**</exclude>
- </excludes>
+ <pattern>com/beust/</pattern>
+ <shadedPattern>${shaded.package}.com.beust.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/clearspring/</pattern>
+ <shadedPattern>${shaded.package}.com.clearspring.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/codahale/</pattern>
+ <shadedPattern>${shaded.package}.come.codahale.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/fasterxml/</pattern>
+ <shadedPattern>${shaded.package}.com.fasterxml.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/force/</pattern>
+ <shadedPattern>${shaded.package}.com.force.</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>com/google/gson/</pattern>
+ <shadedPattern>${shaded.package}.com.google.gson.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/google/inject/</pattern>
+ <shadedPattern>${shaded.package}.com.google.inject.</shadedPattern>
+ </relocation>
+ <!-- This is protobuf 2.5.0 which is shaded to this package in hbase-shaded-client,
+ not the modified protobuf 3.x from hbase-thirdparty -->
+ <relocation>
+ <pattern>com/google/protobuf/</pattern>
+ <shadedPattern>${hbase.shaded.package}.com.google.protobuf.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/ibm/</pattern>
+ <shadedPattern>${shaded.package}.com.ibm.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/lmax/</pattern>
+ <shadedPattern>${shaded.package}.com.lmax.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/sun/jna/</pattern>
+ <shadedPattern>${shaded.package}.com.sun.jna.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/squareup</pattern>
+ <shadedPattern>${shaded.package}.com.squareup.</shadedPattern>
+ </relocation>
+
<relocation>
<pattern>io/</pattern>
<shadedPattern>${shaded.package}.io.</shadedPattern>
@@ -275,8 +508,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<executions>
<execution>
- <id>default-compile</id>
- <phase>none</phase>
+ <id>default-compile</id>
+ <phase>none</phase>
</execution>
</executions>
</plugin>
diff --git a/phoenix5-spark3-shaded/pom.xml b/phoenix5-spark3-shaded/pom.xml
index d10e9f2..cfc90a0 100644
--- a/phoenix5-spark3-shaded/pom.xml
+++ b/phoenix5-spark3-shaded/pom.xml
@@ -23,8 +23,8 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <artifactId>phoenix-connectors</artifactId>
<groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-connectors</artifactId>
<version>6.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -53,6 +53,16 @@
<scope>provided</scope>
</dependency>
+ <!-- Phoenix excludes commons-beanutils from the Omid dependency, but that's basically a bug
+ We need to add it back, so that we don't depend on hadoop's common-beanutils, which may or
+ may not be shaded.
+ This can be removed once we use a Phoenix version that doesn't have this problem -->
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
<!-- Mark every Hadoop jar as provided -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -227,7 +237,7 @@
<build>
<plugins>
- <!-- Taken from phoenix-client-parent this should be kept in sync with
+ <!-- Taken from phoenix-client-parent this should be kept in sync with
Phoenix as much as possible -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -323,24 +333,25 @@
<exclude>org/w3c/dom/**</exclude>
<exclude>org/xml/sax/**</exclude>
<!-- Extras compared to Hadoop -->
- <!-- Hbase classes - Maybe these could be shaded as well
- ? -->
+ <!-- Hbase classes - Maybe these could be shaded as well ? -->
<exclude>org/apache/hbase/**</exclude>
<!-- We use the spark classpath directly -->
<exclude>org/apache/spark/**</exclude>
<!-- Phoenix classes -->
<exclude>org/apache/phoenix/**</exclude>
<exclude>org/apache/omid/**</exclude>
- <!-- We must not shade this, as this is provided by HBase.
- This is specific to -->
- <!-- the cases where we rely on external HBase / Hadoop -->
- <exclude>org.apache.commons.beanutils/**</exclude>
<!-- Do want/need to expose Tephra as well ? -->
+ <!-- See PHOENIX-7118
+ Depending on the Spark classpath we may need to leave this unshaded, relocate
+ it under org/apache/hadoop/shaded/ or under org/apache/hadoop/hbase/shaded/.
+ The only thing that is guaranteed not to work is relocating it under
+ ${shaded.package} -->
+ <exclude>org/apache/commons/configuration2/**</exclude>
</excludes>
</relocation>
<!-- We cannot use the more elegant shading rules in -client
and -server for com packages, but it SHOULD be equivalent, except for the
- changes for hbase-shaded-client compatibility -->
+ protobuf change for hbase-shaded-client compatibility -->
<relocation>
<pattern>com/beust/</pattern>
<shadedPattern>${shaded.package}.com.beust.</shadedPattern>
@@ -369,7 +380,8 @@
<pattern>com/google/inject/</pattern>
<shadedPattern>${shaded.package}.com.google.inject.</shadedPattern>
</relocation>
- <!-- HBase shaded ! -->
+ <!-- This is protobuf 2.5.0 which is shaded to this package in hbase-shaded-client,
+ not the modified protobuf 3.x from hbase-thirdparty -->
<relocation>
<pattern>com/google/protobuf/</pattern>
<shadedPattern>${hbase.shaded.package}.com.google.protobuf.</shadedPattern>
@@ -395,8 +407,7 @@
<pattern>io/</pattern>
<shadedPattern>${shaded.package}.io.</shadedPattern>
<excludes>
- <!-- Exclude config keys for Hadoop that look like package
- names -->
+ <!-- Exclude config keys for Hadoop that look like package names -->
<exclude>io/compression/**</exclude>
<exclude>io/mapfile/**</exclude>
<exclude>io/map/index/*</exclude>
@@ -440,8 +451,7 @@
<pattern>net/</pattern>
<shadedPattern>${shaded.package}.net.</shadedPattern>
<excludes>
- <!-- Exclude config keys for Hadoop that look like package
- names -->
+ <!-- Exclude config keys for Hadoop that look like package names -->
<exclude>net/topology/**</exclude>
</excludes>
</relocation>
diff --git a/pom.xml b/pom.xml
index 5dbfdec..1f01f94 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,6 +57,7 @@
<!-- Phoenix Version -->
<phoenix.version>5.1.3</phoenix.version>
<omid.version>1.0.2</omid.version>
+ <commons-beanutils.version>1.9.4</commons-beanutils.version>
<phoenix.thirdparty.version>2.0.0</phoenix.thirdparty.version>
<!-- The should match the versions used to build HBase and Hadoop -->
<hbase.version>2.4.16</hbase.version>
@@ -549,7 +550,11 @@
<artifactId>phoenix5-hive-shaded</artifactId>
<version>${project.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>${commons-beanutils.version}</version>
+ </dependency>
<!-- HBase dependencies -->
<!-- These are only needed so that we can set them provided and exclude from the shaded jars -->
(phoenix-connectors) 06/07: PHOENIX-6939 Change phoenix-hive connector shading to work with hbase-shaded-mapreduce
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
commit af1045964ba90e553d346a50d3570565790a4884
Author: Istvan Toth <st...@apache.org>
AuthorDate: Wed Nov 22 15:54:46 2023 +0100
PHOENIX-6939 Change phoenix-hive connector shading to work with hbase-shaded-mapreduce
---
phoenix5-hive-shaded/pom.xml | 248 +++++++++++++++++++++++++++++++++++++------
phoenix5-hive/pom.xml | 1 -
2 files changed, 215 insertions(+), 34 deletions(-)
diff --git a/phoenix5-hive-shaded/pom.xml b/phoenix5-hive-shaded/pom.xml
index d5fafdc..dd24215 100644
--- a/phoenix5-hive-shaded/pom.xml
+++ b/phoenix5-hive-shaded/pom.xml
@@ -48,12 +48,6 @@
<artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId>
<scope>runtime</scope>
</dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper-jute</artifactId>
- <version>${zookeeper.version}</version>
- <scope>provided</scope>
- </dependency>
<!-- maven-shade-plugin doesn't inherit dependency settings, we must duplicate them to avoid
adding the provided dependencies -->
<dependency>
@@ -121,14 +115,27 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- Dependencies below should be the same for Hive, Spark and Spark3 shading config -->
+ <!-- Phoenix excludes commons-beanutils from the Omid dependency, but that's basically a bug.
+ We need to add it back, so that we don't depend on hadoop's common-beanutils, which may or
+ may not be shaded.
+ This can be removed once we use a Phoenix version that doesn't have this problem -->
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-mapreduce</artifactId>
- <scope>provided</scope>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <scope>compile</scope>
</dependency>
+
+ <!-- Mark every Hadoop jar as provided -->
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-zookeeper</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ </exclusion>
+ </exclusions>
<scope>provided</scope>
</dependency>
<dependency>
@@ -137,13 +144,23 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-protocol-shaded</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-endpoint</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
@@ -153,21 +170,138 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <scope>test</scope>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
<scope>provided</scope>
</dependency>
<!-- We want to take the implementation from Hive -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>provided</scope>
</dependency>
+
+ <!-- Mark HBase as provided, too -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-mapreduce</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-replication</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-endpoint</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-endpoint</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-metrics-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-metrics</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-protocol</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-protocol-shaded</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-zookeeper</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase.thirdparty</groupId>
+ <artifactId>hbase-shaded-netty</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase.thirdparty</groupId>
+ <artifactId>hbase-shaded-miscellaneous</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase.thirdparty</groupId>
+ <artifactId>hbase-shaded-protobuf</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!-- Other dependencies we don't want to shade in, but are not transitively excluded by the
+ above for some reason -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <!-- random version, for exclusion only -->
+ <version>11.0.2</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.stephenc.findbugs</groupId>
+ <artifactId>findbugs-annotations</artifactId>
+ <!-- random version, for exclusion only -->
+ <version>1.3.9-1</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
@@ -248,6 +382,7 @@
</transformer>
</transformers>
<relocations>
+ <!-- Keep phoenix-client-byo-shaded-hadoop in sync with this -->
<relocation>
<pattern>org/</pattern>
<shadedPattern>${shaded.package}.org.</shadedPattern>
@@ -266,27 +401,74 @@
<!-- Extras compared to Hadoop -->
<!-- Hbase classes - Maybe these could be shaded as well ? -->
<exclude>org/apache/hbase/**</exclude>
+ <!-- We use the spark classpath directly -->
+ <exclude>org/apache/spark/**</exclude>
<!-- Phoenix classes -->
<exclude>org/apache/phoenix/**</exclude>
<exclude>org/apache/omid/**</exclude>
<!-- Do want/need to expose Tephra as well ? -->
+ <!-- See PHOENIX-7118
+ Depending on the Spark classpath we may need to leave this unshaded, relocate
+ it under org/apache/hadoop/shaded/ or under org/apache/hadoop/hbase/shaded/.
+ The only thing that is guaranteed not to work is relocating it under
+ ${shaded.package} -->
+ <exclude>org/apache/commons/configuration2/**</exclude>
</excludes>
</relocation>
+ <!-- We cannot use the more elegant shading rules in -client
+ and -server for com packages, but it SHOULD be equivalent, except for the
+ protobuf change for hbase-shaded-client compatibility -->
<relocation>
- <pattern>com/</pattern>
- <shadedPattern>${shaded.package}.com.</shadedPattern>
- <excludes>
- <!-- Not the com/ packages that are a part of particular jdk implementations -->
- <exclude>com/sun/tools/**</exclude>
- <exclude>com/sun/javadoc/**</exclude>
- <exclude>com/sun/security/**</exclude>
- <exclude>com/sun/jndi/**</exclude>
- <exclude>com/sun/management/**</exclude>
- <!-- We are getting unshaded HBase from Hive, we must leave protobuf generated
- classes alone -->
- <exclude>com/google/protobuf/**</exclude>
- </excludes>
+ <pattern>com/beust/</pattern>
+ <shadedPattern>${shaded.package}.com.beust.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/clearspring/</pattern>
+ <shadedPattern>${shaded.package}.com.clearspring.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/codahale/</pattern>
+ <shadedPattern>${shaded.package}.come.codahale.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/fasterxml/</pattern>
+ <shadedPattern>${shaded.package}.com.fasterxml.</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>com/force/</pattern>
+ <shadedPattern>${shaded.package}.com.force.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/google/gson/</pattern>
+ <shadedPattern>${shaded.package}.com.google.gson.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/google/inject/</pattern>
+ <shadedPattern>${shaded.package}.com.google.inject.</shadedPattern>
+ </relocation>
+ <!-- This is protobuf 2.5.0 which is shaded to this package in hbase-shaded-client,
+ not the modified protobuf 3.x from hbase-thirdparty -->
+ <relocation>
+ <pattern>com/google/protobuf/</pattern>
+ <shadedPattern>${hbase.shaded.package}.com.google.protobuf.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/ibm/</pattern>
+ <shadedPattern>${shaded.package}.com.ibm.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/lmax/</pattern>
+ <shadedPattern>${shaded.package}.com.lmax.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/sun/jna/</pattern>
+ <shadedPattern>${shaded.package}.com.sun.jna.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com/squareup</pattern>
+ <shadedPattern>${shaded.package}.com.squareup.</shadedPattern>
+ </relocation>
+
<relocation>
<pattern>io/</pattern>
<shadedPattern>${shaded.package}.io.</shadedPattern>
diff --git a/phoenix5-hive/pom.xml b/phoenix5-hive/pom.xml
index bd23b41..69ce48a 100644
--- a/phoenix5-hive/pom.xml
+++ b/phoenix5-hive/pom.xml
@@ -147,7 +147,6 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
- <version>${hbase.version}</version>
<scope>test</scope>
</dependency>
(phoenix-connectors) 07/07: PHOENIX-7124 Build connectors with Spark 2.4.8, Spark 3.2.4, and Hive 3.1.3
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
commit d032bd57f0b5f7916654e635a6b8532f51484cd8
Author: Istvan Toth <st...@apache.org>
AuthorDate: Mon Nov 27 19:54:59 2023 +0100
PHOENIX-7124 Build connectors with Spark 2.4.8, Spark 3.2.4, and Hive 3.1.3
---
pom.xml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pom.xml b/pom.xml
index 1f01f94..47b2556 100644
--- a/pom.xml
+++ b/pom.xml
@@ -94,14 +94,14 @@
<argLine>${phoenix-surefire.argLine} ${phoenix-surefire.jdk8.tuning.flags}</argLine>
<!-- Dependency versions -->
- <hive3.version>3.1.2</hive3.version>
+ <hive3.version>3.1.3</hive3.version>
<hive.version>${hive3.version}</hive.version>
<hive3-storage.version>2.7.0</hive3-storage.version>
<hive-storage.version>${hive3-storage.version}</hive-storage.version>
- <spark.version>2.4.0</spark.version>
+ <spark.version>2.4.8</spark.version>
<scala.version.for.spark2>2.11.12</scala.version.for.spark2>
<scala.binary.version.for.spark2>2.11</scala.binary.version.for.spark2>
- <spark3.version>3.0.3</spark3.version>
+ <spark3.version>3.2.4</spark3.version>
<scala.version.for.spark3>2.12.18</scala.version.for.spark3>
<scala.binary.version.for.spark3>2.12</scala.binary.version.for.spark3>
(phoenix-connectors) 03/07: PHOENIX-7119 Java ITs are not run for either Spark Connector
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
commit 592803ee37ea502c5a72b942f17aa597043087bd
Author: Istvan Toth <st...@apache.org>
AuthorDate: Tue Nov 21 15:33:45 2023 +0100
PHOENIX-7119 Java ITs are not run for either Spark Connector
---
phoenix5-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java | 3 +++
.../src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java | 3 +++
phoenix5-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java | 3 +++
phoenix5-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java | 3 +++
.../src/it/java/org/apache/phoenix/spark/AggregateIT.java | 3 +++
.../src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java | 3 +++
phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/OrderByIT.java | 3 +++
.../src/it/java/org/apache/phoenix/spark/SaltedTableIT.java | 3 +++
8 files changed, 24 insertions(+)
diff --git a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
index e4b96a3..b00c613 100644
--- a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
+++ b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
@@ -27,8 +27,11 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.phoenix.end2end.BaseAggregateIT;
+import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.util.QueryBuilder;
+import org.junit.experimental.categories.Category;
+@Category(ParallelStatsDisabledTest.class)
public class AggregateIT extends BaseAggregateIT {
@Override
diff --git a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
index bc2637d..a49bc0a 100644
--- a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
+++ b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.spark;
import org.apache.hadoop.conf.Configuration;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.query.ConfigurationFactory;
import org.apache.phoenix.spark.datasource.v2.PhoenixDataSource;
import org.apache.phoenix.util.InstanceResolver;
@@ -32,6 +33,7 @@ import org.apache.spark.sql.types.StructType;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import java.sql.*;
import java.util.Arrays;
@@ -42,6 +44,7 @@ import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
import static org.junit.Assert.*;
+@Category(ParallelStatsDisabledTest.class)
public class DataSourceApiIT extends ParallelStatsDisabledIT {
public DataSourceApiIT() {
diff --git a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
index 043bb32..342edae 100644
--- a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
+++ b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
@@ -36,6 +36,7 @@ import java.util.List;
import java.util.Properties;
import org.apache.phoenix.end2end.BaseOrderByIT;
+import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.spark.datasource.v2.PhoenixDataSource;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryBuilder;
@@ -44,6 +45,7 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.sources.v2.DataSourceOptions;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.Ignore;
@@ -51,6 +53,7 @@ import org.junit.Ignore;
import scala.Option;
import scala.collection.JavaConverters;
+@Category(ParallelStatsDisabledTest.class)
public class OrderByIT extends BaseOrderByIT {
@Override
diff --git a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
index d72acbd..9f2cc91 100644
--- a/phoenix5-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
+++ b/phoenix5-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
@@ -26,9 +26,12 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
+import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.end2end.salted.BaseSaltedTableIT;
import org.apache.phoenix.util.QueryBuilder;
+import org.junit.experimental.categories.Category;
+@Category(ParallelStatsDisabledTest.class)
public class SaltedTableIT extends BaseSaltedTableIT {
@Override
diff --git a/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/AggregateIT.java b/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/AggregateIT.java
index e4b96a3..b00c613 100644
--- a/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/AggregateIT.java
+++ b/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/AggregateIT.java
@@ -27,8 +27,11 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.phoenix.end2end.BaseAggregateIT;
+import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.util.QueryBuilder;
+import org.junit.experimental.categories.Category;
+@Category(ParallelStatsDisabledTest.class)
public class AggregateIT extends BaseAggregateIT {
@Override
diff --git a/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java b/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
index 774454f..f25a693 100644
--- a/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
+++ b/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/DataSourceApiIT.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.spark;
import org.apache.hadoop.conf.Configuration;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.query.ConfigurationFactory;
import org.apache.phoenix.spark.sql.connector.PhoenixDataSource;
import org.apache.phoenix.util.InstanceResolver;
@@ -32,6 +33,7 @@ import org.apache.spark.sql.types.StructType;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import java.sql.*;
import java.util.Arrays;
@@ -42,6 +44,7 @@ import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
import static org.junit.Assert.*;
+@Category(ParallelStatsDisabledTest.class)
public class DataSourceApiIT extends ParallelStatsDisabledIT {
public DataSourceApiIT() {
diff --git a/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/OrderByIT.java b/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/OrderByIT.java
index e7aa424..ee87f45 100644
--- a/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/OrderByIT.java
+++ b/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/OrderByIT.java
@@ -36,6 +36,7 @@ import java.util.List;
import java.util.Properties;
import org.apache.phoenix.end2end.BaseOrderByIT;
+import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.spark.sql.connector.PhoenixDataSource;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryBuilder;
@@ -44,6 +45,7 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.Ignore;
@@ -51,6 +53,7 @@ import org.junit.Ignore;
import scala.Option;
import scala.collection.JavaConverters;
+@Category(ParallelStatsDisabledTest.class)
public class OrderByIT extends BaseOrderByIT {
@Override
diff --git a/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java b/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
index d72acbd..9f2cc91 100644
--- a/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
+++ b/phoenix5-spark3-it/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
@@ -26,9 +26,12 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
+import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.end2end.salted.BaseSaltedTableIT;
import org.apache.phoenix.util.QueryBuilder;
+import org.junit.experimental.categories.Category;
+@Category(ParallelStatsDisabledTest.class)
public class SaltedTableIT extends BaseSaltedTableIT {
@Override