You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2020/04/29 11:59:48 UTC
[carbondata] branch master updated: [CARBONDATA-3784] Added spark
binary version to related modules
This is an automated email from the ASF dual-hosted git repository.
jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 0119007 [CARBONDATA-3784] Added spark binary version to related modules
0119007 is described below
commit 0119007db9b312c848181dfb44162a027a660654
Author: kunal642 <ku...@gmail.com>
AuthorDate: Wed Apr 8 15:32:39 2020 +0530
[CARBONDATA-3784] Added spark binary version to related modules
Why is this PR needed?
For deploying multiple carbon jars version with different spark versions, the jar/module names should be different
What changes were proposed in this PR?
Add spark binary version to the related modules to distinguish the jars based on spark version
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3700
---
.gitignore | 1 +
assembly/pom.xml | 4 ++--
examples/flink/pom.xml | 2 +-
examples/spark/pom.xml | 17 +++++++++++++-
index/examples/pom.xml | 20 ++++++++++++++++-
index/secondary-index/pom.xml | 17 +++++++++++++-
integration/flink/pom.xml | 10 +++++++--
integration/spark/pom.xml | 20 +++++++----------
mv/plan/pom.xml | 8 ++++++-
pom.xml | 52 +++++++++++++++++++++++++++++++++++++++++++
10 files changed, 130 insertions(+), 21 deletions(-)
diff --git a/.gitignore b/.gitignore
index 0af3918..41d6f18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,4 +24,5 @@ python/.idea/
*/*/.cache-tests
*/*/*/.cache-main
*/*/*/.cache-tests
+*.flattened-pom.xml
python/pycarbon/.pylintrc
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 3003611..6c0e30e 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -58,12 +58,12 @@
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-mv-plan</artifactId>
+ <artifactId>carbondata-mv-plan_${spark.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index f746060..9d3bb76 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -63,7 +63,7 @@
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 5f10c06..a28875b 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -36,7 +36,7 @@
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -202,5 +202,20 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
+ <profile>
+ <id>spark-2.3</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <spark.binary.version>2.3</spark.binary.version>
+ </properties>
+ </profile>
+ <profile>
+ <id>spark-2.4</id>
+ <properties>
+ <spark.binary.version>2.4</spark.binary.version>
+ </properties>
+ </profile>
</profiles>
</project>
diff --git a/index/examples/pom.xml b/index/examples/pom.xml
index 3fa440b..c634b99 100644
--- a/index/examples/pom.xml
+++ b/index/examples/pom.xml
@@ -38,7 +38,7 @@
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -80,4 +80,22 @@
</plugins>
</build>
+ <profiles>
+ <profile>
+ <id>spark-2.3</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <spark.binary.version>2.3</spark.binary.version>
+ </properties>
+ </profile>
+ <profile>
+ <id>spark-2.4</id>
+ <properties>
+ <spark.binary.version>2.4</spark.binary.version>
+ </properties>
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file
diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml
index b3034f5..f6bd005 100644
--- a/index/secondary-index/pom.xml
+++ b/index/secondary-index/pom.xml
@@ -37,7 +37,7 @@
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
@@ -205,6 +205,21 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
+ <profile>
+ <id>spark-2.3</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <spark.binary.version>2.3</spark.binary.version>
+ </properties>
+ </profile>
+ <profile>
+ <id>spark-2.4</id>
+ <properties>
+ <spark.binary.version>2.4</spark.binary.version>
+ </properties>
+ </profile>
</profiles>
</project>
diff --git a/integration/flink/pom.xml b/integration/flink/pom.xml
index de69f41..1b37465 100644
--- a/integration/flink/pom.xml
+++ b/integration/flink/pom.xml
@@ -198,10 +198,13 @@
<activation>
<activeByDefault>true</activeByDefault>
</activation>
+ <properties>
+ <spark.binary.version>2.3</spark.binary.version>
+ </properties>
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
@@ -215,10 +218,13 @@
</profile>
<profile>
<id>spark-2.4</id>
+ <properties>
+ <spark.binary.version>2.4</spark.binary.version>
+ </properties>
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml
index 0b7a1a7..55b739a 100644
--- a/integration/spark/pom.xml
+++ b/integration/spark/pom.xml
@@ -26,7 +26,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark_${spark.binary.version}</artifactId>
<name>Apache CarbonData :: Spark</name>
<properties>
@@ -39,8 +39,6 @@
<build.directory.projectHadoop>../../hadoop/target</build.directory.projectHadoop>
<build.directory.projectFormat>../../format/target</build.directory.projectFormat>
<build.directory.projectSpark>../../integration/spark/target</build.directory.projectSpark>
- <!--<build.directory.projectHive>../../integration/hive/target</build.directory.projectHive>-->
- <!--<build.directory.projectPresto>../../integration/presto/target</build.directory.projectPresto>-->
<build.directory.projectStoreSdk>../../sdk/sdk/target</build.directory.projectStoreSdk>
<build.directory.projectStreaming>../../streaming/target</build.directory.projectStreaming>
<build.directory.projectBloom>../../index/bloom/target</build.directory.projectBloom>
@@ -52,8 +50,6 @@
<classes.directory.projectHadoop>../../hadoop/target/classes</classes.directory.projectHadoop>
<classes.directory.projectFormat>../../format/target/classes</classes.directory.projectFormat>
<classes.directory.projectSpark>../../integration/spark/target/classes</classes.directory.projectSpark>
- <!--<classes.directory.projectHive>../../integration/hive/target/classes</classes.directory.projectHive>-->
- <!--<classes.directory.projectPresto>../../integration/presto/target/classes</classes.directory.projectPresto>-->
<classes.directory.projectStoreSdk>../../sdk/sdk/target/classes</classes.directory.projectStoreSdk>
<classes.directory.projectStreaming>../../streaming/target/classes</classes.directory.projectStreaming>
<classes.directory.projectBloom>../../index/bloom/target/classes</classes.directory.projectBloom>
@@ -66,10 +62,6 @@
<sources.directory.projectFormat>../../format/src/main/thrift</sources.directory.projectFormat>
<sources.directory.projectSpark>../../integration/spark/src/main/scala</sources.directory.projectSpark>
<sources.directory.projectSpark>../../integration/spark/src/main/java</sources.directory.projectSpark>
- <!--<sources.directory.projectHive>../../integration/hive/src/main/java</sources.directory.projectHive>-->
- <!--<sources.directory.projectHive>../../integration/hive/src/main/scala</sources.directory.projectHive>-->
- <!--<sources.directory.projectPresto>../../integration/presto/src/main/java</sources.directory.projectPresto>-->
- <!--<sources.directory.projectPresto>../../integration/presto/src/main/scala</sources.directory.projectPresto>-->
<sources.directory.projectStoreSdk>../../sdk/sdk/src/main/java</sources.directory.projectStoreSdk>
<sources.directory.projectStreaming>../../streaming/src/main/java</sources.directory.projectStreaming>
<sources.directory.projectStreaming>../../streaming/src/main/scala</sources.directory.projectStreaming>
@@ -82,8 +74,6 @@
<generated-sources.directory.projectHadoop>../../hadoop/target/generated-sources/annotations</generated-sources.directory.projectHadoop>
<generated-sources.directory.projectFormat>../../format/target/generated-sources/annotations</generated-sources.directory.projectFormat>
<generated-sources.directory.projectSpark>../../integration/spark/target/generated-sources/annotations</generated-sources.directory.projectSpark>
- <!--<generated-sources.directory.projectHive>../../integration/hive/target/generated-sources/annotations</generated-sources.directory.projectHive>-->
- <!--<generated-sources.directory.projectPresto>../../integration/presto/target/generated-sources/annotations</generated-sources.directory.projectPresto>-->
<generated-sources.directory.projectStoreSdk>../../sdk/sdk/target/generated-sources/annotations</generated-sources.directory.projectStoreSdk>
<generated-sources.directory.projectStreaming>../../streaming/target/generated-sources/annotations</generated-sources.directory.projectStreaming>
<generated-sources.directory.projectBloom>../../index/bloom/target/generated-sources/annotations</generated-sources.directory.projectBloom>
@@ -139,7 +129,7 @@
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-mv-plan</artifactId>
+ <artifactId>carbondata-mv-plan_${spark.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<!-- spark -->
@@ -495,6 +485,9 @@
<activation>
<activeByDefault>true</activeByDefault>
</activation>
+ <properties>
+ <spark.binary.version>2.3</spark.binary.version>
+ </properties>
<build>
<plugins>
<plugin>
@@ -530,6 +523,9 @@
</profile>
<profile>
<id>spark-2.4</id>
+ <properties>
+ <spark.binary.version>2.4</spark.binary.version>
+ </properties>
<build>
<plugins>
<plugin>
diff --git a/mv/plan/pom.xml b/mv/plan/pom.xml
index da9080c..a26ce4c 100644
--- a/mv/plan/pom.xml
+++ b/mv/plan/pom.xml
@@ -26,7 +26,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>carbondata-mv-plan</artifactId>
+ <artifactId>carbondata-mv-plan_${spark.binary.version}</artifactId>
<name>Apache CarbonData :: Materialized View Plan</name>
<properties>
@@ -157,6 +157,9 @@
<activation>
<activeByDefault>true</activeByDefault>
</activation>
+ <properties>
+ <spark.binary.version>2.3</spark.binary.version>
+ </properties>
<build>
<plugins>
<plugin>
@@ -192,6 +195,9 @@
</profile>
<profile>
<id>spark-2.4</id>
+ <properties>
+ <spark.binary.version>2.4</spark.binary.version>
+ </properties>
<build>
<plugins>
<plugin>
diff --git a/pom.xml b/pom.xml
index f603b23..0a9db3f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -538,6 +538,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
+ <spark.binary.version>2.3</spark.binary.version>
<spark.version>2.3.4</spark.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.8</scala.version>
@@ -545,6 +546,31 @@
<build>
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>flatten-maven-plugin</artifactId>
+ <!--<version>1.2.2</version>-->
+ <configuration>
+ </configuration>
+ <executions>
+ <!-- enable flattening -->
+ <execution>
+ <id>flatten</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>flatten</goal>
+ </goals>
+ </execution>
+ <!-- ensure proper cleanup -->
+ <execution>
+ <id>flatten.clean</id>
+ <phase>clean</phase>
+ <goals>
+ <goal>clean</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.eluder.coveralls</groupId>
<artifactId>coveralls-maven-plugin</artifactId>
<version>4.3.0</version>
@@ -583,6 +609,7 @@
<profile>
<id>spark-2.4</id>
<properties>
+ <spark.binary.version>2.4</spark.binary.version>
<spark.version>2.4.5</spark.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.12</scala.version>
@@ -590,6 +617,31 @@
<build>
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>flatten-maven-plugin</artifactId>
+ <!--<version>1.2.2</version>-->
+ <configuration>
+ </configuration>
+ <executions>
+ <!-- enable flattening -->
+ <execution>
+ <id>flatten</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>flatten</goal>
+ </goals>
+ </execution>
+ <!-- ensure proper cleanup -->
+ <execution>
+ <id>flatten.clean</id>
+ <phase>clean</phase>
+ <goals>
+ <goal>clean</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.eluder.coveralls</groupId>
<artifactId>coveralls-maven-plugin</artifactId>
<version>4.3.0</version>