You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by zh...@apache.org on 2018/11/30 05:19:07 UTC
carbondata git commit: [CARBONDATA-3128]Fix the HiveExample exception
Repository: carbondata
Updated Branches:
refs/heads/master 0bcd8677a -> 6e04fe65a
[CARBONDATA-3128]Fix the HiveExample exception
This pull request fixes HiveExample exception with different spark versions.
When through idea running HiveExample application occurs exception with spark-2.1 and spark-2.3 profile, this fix update pom.xml of carbondata modules for spark dependencies.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6e04fe65
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6e04fe65
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6e04fe65
Branch: refs/heads/master
Commit: 6e04fe65ad5bfa775e4153ed53b0890c6da5b19e
Parents: 0bcd867
Author: Nicholas Jiang <pr...@163.com>
Authored: Tue Nov 27 14:51:56 2018 +0800
Committer: Zhang Zhichao <44...@qq.com>
Committed: Fri Nov 30 13:16:22 2018 +0800
----------------------------------------------------------------------
datamap/mv/core/pom.xml | 1 -
datamap/mv/plan/pom.xml | 1 -
examples/spark2/pom.xml | 3 --
integration/hive/pom.xml | 6 ++-
.../carbondata/hiveexample/HiveExample.scala | 1 -
.../hiveexampleCI/RunHiveExampleTest.scala | 43 ++++++++++++++++++++
integration/presto/pom.xml | 3 --
integration/spark-common-test/pom.xml | 1 -
integration/spark2/pom.xml | 6 ---
pom.xml | 29 ++++++++++++-
streaming/pom.xml | 3 --
11 files changed, 76 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/datamap/mv/core/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/mv/core/pom.xml b/datamap/mv/core/pom.xml
index 96ffb3f..dbfb22e 100644
--- a/datamap/mv/core/pom.xml
+++ b/datamap/mv/core/pom.xml
@@ -53,7 +53,6 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/datamap/mv/plan/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/pom.xml b/datamap/mv/plan/pom.xml
index 9286c14..6c965e8 100644
--- a/datamap/mv/plan/pom.xml
+++ b/datamap/mv/plan/pom.xml
@@ -47,7 +47,6 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/examples/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index 6f7aba6..48728ac 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -74,8 +74,6 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.deps.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
@@ -91,7 +89,6 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/hive/pom.xml
----------------------------------------------------------------------
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index 35c431f..b2256e7 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -82,7 +82,6 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
@@ -104,6 +103,11 @@
<artifactId>carbondata-hadoop</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala b/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala
index 11839c9..3638321 100644
--- a/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala
+++ b/integration/hive/src/main/scala/org/apache/carbondata/hiveexample/HiveExample.scala
@@ -192,7 +192,6 @@ object HiveExample {
outOfOrderColFetched = outOfOrderColFetched + 1
}
hiveEmbeddedServer2.stop()
- System.exit(0)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/hive/src/test/scala/org/apache/carbondata/hiveexampleCI/RunHiveExampleTest.scala
----------------------------------------------------------------------
diff --git a/integration/hive/src/test/scala/org/apache/carbondata/hiveexampleCI/RunHiveExampleTest.scala b/integration/hive/src/test/scala/org/apache/carbondata/hiveexampleCI/RunHiveExampleTest.scala
new file mode 100644
index 0000000..92f7374
--- /dev/null
+++ b/integration/hive/src/test/scala/org/apache/carbondata/hiveexampleCI/RunHiveExampleTest.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.hiveexampleCI
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.hiveexample.HiveExample
+
+class RunHiveExampleTest extends QueryTest with BeforeAndAfterAll {
+
+ private val spark = sqlContext.sparkSession
+
+ override def beforeAll: Unit = {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ }
+
+ test("HiveExample") {
+ HiveExample.main(null)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 5ac146c..31a4de6 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -437,13 +437,11 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-network-common_${scala.binary.version}</artifactId>
<scope>test</scope>
- <version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<scope>test</scope>
- <version>${spark.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -487,7 +485,6 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<scope>test</scope>
- <version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.lz4</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index 5da0c67..fd96f88 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -135,7 +135,6 @@
will fail to execute -->
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
<scope>test</scope>
<exclusions>
<!-- need to Exclude Avro jar from this project,spark core is using
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 69594eb..d066142 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -71,18 +71,14 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.deps.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
<exclusions>
<!-- from transitive dependency com.univocity:univocity-parsers:2.5.9
is added from the org.apache.spark:spark-sql_2.11,so need to remove
@@ -96,8 +92,6 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.deps.scope}</scope>
<exclusions>
<exclusion>
<groupId>net.jpountz.lz4</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8e67c26..df8ae42 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,7 +100,6 @@
<module>processing</module>
<module>hadoop</module>
<module>integration/spark-common</module>
- <module>integration/spark-datasource</module>
<module>integration/spark-common-test</module>
<module>datamap/examples</module>
<module>store/sdk</module>
@@ -202,12 +201,30 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.deps.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>${spark.deps.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
+ <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.deps.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.deps.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>${spark.deps.scope}</scope>
@@ -220,6 +237,12 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
+ <artifactId>spark-network-common_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.deps.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>${spark.deps.scope}</scope>
@@ -461,6 +484,7 @@
<modules>
<module>format</module>
<module>integration/spark2</module>
+ <module>integration/spark-datasource</module>
<module>examples/spark2</module>
<module>integration/hive</module>
<module>integration/presto</module>
@@ -495,6 +519,7 @@
</properties>
<modules>
<module>integration/spark2</module>
+ <module>integration/spark-datasource</module>
<module>integration/hive</module>
<module>integration/presto</module>
<module>streaming</module>
@@ -555,6 +580,7 @@
</properties>
<modules>
<module>integration/spark2</module>
+ <module>integration/spark-datasource</module>
<module>integration/hive</module>
<module>integration/presto</module>
<module>streaming</module>
@@ -613,6 +639,7 @@
</properties>
<modules>
<module>integration/spark2</module>
+ <module>integration/spark-datasource</module>
<module>integration/hive</module>
<module>integration/presto</module>
<module>streaming</module>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e04fe65/streaming/pom.xml
----------------------------------------------------------------------
diff --git a/streaming/pom.xml b/streaming/pom.xml
index f8b9c22..4e8eb3b 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -28,13 +28,10 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${spark.deps.scope}</scope>
</dependency>
<dependency>
<groupId>junit</groupId>