You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by he...@apache.org on 2015/12/10 22:42:15 UTC
camel git commit: Spark+Hive tests should not be executed if JVM max
memory is less than 1 GB.
Repository: camel
Updated Branches:
refs/heads/master 8230d49b5 -> 8ef757a78
Spark+Hive tests should not be executed if JVM max memory is less than 1 GB.
Project: http://git-wip-us.apache.org/repos/asf/camel/repo
Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/8ef757a7
Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/8ef757a7
Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/8ef757a7
Branch: refs/heads/master
Commit: 8ef757a784ff7796ffca712a8f307439eccdca72
Parents: 8230d49
Author: Henryk Konsek <he...@gmail.com>
Authored: Thu Dec 10 22:42:11 2015 +0100
Committer: Henryk Konsek <he...@gmail.com>
Committed: Thu Dec 10 22:42:11 2015 +0100
----------------------------------------------------------------------
.../component/spark/SparkProducerTest.java | 27 ++++++++++++++++----
1 file changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/camel/blob/8ef757a7/components/camel-spark/src/test/java/org/apache/camel/component/spark/SparkProducerTest.java
----------------------------------------------------------------------
diff --git a/components/camel-spark/src/test/java/org/apache/camel/component/spark/SparkProducerTest.java b/components/camel-spark/src/test/java/org/apache/camel/component/spark/SparkProducerTest.java
index 81a28e6..6c85148 100644
--- a/components/camel-spark/src/test/java/org/apache/camel/component/spark/SparkProducerTest.java
+++ b/components/camel-spark/src/test/java/org/apache/camel/component/spark/SparkProducerTest.java
@@ -19,6 +19,7 @@ package org.apache.camel.component.spark;
import java.io.File;
import java.io.IOException;
+import static java.lang.Runtime.getRuntime;
import static java.util.Arrays.asList;
import com.google.common.truth.Truth;
@@ -30,12 +31,14 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.hive.HiveContext;
+import org.junit.BeforeClass;
import org.junit.Test;
import static org.apache.camel.component.spark.SparkConstants.SPARK_DATAFRAME_CALLBACK_HEADER;
import static org.apache.camel.component.spark.SparkConstants.SPARK_RDD_CALLBACK_HEADER;
import static org.apache.camel.component.spark.Sparks.createLocalSparkContext;
import static org.apache.camel.component.spark.annotations.AnnotatedRddCallback.annotatedRddCallback;
+import static org.junit.Assume.assumeTrue;
public class SparkProducerTest extends CamelTestSupport {
@@ -43,7 +46,9 @@ public class SparkProducerTest extends CamelTestSupport {
static JavaSparkContext sparkContext = createLocalSparkContext();
- static HiveContext hiveContext = new HiveContext(sparkContext.sc());
+ static boolean shouldRunHive = getRuntime().maxMemory() > 1024 * 1024 * 1024;
+
+ static HiveContext hiveContext;
String sparkUri = "spark:rdd?rdd=#pomRdd";
@@ -51,6 +56,13 @@ public class SparkProducerTest extends CamelTestSupport {
String sparkHiveUri = "spark:hive";
+ @BeforeClass
+ public static void beforeClass() {
+ if (shouldRunHive) {
+ hiveContext = new HiveContext(sparkContext.sc());
+ }
+ }
+
// Routes fixtures
@Override
@@ -59,10 +71,12 @@ public class SparkProducerTest extends CamelTestSupport {
registry.bind("pomRdd", sparkContext.textFile("testrdd.txt"));
- registry.bind("hiveContext", hiveContext);
- DataFrame jsonCars = hiveContext.read().json("src/test/resources/cars.json");
- jsonCars.registerTempTable("cars");
- registry.bind("jsonCars", jsonCars);
+ if (shouldRunHive) {
+ registry.bind("hiveContext", hiveContext);
+ DataFrame jsonCars = hiveContext.read().json("src/test/resources/cars.json");
+ jsonCars.registerTempTable("cars");
+ registry.bind("jsonCars", jsonCars);
+ }
registry.bind("countLinesTransformation", new org.apache.camel.component.spark.RddCallback() {
@Override
@@ -180,6 +194,7 @@ public class SparkProducerTest extends CamelTestSupport {
@Test
public void shouldExecuteHiveQuery() {
+ assumeTrue(shouldRunHive);
long tablesCount = template.requestBody(sparkHiveUri + "?collect=false", "SELECT * FROM cars", Long.class);
Truth.assertThat(tablesCount).isEqualTo(2);
}
@@ -188,6 +203,7 @@ public class SparkProducerTest extends CamelTestSupport {
@Test
public void shouldCountFrame() {
+ assumeTrue(shouldRunHive);
DataFrameCallback callback = new DataFrameCallback<Long>() {
@Override
public Long onDataFrame(DataFrame dataFrame, Object... payloads) {
@@ -200,6 +216,7 @@ public class SparkProducerTest extends CamelTestSupport {
@Test
public void shouldExecuteConditionalFrameCount() {
+ assumeTrue(shouldRunHive);
DataFrameCallback callback = new DataFrameCallback<Long>() {
@Override
public Long onDataFrame(DataFrame dataFrame, Object... payloads) {