You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2021/09/07 03:47:03 UTC
[incubator-wayang] branch WAYANG-38 created (now d4a45fd)
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a change to branch WAYANG-38
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git.
at d4a45fd [WAYANG-38] notice added on JSONArray and JSONObject
This branch includes the following new commits:
new 459486e [WAYANG-38] Add the License Checker
new 7c9caf0 [WAYANG-38] remove the dependency org.json because of conflict with license
new d4a45fd [WAYANG-38] notice added on JSONArray and JSONObject
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-wayang] 03/03: [WAYANG-38] notice added on JSONArray and
JSONObject
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-38
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit d4a45fd6cd1515b38702c350acacedff7c47f6ca
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue Sep 7 05:46:27 2021 +0200
[WAYANG-38] notice added on JSONArray and JSONObject
Signed-off-by: bertty <be...@gmail.com>
---
.../main/java/org/apache/wayang/core/util/json/JSONArray.java | 10 ++++++++++
.../main/java/org/apache/wayang/core/util/json/JSONObject.java | 10 ++++++++++
2 files changed, 20 insertions(+)
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java
index 6764188..35155a4 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java
@@ -34,6 +34,16 @@ import java.util.function.Consumer;
import java.util.stream.StreamSupport;
import org.apache.wayang.core.api.exception.WayangException;
+/**
+ * JSONArray is the wrapper for the {@link ArrayNode} to enable the
+ * similar functionality as the http://javadox.com/org.json/json/20151123/org/json/JSONArray.html
+ *
+ * TODO: require implement the rest of the functionalities follow the link below
+ * https://github.com/stleary/JSON-java/blob/master/src/main/java/org/json/JSONArray.java
+ *
+ * TODO: the java doc is not done because is missing implementation and it performed some
+ * modification on the code
+ */
public class JSONArray implements Iterable<Object>{
private ArrayNode node;
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java
index 3d49152..ebb4c2f 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java
@@ -33,6 +33,16 @@ import java.util.Set;
import java.util.function.BiConsumer;
import org.apache.wayang.core.api.exception.WayangException;
+/**
+ * JSONObject is the wrapper for the {@link ObjectNode} to enable the
+ * similar functionality as the http://javadox.com/org.json/json/20151123/org/json/JSONObject.html
+ *
+ * TODO: require implement the rest of the functionalities follow the link below
+ * https://github.com/stleary/JSON-java/blob/master/src/main/java/org/json/JSONObject.java
+ *
+ * TODO: the java doc is not done because is missing implementation and it performed some
+ * modification on the code
+ */
public class JSONObject {
public static final NullNode NULL = NullNode.getInstance();
[incubator-wayang] 01/03: [WAYANG-38] Add the License Checker
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-38
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 459486e386f4a17489c18c9e61419f5c16675f69
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue Sep 7 05:40:08 2021 +0200
[WAYANG-38] Add the License Checker
Signed-off-by: bertty <be...@gmail.com>
---
pom.xml | 56 ++++++++++++++++++++++++++++++++++----------------------
1 file changed, 34 insertions(+), 22 deletions(-)
diff --git a/pom.xml b/pom.xml
index 37ba3ec..f780094 100644
--- a/pom.xml
+++ b/pom.xml
@@ -387,27 +387,6 @@
</plugins>
</build>
-
- <dependencies>
- <dependency>
- <groupId>javax.xml.bind</groupId>
- <artifactId>jaxb-api</artifactId>
- <version>${jaxb.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>javax.activation</groupId>
- <artifactId>activation</artifactId>
- <version>${java-activation.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>javax.annotation</groupId>
- <artifactId>javax.annotation-api</artifactId>
- <version>${javax-annotation-api}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
</profile>
<!--
@@ -1095,6 +1074,39 @@
</excludes>
</configuration>
</plugin>
+ <!-- Fail the build if some dependency don't have a valid license -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <version>2.0.0</version>
+ <executions>
+ <execution>
+ <id>license-check</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>aggregate-add-third-party</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <licenseMerges>
+ <licenseMerge>ASF 2.0 | The Apache Software License, Version 2.0 | Apache License, Version 2.0 | Apache 2.0 License | Apache License Version 2.0 | Apache 2.0 | Apache-2.0 | The Apache License, Version 2.0 | Apache License Version 2 | Apache 2 | http://www.apache.org/licenses/LICENSE-2.0.txt | Apache License 2.0 | Apache Software License - Version 2.0</licenseMerge>
+ <licenseMerge>BSD 3-claus | 3-Clause BSD License | BSD 3 Clause License | BSD 3 Clause | BSD 3-Clause "New" or "Revised" License (BSD-3-Clause) | BSD licence | BSD | New BSD License | Revised BSD | The BSD 3-Clause License | The BSD License | The New BSD License | New BSD license | BSD 3-clause</licenseMerge>
+ <licenseMerge>MIT | MIT License | The MIT License </licenseMerge>
+ <licenseMerge>BSD 2-claus | BSD 2-Clause License | BSD 2-Clause</licenseMerge>
+ <licenseMerge>HSQLDB | HSQLDB License, a BSD open source license</licenseMerge>
+ <licenseMerge>PostgreSQL | The PostgreSQL License</licenseMerge>
+ </licenseMerges>
+ <includedLicenses>
+ ASF 2.0 | BSD 2-claus | BSD 3-claus | MIT | CC0 | HSQLDB | PostgreSQL
+ </includedLicenses>
+ <failOnBlacklist>true</failOnBlacklist>
+ <excludedGroups>
+ org.apache.spark.*|org.apache.hadoop.*|org.apache.giraph.*|org.antlr.*|junit.*
+ </excludedGroups>
+ <excludeTransitiveDependencies>true</excludeTransitiveDependencies>
+ </configuration>
+ </plugin>
<!-- Generate the legally required text files in the jars -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -1302,6 +1314,6 @@
<module>wayang-profiler</module>
<module>wayang-plugins</module>
<module>wayang-resources</module>
- <module>wayang-benchmark</module>
+ <module>wayang-benchmark</module>
</modules>
</project>
[incubator-wayang] 02/03: [WAYANG-38] remove the dependency
org.json because of conflict with license
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-38
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 7c9caf0706ac449abf9365e5b511c1bd568936fc
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue Sep 7 05:41:20 2021 +0200
[WAYANG-38] remove the dependency org.json because of conflict with license
Signed-off-by: bertty <be...@gmail.com>
---
wayang-api/wayang-api-scala-java/pom.xml | 1 +
wayang-commons/pom.xml | 8 +-
wayang-commons/wayang-core/pom.xml | 8 +-
.../apache/wayang/core/monitor/FileMonitor.java | 3 +-
.../optimizer/cardinality/CardinalityEstimate.java | 2 +-
.../core/optimizer/costs/EstimationContext.java | 2 +-
.../wayang/core/optimizer/costs/LoadEstimate.java | 2 +-
.../wayang/core/optimizer/costs/LoadProfile.java | 2 +-
.../optimizer/costs/LoadProfileEstimators.java | 2 +-
.../optimizer/costs/SimpleEstimationContext.java | 2 +-
.../wayang/core/platform/AtomicExecution.java | 4 +-
.../wayang/core/platform/AtomicExecutionGroup.java | 2 +-
.../wayang/core/platform/PartialExecution.java | 2 +-
.../org/apache/wayang/core/platform/Platform.java | 2 +-
.../core/profiling/CardinalityRepository.java | 4 +-
.../apache/wayang/core/profiling/ExecutionLog.java | 2 +-
.../apache/wayang/core/util/JsonSerializable.java | 2 +-
.../apache/wayang/core/util/JsonSerializables.java | 5 +-
.../apache/wayang/core/util/JsonSerializer.java | 3 +-
.../apache/wayang/core/util/json/JSONArray.java | 128 ++++++++++
.../apache/wayang/core/util/json/JSONObject.java | 275 +++++++++++++++++++++
.../wayang/core/platform/PartialExecutionTest.java | 2 +-
.../test/SerializableDummyExecutionOperator.java | 2 +-
.../wayang/jdbc/operators/SqlToStreamOperator.java | 2 +-
wayang-platforms/wayang-spark/pom.xml | 1 +
wayang-profiler/pom.xml | 1 +
.../profiler/log/DynamicLoadProfileEstimators.java | 22 +-
wayang-tests-integration/pom.xml | 1 +
28 files changed, 451 insertions(+), 41 deletions(-)
diff --git a/wayang-api/wayang-api-scala-java/pom.xml b/wayang-api/wayang-api-scala-java/pom.xml
index a1cbb5e..ed16583 100644
--- a/wayang-api/wayang-api-scala-java/pom.xml
+++ b/wayang-api/wayang-api-scala-java/pom.xml
@@ -35,6 +35,7 @@
<properties>
<java-module-name>org.apache.wayang.api</java-module-name>
<spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
</properties>
<dependencyManagement>
diff --git a/wayang-commons/pom.xml b/wayang-commons/pom.xml
index b5c111b..89930b9 100644
--- a/wayang-commons/pom.xml
+++ b/wayang-commons/pom.xml
@@ -161,13 +161,11 @@
</exclusion>
</exclusions>
</dependency>
-
- <!-- Miscellaneous -->
<dependency>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- <version>20160212</version>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
</dependency>
+ <!-- Miscellaneous -->
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
diff --git a/wayang-commons/wayang-core/pom.xml b/wayang-commons/wayang-core/pom.xml
index 5cc0331..536f97e 100644
--- a/wayang-commons/wayang-core/pom.xml
+++ b/wayang-commons/wayang-core/pom.xml
@@ -40,10 +40,6 @@
<dependencies>
<dependency>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- </dependency>
- <dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
@@ -93,6 +89,10 @@
<artifactId>wayang-utils-profile-db</artifactId>
<version>0.6.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/monitor/FileMonitor.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/monitor/FileMonitor.java
index f6e6998..aee3af3 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/monitor/FileMonitor.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/monitor/FileMonitor.java
@@ -19,7 +19,6 @@
package org.apache.wayang.core.monitor;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.util.fs.FileSystem;
import org.apache.wayang.core.util.fs.FileSystems;
@@ -30,6 +29,7 @@ import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.wayang.core.util.json.JSONObject;
public class FileMonitor extends Monitor {
@@ -48,6 +48,7 @@ public class FileMonitor extends Monitor {
jsonPlanMap.put("stages", initialExecutionPlan);
jsonPlanMap.put("run_id", runId);
JSONObject jsonPlan = new JSONObject(jsonPlanMap);
+
writer.write(jsonPlan.toString());
} catch (UncheckedIOException e) {
throw e.getCause();
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/cardinality/CardinalityEstimate.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/cardinality/CardinalityEstimate.java
index 1bd5987..e757dc8 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/cardinality/CardinalityEstimate.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/cardinality/CardinalityEstimate.java
@@ -18,11 +18,11 @@
package org.apache.wayang.core.optimizer.cardinality;
-import org.json.JSONObject;
import org.apache.wayang.core.optimizer.ProbabilisticIntervalEstimate;
import org.apache.wayang.core.plan.wayangplan.WayangPlan;
import org.apache.wayang.core.util.Formats;
import org.apache.wayang.core.util.JsonSerializable;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* An estimate of cardinality within a {@link WayangPlan} expressed as a {@link ProbabilisticIntervalEstimate}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/EstimationContext.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/EstimationContext.java
index b990ba9..d1896fd 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/EstimationContext.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/EstimationContext.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.optimizer.costs;
-import org.json.JSONObject;
import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate;
import org.apache.wayang.core.plan.wayangplan.Operator;
import org.apache.wayang.core.util.JsonSerializables;
@@ -26,6 +25,7 @@ import org.apache.wayang.core.util.JsonSerializer;
import java.util.Arrays;
import java.util.Collection;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Provides parameters required by {@link LoadProfileEstimator}s.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadEstimate.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadEstimate.java
index 8453d87..87d4360 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadEstimate.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadEstimate.java
@@ -18,10 +18,10 @@
package org.apache.wayang.core.optimizer.costs;
-import org.json.JSONObject;
import org.apache.wayang.core.optimizer.ProbabilisticIntervalEstimate;
import org.apache.wayang.core.util.JsonSerializable;
import org.apache.wayang.core.util.JsonSerializables;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* An estimate of costs of some executable code expressed as a {@link ProbabilisticIntervalEstimate}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfile.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfile.java
index b367589..94f5856 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfile.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfile.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.optimizer.costs;
-import org.json.JSONObject;
import org.apache.wayang.core.function.FunctionDescriptor;
import org.apache.wayang.core.plan.wayangplan.Operator;
import org.apache.wayang.core.util.JsonSerializable;
@@ -26,6 +25,7 @@ import org.apache.wayang.core.util.JsonSerializables;
import java.util.Collection;
import java.util.LinkedList;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Reflects the (estimated) required resources of an {@link Operator} or {@link FunctionDescriptor}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfileEstimators.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfileEstimators.java
index 46792c0..fcfe1fe 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfileEstimators.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfileEstimators.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.optimizer.costs;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.function.FunctionDescriptor;
@@ -28,6 +27,7 @@ import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate;
import org.apache.wayang.core.optimizer.costs.LoadEstimator.SinglePointEstimationFunction;
import org.apache.wayang.core.plan.wayangplan.ExecutionOperator;
import org.apache.wayang.core.util.JuelUtils;
+import org.apache.wayang.core.util.json.JSONObject;
import org.apache.wayang.core.util.mathex.Context;
import org.apache.wayang.core.util.mathex.DefaultContext;
import org.apache.wayang.core.util.mathex.Expression;
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
index c0d0d9c..5e3cac2 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
@@ -19,13 +19,13 @@
package org.apache.wayang.core.optimizer.costs;
import java.util.HashMap;
-import org.json.JSONObject;
import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate;
import org.apache.wayang.core.util.JsonSerializables;
import org.apache.wayang.core.util.JsonSerializer;
import java.util.Collection;
import java.util.List;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* This {@link EstimationContext} implementation just stores all required variables without any further logic.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecution.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecution.java
index d82aa1a..d697d42 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecution.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecution.java
@@ -19,8 +19,6 @@
package org.apache.wayang.core.platform;
import org.apache.commons.lang3.SerializationException;
-import org.json.JSONArray;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.optimizer.costs.ConstantLoadProfileEstimator;
import org.apache.wayang.core.optimizer.costs.EstimationContext;
@@ -29,6 +27,8 @@ import org.apache.wayang.core.optimizer.costs.LoadProfileEstimator;
import org.apache.wayang.core.optimizer.costs.LoadProfileEstimators;
import org.apache.wayang.core.util.JsonSerializables;
import org.apache.wayang.core.util.JsonSerializer;
+import org.apache.wayang.core.util.json.JSONArray;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* An atomic execution describes the smallest work unit considered by Wayang's cost model.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecutionGroup.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecutionGroup.java
index a54511f..1f8ba8a 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecutionGroup.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecutionGroup.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.platform;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.optimizer.costs.EstimationContext;
import org.apache.wayang.core.optimizer.costs.LoadProfile;
@@ -29,6 +28,7 @@ import org.apache.wayang.core.util.JsonSerializables;
import org.apache.wayang.core.util.JsonSerializer;
import java.util.Collection;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* This class groups {@link AtomicExecution}s with a common {@link EstimationContext} and {@link Platform}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/PartialExecution.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/PartialExecution.java
index eeae2d8..ef3a8f1 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/PartialExecution.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/PartialExecution.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.platform;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.optimizer.OptimizationContext;
import org.apache.wayang.core.optimizer.ProbabilisticDoubleInterval;
@@ -34,6 +33,7 @@ import java.util.Collection;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Captures data of a execution of a set of {@link ExecutionOperator}s.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/Platform.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/Platform.java
index 9862865..a8c2a30 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/Platform.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/Platform.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.platform;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.Job;
import org.apache.wayang.core.api.exception.WayangException;
@@ -31,6 +30,7 @@ import org.apache.wayang.core.plan.wayangplan.ExecutionOperator;
import org.apache.wayang.core.util.JsonSerializables;
import org.apache.wayang.core.util.JsonSerializer;
import org.apache.wayang.core.util.ReflectionUtils;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* A platform describes an execution engine that executes {@link ExecutionOperator}s.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/CardinalityRepository.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/CardinalityRepository.java
index 6d7069e..3418ba0 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/CardinalityRepository.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/CardinalityRepository.java
@@ -19,8 +19,6 @@
package org.apache.wayang.core.profiling;
import org.apache.commons.io.IOUtils;
-import org.json.JSONArray;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.optimizer.OptimizationContext;
@@ -41,6 +39,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
+import org.apache.wayang.core.util.json.JSONArray;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Stores cardinalities that have been collected by the {@link CrossPlatformExecutor}. Current version uses
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/ExecutionLog.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/ExecutionLog.java
index 7d53872..ce1a3d8 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/ExecutionLog.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/ExecutionLog.java
@@ -19,7 +19,6 @@
package org.apache.wayang.core.profiling;
import org.apache.commons.io.IOUtils;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.platform.CrossPlatformExecutor;
@@ -40,6 +39,7 @@ import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.stream.Stream;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Stores execution data have been collected by the {@link CrossPlatformExecutor}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializable.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializable.java
index 97594b8..dba692e 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializable.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializable.java
@@ -18,11 +18,11 @@
package org.apache.wayang.core.util;
-import org.json.JSONObject;
import org.apache.wayang.core.api.exception.WayangException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* This interface prescribes implementing instances to be able to provide itself as a {@link JSONObject}. To allow
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializables.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializables.java
index 1176ecd..b164eac 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializables.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializables.java
@@ -19,12 +19,13 @@
package org.apache.wayang.core.util;
import org.apache.commons.lang3.SerializationException;
-import org.json.JSONArray;
-import org.json.JSONObject;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import org.apache.wayang.core.util.json.JSONArray;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Utility to deal with {@link JsonSerializable}s.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializer.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializer.java
index 48a5447..f2ae7fa 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializer.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializer.java
@@ -19,7 +19,8 @@
package org.apache.wayang.core.util;
import org.apache.commons.lang3.SerializationException;
-import org.json.JSONObject;
+import org.apache.wayang.core.util.json.JSONObject;
+
/**
* Alternative to {@link JsonSerializable}: Externalizes serialization logic.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java
new file mode 100644
index 0000000..6764188
--- /dev/null
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.wayang.core.util.json;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.DoubleNode;
+import com.fasterxml.jackson.databind.node.IntNode;
+import com.fasterxml.jackson.databind.node.LongNode;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.function.Consumer;
+import java.util.stream.StreamSupport;
+import org.apache.wayang.core.api.exception.WayangException;
+
+public class JSONArray implements Iterable<Object>{
+
+ private ArrayNode node;
+
+ public JSONArray(){
+ ObjectMapper mapper = new ObjectMapper();
+ // create a JSON object
+ this.node = mapper.createArrayNode();
+ }
+
+ JSONArray(ArrayNode node){
+ this.node = node;
+ }
+
+ ArrayNode getNode(){
+ return this.node;
+ }
+
+ public int length(){
+ return this.node.size();
+ }
+
+ public JSONObject getJSONObject(int index){
+ return new JSONObject((ObjectNode) this.node.get(index));
+ }
+
+ public void put(JSONObject value){
+ this.node.add(value.getNode());
+ }
+
+ public void put(Object value){
+ this.insertType(value).accept(value);
+ }
+
+ Consumer<Object> insertType(Object value){
+ ArrayNode node = this.getNode();
+ if(value == null){
+ return (v) -> node.add(JSONObject.NULL);
+ }else if(value instanceof Integer){
+ return (v) -> node.add((Integer) v);
+ }else if(value instanceof Long){
+ return (v) -> node.add((Long) v);
+ }else if(value instanceof Float){
+ return (v) -> node.add((Float) v);
+ }else if(value instanceof String){
+ return (v) -> node.add((String) v);
+ }else if(value instanceof Double){
+ return (v) -> node.add((Double) v);
+ }else if(value instanceof JsonNode){
+ return (v) -> node.add((JsonNode) v);
+ }else if(value instanceof JSONArray){
+ return (v) -> node.add(((JSONArray)v).getNode());
+ }else if(value instanceof JSONObject){
+ return (v) -> node.add(((JSONObject)v).getNode());
+ }
+ throw new WayangException("The type is not recognizable "+ value.getClass());
+ }
+
+ @Override
+ public String toString() {
+ return this.getNode().toString();
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return StreamSupport.stream(
+ Spliterators.spliteratorUnknownSize(this.getNode().iterator(), Spliterator.ORDERED),
+ false)
+ .map(v -> {
+ if(v instanceof ArrayNode){
+ return new JSONArray((ArrayNode) v);
+ }else if(v instanceof ObjectNode) {
+ return new JSONObject((ObjectNode) v);
+ }else if(v instanceof NullNode) {
+ return null;
+ } else if(v instanceof TextNode){
+ return ((TextNode)v).asText();
+ }else if(v instanceof DoubleNode){
+ return ((DoubleNode)v).asDouble();
+ }else if(v instanceof IntNode){
+ return ((IntNode)v).asInt();
+ }else if(v instanceof LongNode){
+ return ((LongNode)v).asLong();
+ }
+
+ else{
+ throw new WayangException("the object type is not valid "+ v.getClass());
+ }
+ })
+ .iterator();
+ }
+}
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java
new file mode 100644
index 0000000..3d49152
--- /dev/null
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.wayang.core.util.json;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import org.apache.wayang.core.api.exception.WayangException;
+
+public class JSONObject {
+ public static final NullNode NULL = NullNode.getInstance();
+
+ private ObjectNode node;
+
+ public JSONObject(){
+ ObjectMapper mapper = new ObjectMapper();
+ // create a JSON object
+ this.node = mapper.createObjectNode();
+ }
+
+ public JSONObject(String content){
+ ObjectMapper mapper = new ObjectMapper();
+ // create a JSON object
+ try {
+ this.node = (ObjectNode) mapper.readTree(content);
+ } catch (JsonProcessingException e) {
+ throw new WayangException(e);
+ }
+ }
+
+ public JSONObject(Map<String, Object> map){
+ ObjectMapper mapper = new ObjectMapper();
+ this.node = mapper.createObjectNode();
+ // create a JSON object
+ map.entrySet()
+ .stream()
+ .forEach(
+ entry -> {
+ insertType(entry.getValue()).accept(entry.getKey(), entry.getValue());
+ }
+ );
+
+ }
+
+ JSONObject(ObjectNode node){
+ this.node = node;
+ }
+
+ public ObjectNode getNode(){
+ return this.node;
+ }
+
+ public boolean has(String key){
+ return this.node.has(key);
+ }
+
+ public String get(String key){
+ return this.node.get(key).asText();
+ }
+
+ public String getString(String key){
+ return this.node.get(key).asText();
+ }
+
+ public double getDouble(String key){
+ return this.node.get(key).asDouble();
+ }
+
+ public long getLong(String key){
+ return this.node.get(key).asLong();
+ }
+
+ public int getInt(String key){
+ return this.node.get(key).asInt();
+ }
+
+ public JSONObject getJSONObject(String key){
+ JsonNode value = this.getNode().get(key);
+ if(value == null){
+ return null;
+ }
+ if(!value.isObject()) {
+ throw new WayangException("the key does not exist on the component");
+ }
+ return new JSONObject((ObjectNode) value);
+ }
+
+ public JSONArray getJSONArray(String key){
+ JsonNode value = this.getNode().get(key);
+ if(value == null){
+ return null;
+ }
+ if(!value.isArray()) {
+ throw new WayangException("the key does not exist on the component");
+ }
+ return new JSONArray((ArrayNode) value);
+ }
+
+ public JSONObject put(String key, String value){
+ this.getNode().put(key, value);
+ return this;
+ }
+ public JSONObject put(String key, int value){
+ this.getNode().put(key, value);
+ return this;
+ }
+
+ public JSONObject put(String key, double value){
+ this.getNode().put(key, value);
+ return this;
+ }
+
+ public JSONObject put(String key, long value){
+ this.getNode().put(key, value);
+ return this;
+ }
+
+ public JSONObject put(String key, Object value){
+ this.insertType(value).accept(key, value);
+ return this;
+ }
+
+ public void write(BufferedWriter writter) throws IOException {
+ writter.write(this.getNode().toString());
+ }
+
+ public JSONObject put(String key, JSONObject value){
+ if(this.getNode().has(key)){
+ this.getNode().replace(key, (value == null)? NULL:value.getNode());
+ }else {
+ this.getNode().set(key, (value == null)? NULL: value.getNode());
+ }
+ return this;
+ }
+
+ public JSONObject put(String key, JSONArray value){
+ if(this.getNode().has(key)){
+ this.getNode().replace(key, value.getNode());
+ }else {
+ this.getNode().set(key, value.getNode());
+ }
+ return this;
+ }
+
+ public JSONObject putOpt(String key, Object value){
+ if(value == null){
+ return this;
+ }
+ this.put(key, value);
+ return this;
+ }
+
+ public JSONObject optJSONObject(String key){
+ try {
+ return this.getJSONObject(key);
+ }catch (WayangException ex){
+ return null;
+ }
+ }
+
+ public JSONArray optJSONArray(String key){
+ try {
+ return this.getJSONArray(key);
+ }catch (WayangException ex){
+ return null;
+ }
+ }
+
+ public double optDouble(String key){
+ return this.optDouble(key, Double.NaN);
+ }
+
+ public double optDouble(String key, double value){
+ try {
+ return this.getDouble(key);
+ }catch (WayangException ex){
+ return value;
+ }
+ }
+
+ public Set<String> keySet(){
+ return Sets.newHashSet(Iterators.filter(this.node.fieldNames(), String.class));
+ }
+
+ public int length(){
+ return this.node.size();
+ }
+
+ BiConsumer<String, Object> insertType(Object value){
+ ObjectNode node = this.getNode();
+ if(value == null){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, NULL);
+ }else{
+ node.set(k, NULL);
+ }
+ };
+ }else if(value instanceof Integer){
+ return (k, v) -> node.put(k, (Integer) v);
+ }else if(value instanceof Long){
+ return (k, v) -> node.put(k, (Long) v);
+ }else if(value instanceof Float){
+ return (k, v) -> node.put(k, (Float) v);
+ }else if(value instanceof String){
+ return (k, v) -> node.put(k, (String) v);
+ }else if(value instanceof Double){
+ return (k, v) -> node.put(k, (Double) v);
+ }else if(value instanceof ObjectNode){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, (ObjectNode)v);
+ }else{
+ node.set(k, (ObjectNode)v);
+ }
+ };
+ }else if(value instanceof ArrayNode){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, (ArrayNode)v);
+ }else{
+ node.set(k, (ArrayNode)v);
+ }
+ };
+ }else if(value instanceof JSONArray){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, ((JSONArray)v).getNode());
+ }else{
+ node.set(k, ((JSONArray)v).getNode());
+ }
+ };
+ }else if(value instanceof JSONObject){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, ((JSONObject)v).getNode());
+ }else{
+ node.set(k, ((JSONObject)v).getNode());
+ }
+ };
+ }
+ throw new WayangException("The type is not recognizable "+ value.getClass());
+ }
+
+ @Override
+ public String toString() {
+ return this.getNode().toString();
+ }
+}
diff --git a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/platform/PartialExecutionTest.java b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/platform/PartialExecutionTest.java
index 0fc5f64..f39d331 100644
--- a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/platform/PartialExecutionTest.java
+++ b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/platform/PartialExecutionTest.java
@@ -18,7 +18,7 @@
package org.apache.wayang.core.platform;
-import org.json.JSONObject;
+import org.apache.wayang.core.util.json.JSONObject;
import org.junit.Assert;
import org.junit.Test;
import org.apache.wayang.core.api.Configuration;
diff --git a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/test/SerializableDummyExecutionOperator.java b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/test/SerializableDummyExecutionOperator.java
index 4b60f15..81a1ef7 100644
--- a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/test/SerializableDummyExecutionOperator.java
+++ b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/test/SerializableDummyExecutionOperator.java
@@ -18,9 +18,9 @@
package org.apache.wayang.core.test;
-import org.json.JSONObject;
import org.apache.wayang.core.plan.wayangplan.ExecutionOperator;
import org.apache.wayang.core.util.JsonSerializable;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Dummy {@link ExecutionOperator} for test purposes.
diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/SqlToStreamOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/SqlToStreamOperator.java
index f6f4a34..12d77f0 100644
--- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/SqlToStreamOperator.java
+++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/SqlToStreamOperator.java
@@ -18,7 +18,6 @@
package org.apache.wayang.jdbc.operators;
-import org.json.JSONObject;
import org.apache.wayang.basic.data.Record;
import org.apache.wayang.basic.types.RecordType;
import org.apache.wayang.core.api.exception.WayangException;
@@ -33,6 +32,7 @@ import org.apache.wayang.core.types.DataSetType;
import org.apache.wayang.core.util.JsonSerializable;
import org.apache.wayang.core.util.ReflectionUtils;
import org.apache.wayang.core.util.Tuple;
+import org.apache.wayang.core.util.json.JSONObject;
import org.apache.wayang.java.channels.StreamChannel;
import org.apache.wayang.java.execution.JavaExecutor;
import org.apache.wayang.java.operators.JavaExecutionOperator;
diff --git a/wayang-platforms/wayang-spark/pom.xml b/wayang-platforms/wayang-spark/pom.xml
index 1a0d786..7266a7b 100644
--- a/wayang-platforms/wayang-spark/pom.xml
+++ b/wayang-platforms/wayang-spark/pom.xml
@@ -37,6 +37,7 @@
<properties>
<java-module-name>org.apache.wayang.platform.spark</java-module-name>
<spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
</properties>
diff --git a/wayang-profiler/pom.xml b/wayang-profiler/pom.xml
index 0d9d15e..95de519 100644
--- a/wayang-profiler/pom.xml
+++ b/wayang-profiler/pom.xml
@@ -35,6 +35,7 @@
<properties>
<java-module-name>org.apache.wayang.profiler</java-module-name>
<spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
</properties>
<dependencyManagement>
diff --git a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/DynamicLoadProfileEstimators.java b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/DynamicLoadProfileEstimators.java
index d5a209b..c487766 100644
--- a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/DynamicLoadProfileEstimators.java
+++ b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/DynamicLoadProfileEstimators.java
@@ -18,7 +18,8 @@
package org.apache.wayang.profiler.log;
-import org.json.JSONObject;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.optimizer.costs.EstimationContext;
@@ -650,10 +651,11 @@ public class DynamicLoadProfileEstimators {
String specification,
OptimizationSpace optimizationSpace) {
try {
- final JSONObject spec = new JSONObject(specification);
- if (!spec.has("type") || "mathex".equalsIgnoreCase(spec.getString("type"))) {
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode spec = mapper.readTree(specification);
+ if (!spec.has("type") || "mathex".equalsIgnoreCase(spec.get("type").asText())) {
return createFromMathExTemplate(configKey, spec, optimizationSpace);
- } else if ("juel".equalsIgnoreCase(spec.getString("type"))) {
+ } else if ("juel".equalsIgnoreCase(spec.get("type").asText())) {
throw new IllegalStateException("JUEL templates not supported");
} else {
throw new WayangException(String.format("Unknown specification type: %s", spec.get("type")));
@@ -672,21 +674,21 @@ public class DynamicLoadProfileEstimators {
* @return the {@link DynamicLoadEstimator}
*/
private static DynamicLoadProfileEstimator createFromMathExTemplate(String configKey,
- JSONObject spec,
+ JsonNode spec,
OptimizationSpace optimizationSpace) {
- int numInputs = spec.getInt("in");
- int numOutputs = spec.getInt("out");
+ int numInputs = spec.get("in").asInt();
+ int numOutputs = spec.get("out").asInt();
DynamicLoadEstimator cpuEstimator =
- DynamicLoadEstimator.createFor(configKey, "cpu", spec.getString("cpu"), optimizationSpace);
+ DynamicLoadEstimator.createFor(configKey, "cpu", spec.get("cpu").textValue(), optimizationSpace);
// DynamicLoadEstimator ramEstimator =
// DynamicLoadEstimator.createFor(configKey, "ram", spec.getString("ram"), optimizationSpace);
DynamicLoadEstimator diskEstimator = !spec.has("disk") ?
DynamicLoadEstimator.zeroLoad :
- DynamicLoadEstimator.createFor(configKey, "disk", spec.getString("disk"), optimizationSpace);
+ DynamicLoadEstimator.createFor(configKey, "disk", spec.get("disk").textValue(), optimizationSpace);
DynamicLoadEstimator networkEstimator = !spec.has("network") ?
DynamicLoadEstimator.zeroLoad :
- DynamicLoadEstimator.createFor(configKey, "network", spec.getString("network"), optimizationSpace);
+ DynamicLoadEstimator.createFor(configKey, "network", spec.get("network").textValue(), optimizationSpace);
if (spec.has("overhead")) {
logger.warn("Overhead specification in {} will be ignored.", configKey);
diff --git a/wayang-tests-integration/pom.xml b/wayang-tests-integration/pom.xml
index 71a0957..583886c 100644
--- a/wayang-tests-integration/pom.xml
+++ b/wayang-tests-integration/pom.xml
@@ -36,6 +36,7 @@
<java-module-name>org.apache.wayang.test.integration</java-module-name>
<graphchi.version>0.2.2</graphchi.version>
<spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
<flink.version>1.7.1</flink.version>
<giraph.version>1.2.0-hadoop2</giraph.version>
</properties>