You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2021/09/07 03:47:05 UTC
[incubator-wayang] 02/03: [WAYANG-38] remove the dependency
org.json because of conflict with license
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-38
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 7c9caf0706ac449abf9365e5b511c1bd568936fc
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue Sep 7 05:41:20 2021 +0200
[WAYANG-38] remove the dependency org.json because of conflict with license
Signed-off-by: bertty <be...@gmail.com>
---
wayang-api/wayang-api-scala-java/pom.xml | 1 +
wayang-commons/pom.xml | 8 +-
wayang-commons/wayang-core/pom.xml | 8 +-
.../apache/wayang/core/monitor/FileMonitor.java | 3 +-
.../optimizer/cardinality/CardinalityEstimate.java | 2 +-
.../core/optimizer/costs/EstimationContext.java | 2 +-
.../wayang/core/optimizer/costs/LoadEstimate.java | 2 +-
.../wayang/core/optimizer/costs/LoadProfile.java | 2 +-
.../optimizer/costs/LoadProfileEstimators.java | 2 +-
.../optimizer/costs/SimpleEstimationContext.java | 2 +-
.../wayang/core/platform/AtomicExecution.java | 4 +-
.../wayang/core/platform/AtomicExecutionGroup.java | 2 +-
.../wayang/core/platform/PartialExecution.java | 2 +-
.../org/apache/wayang/core/platform/Platform.java | 2 +-
.../core/profiling/CardinalityRepository.java | 4 +-
.../apache/wayang/core/profiling/ExecutionLog.java | 2 +-
.../apache/wayang/core/util/JsonSerializable.java | 2 +-
.../apache/wayang/core/util/JsonSerializables.java | 5 +-
.../apache/wayang/core/util/JsonSerializer.java | 3 +-
.../apache/wayang/core/util/json/JSONArray.java | 128 ++++++++++
.../apache/wayang/core/util/json/JSONObject.java | 275 +++++++++++++++++++++
.../wayang/core/platform/PartialExecutionTest.java | 2 +-
.../test/SerializableDummyExecutionOperator.java | 2 +-
.../wayang/jdbc/operators/SqlToStreamOperator.java | 2 +-
wayang-platforms/wayang-spark/pom.xml | 1 +
wayang-profiler/pom.xml | 1 +
.../profiler/log/DynamicLoadProfileEstimators.java | 22 +-
wayang-tests-integration/pom.xml | 1 +
28 files changed, 451 insertions(+), 41 deletions(-)
diff --git a/wayang-api/wayang-api-scala-java/pom.xml b/wayang-api/wayang-api-scala-java/pom.xml
index a1cbb5e..ed16583 100644
--- a/wayang-api/wayang-api-scala-java/pom.xml
+++ b/wayang-api/wayang-api-scala-java/pom.xml
@@ -35,6 +35,7 @@
<properties>
<java-module-name>org.apache.wayang.api</java-module-name>
<spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
</properties>
<dependencyManagement>
diff --git a/wayang-commons/pom.xml b/wayang-commons/pom.xml
index b5c111b..89930b9 100644
--- a/wayang-commons/pom.xml
+++ b/wayang-commons/pom.xml
@@ -161,13 +161,11 @@
</exclusion>
</exclusions>
</dependency>
-
- <!-- Miscellaneous -->
<dependency>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- <version>20160212</version>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
</dependency>
+ <!-- Miscellaneous -->
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
diff --git a/wayang-commons/wayang-core/pom.xml b/wayang-commons/wayang-core/pom.xml
index 5cc0331..536f97e 100644
--- a/wayang-commons/wayang-core/pom.xml
+++ b/wayang-commons/wayang-core/pom.xml
@@ -40,10 +40,6 @@
<dependencies>
<dependency>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- </dependency>
- <dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
@@ -93,6 +89,10 @@
<artifactId>wayang-utils-profile-db</artifactId>
<version>0.6.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/monitor/FileMonitor.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/monitor/FileMonitor.java
index f6e6998..aee3af3 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/monitor/FileMonitor.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/monitor/FileMonitor.java
@@ -19,7 +19,6 @@
package org.apache.wayang.core.monitor;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.util.fs.FileSystem;
import org.apache.wayang.core.util.fs.FileSystems;
@@ -30,6 +29,7 @@ import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.wayang.core.util.json.JSONObject;
public class FileMonitor extends Monitor {
@@ -48,6 +48,7 @@ public class FileMonitor extends Monitor {
jsonPlanMap.put("stages", initialExecutionPlan);
jsonPlanMap.put("run_id", runId);
JSONObject jsonPlan = new JSONObject(jsonPlanMap);
+
writer.write(jsonPlan.toString());
} catch (UncheckedIOException e) {
throw e.getCause();
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/cardinality/CardinalityEstimate.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/cardinality/CardinalityEstimate.java
index 1bd5987..e757dc8 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/cardinality/CardinalityEstimate.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/cardinality/CardinalityEstimate.java
@@ -18,11 +18,11 @@
package org.apache.wayang.core.optimizer.cardinality;
-import org.json.JSONObject;
import org.apache.wayang.core.optimizer.ProbabilisticIntervalEstimate;
import org.apache.wayang.core.plan.wayangplan.WayangPlan;
import org.apache.wayang.core.util.Formats;
import org.apache.wayang.core.util.JsonSerializable;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* An estimate of cardinality within a {@link WayangPlan} expressed as a {@link ProbabilisticIntervalEstimate}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/EstimationContext.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/EstimationContext.java
index b990ba9..d1896fd 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/EstimationContext.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/EstimationContext.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.optimizer.costs;
-import org.json.JSONObject;
import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate;
import org.apache.wayang.core.plan.wayangplan.Operator;
import org.apache.wayang.core.util.JsonSerializables;
@@ -26,6 +25,7 @@ import org.apache.wayang.core.util.JsonSerializer;
import java.util.Arrays;
import java.util.Collection;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Provides parameters required by {@link LoadProfileEstimator}s.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadEstimate.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadEstimate.java
index 8453d87..87d4360 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadEstimate.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadEstimate.java
@@ -18,10 +18,10 @@
package org.apache.wayang.core.optimizer.costs;
-import org.json.JSONObject;
import org.apache.wayang.core.optimizer.ProbabilisticIntervalEstimate;
import org.apache.wayang.core.util.JsonSerializable;
import org.apache.wayang.core.util.JsonSerializables;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* An estimate of costs of some executable code expressed as a {@link ProbabilisticIntervalEstimate}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfile.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfile.java
index b367589..94f5856 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfile.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfile.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.optimizer.costs;
-import org.json.JSONObject;
import org.apache.wayang.core.function.FunctionDescriptor;
import org.apache.wayang.core.plan.wayangplan.Operator;
import org.apache.wayang.core.util.JsonSerializable;
@@ -26,6 +25,7 @@ import org.apache.wayang.core.util.JsonSerializables;
import java.util.Collection;
import java.util.LinkedList;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Reflects the (estimated) required resources of an {@link Operator} or {@link FunctionDescriptor}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfileEstimators.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfileEstimators.java
index 46792c0..fcfe1fe 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfileEstimators.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/LoadProfileEstimators.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.optimizer.costs;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.function.FunctionDescriptor;
@@ -28,6 +27,7 @@ import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate;
import org.apache.wayang.core.optimizer.costs.LoadEstimator.SinglePointEstimationFunction;
import org.apache.wayang.core.plan.wayangplan.ExecutionOperator;
import org.apache.wayang.core.util.JuelUtils;
+import org.apache.wayang.core.util.json.JSONObject;
import org.apache.wayang.core.util.mathex.Context;
import org.apache.wayang.core.util.mathex.DefaultContext;
import org.apache.wayang.core.util.mathex.Expression;
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
index c0d0d9c..5e3cac2 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java
@@ -19,13 +19,13 @@
package org.apache.wayang.core.optimizer.costs;
import java.util.HashMap;
-import org.json.JSONObject;
import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate;
import org.apache.wayang.core.util.JsonSerializables;
import org.apache.wayang.core.util.JsonSerializer;
import java.util.Collection;
import java.util.List;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* This {@link EstimationContext} implementation just stores all required variables without any further logic.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecution.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecution.java
index d82aa1a..d697d42 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecution.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecution.java
@@ -19,8 +19,6 @@
package org.apache.wayang.core.platform;
import org.apache.commons.lang3.SerializationException;
-import org.json.JSONArray;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.optimizer.costs.ConstantLoadProfileEstimator;
import org.apache.wayang.core.optimizer.costs.EstimationContext;
@@ -29,6 +27,8 @@ import org.apache.wayang.core.optimizer.costs.LoadProfileEstimator;
import org.apache.wayang.core.optimizer.costs.LoadProfileEstimators;
import org.apache.wayang.core.util.JsonSerializables;
import org.apache.wayang.core.util.JsonSerializer;
+import org.apache.wayang.core.util.json.JSONArray;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* An atomic execution describes the smallest work unit considered by Wayang's cost model.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecutionGroup.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecutionGroup.java
index a54511f..1f8ba8a 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecutionGroup.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/AtomicExecutionGroup.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.platform;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.optimizer.costs.EstimationContext;
import org.apache.wayang.core.optimizer.costs.LoadProfile;
@@ -29,6 +28,7 @@ import org.apache.wayang.core.util.JsonSerializables;
import org.apache.wayang.core.util.JsonSerializer;
import java.util.Collection;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* This class groups {@link AtomicExecution}s with a common {@link EstimationContext} and {@link Platform}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/PartialExecution.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/PartialExecution.java
index eeae2d8..ef3a8f1 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/PartialExecution.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/PartialExecution.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.platform;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.optimizer.OptimizationContext;
import org.apache.wayang.core.optimizer.ProbabilisticDoubleInterval;
@@ -34,6 +33,7 @@ import java.util.Collection;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Captures data of a execution of a set of {@link ExecutionOperator}s.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/Platform.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/Platform.java
index 9862865..a8c2a30 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/Platform.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/platform/Platform.java
@@ -18,7 +18,6 @@
package org.apache.wayang.core.platform;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.Job;
import org.apache.wayang.core.api.exception.WayangException;
@@ -31,6 +30,7 @@ import org.apache.wayang.core.plan.wayangplan.ExecutionOperator;
import org.apache.wayang.core.util.JsonSerializables;
import org.apache.wayang.core.util.JsonSerializer;
import org.apache.wayang.core.util.ReflectionUtils;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* A platform describes an execution engine that executes {@link ExecutionOperator}s.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/CardinalityRepository.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/CardinalityRepository.java
index 6d7069e..3418ba0 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/CardinalityRepository.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/CardinalityRepository.java
@@ -19,8 +19,6 @@
package org.apache.wayang.core.profiling;
import org.apache.commons.io.IOUtils;
-import org.json.JSONArray;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.optimizer.OptimizationContext;
@@ -41,6 +39,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
+import org.apache.wayang.core.util.json.JSONArray;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Stores cardinalities that have been collected by the {@link CrossPlatformExecutor}. Current version uses
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/ExecutionLog.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/ExecutionLog.java
index 7d53872..ce1a3d8 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/ExecutionLog.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/profiling/ExecutionLog.java
@@ -19,7 +19,6 @@
package org.apache.wayang.core.profiling;
import org.apache.commons.io.IOUtils;
-import org.json.JSONObject;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.platform.CrossPlatformExecutor;
@@ -40,6 +39,7 @@ import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.stream.Stream;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Stores execution data have been collected by the {@link CrossPlatformExecutor}.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializable.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializable.java
index 97594b8..dba692e 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializable.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializable.java
@@ -18,11 +18,11 @@
package org.apache.wayang.core.util;
-import org.json.JSONObject;
import org.apache.wayang.core.api.exception.WayangException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* This interface prescribes implementing instances to be able to provide itself as a {@link JSONObject}. To allow
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializables.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializables.java
index 1176ecd..b164eac 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializables.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializables.java
@@ -19,12 +19,13 @@
package org.apache.wayang.core.util;
import org.apache.commons.lang3.SerializationException;
-import org.json.JSONArray;
-import org.json.JSONObject;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import org.apache.wayang.core.util.json.JSONArray;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Utility to deal with {@link JsonSerializable}s.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializer.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializer.java
index 48a5447..f2ae7fa 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializer.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/JsonSerializer.java
@@ -19,7 +19,8 @@
package org.apache.wayang.core.util;
import org.apache.commons.lang3.SerializationException;
-import org.json.JSONObject;
+import org.apache.wayang.core.util.json.JSONObject;
+
/**
* Alternative to {@link JsonSerializable}: Externalizes serialization logic.
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java
new file mode 100644
index 0000000..6764188
--- /dev/null
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONArray.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.wayang.core.util.json;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.DoubleNode;
+import com.fasterxml.jackson.databind.node.IntNode;
+import com.fasterxml.jackson.databind.node.LongNode;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.function.Consumer;
+import java.util.stream.StreamSupport;
+import org.apache.wayang.core.api.exception.WayangException;
+
+public class JSONArray implements Iterable<Object>{
+
+ private ArrayNode node;
+
+ public JSONArray(){
+ ObjectMapper mapper = new ObjectMapper();
+ // create a JSON object
+ this.node = mapper.createArrayNode();
+ }
+
+ JSONArray(ArrayNode node){
+ this.node = node;
+ }
+
+ ArrayNode getNode(){
+ return this.node;
+ }
+
+ public int length(){
+ return this.node.size();
+ }
+
+ public JSONObject getJSONObject(int index){
+ return new JSONObject((ObjectNode) this.node.get(index));
+ }
+
+ public void put(JSONObject value){
+ this.node.add(value.getNode());
+ }
+
+ public void put(Object value){
+ this.insertType(value).accept(value);
+ }
+
+ Consumer<Object> insertType(Object value){
+ ArrayNode node = this.getNode();
+ if(value == null){
+ return (v) -> node.add(JSONObject.NULL);
+ }else if(value instanceof Integer){
+ return (v) -> node.add((Integer) v);
+ }else if(value instanceof Long){
+ return (v) -> node.add((Long) v);
+ }else if(value instanceof Float){
+ return (v) -> node.add((Float) v);
+ }else if(value instanceof String){
+ return (v) -> node.add((String) v);
+ }else if(value instanceof Double){
+ return (v) -> node.add((Double) v);
+ }else if(value instanceof JsonNode){
+ return (v) -> node.add((JsonNode) v);
+ }else if(value instanceof JSONArray){
+ return (v) -> node.add(((JSONArray)v).getNode());
+ }else if(value instanceof JSONObject){
+ return (v) -> node.add(((JSONObject)v).getNode());
+ }
+ throw new WayangException("The type is not recognizable "+ value.getClass());
+ }
+
+ @Override
+ public String toString() {
+ return this.getNode().toString();
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return StreamSupport.stream(
+ Spliterators.spliteratorUnknownSize(this.getNode().iterator(), Spliterator.ORDERED),
+ false)
+ .map(v -> {
+ if(v instanceof ArrayNode){
+ return new JSONArray((ArrayNode) v);
+ }else if(v instanceof ObjectNode) {
+ return new JSONObject((ObjectNode) v);
+ }else if(v instanceof NullNode) {
+ return null;
+ } else if(v instanceof TextNode){
+ return ((TextNode)v).asText();
+ }else if(v instanceof DoubleNode){
+ return ((DoubleNode)v).asDouble();
+ }else if(v instanceof IntNode){
+ return ((IntNode)v).asInt();
+ }else if(v instanceof LongNode){
+ return ((LongNode)v).asLong();
+ }
+
+ else{
+ throw new WayangException("the object type is not valid "+ v.getClass());
+ }
+ })
+ .iterator();
+ }
+}
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java
new file mode 100644
index 0000000..3d49152
--- /dev/null
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/json/JSONObject.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.wayang.core.util.json;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import org.apache.wayang.core.api.exception.WayangException;
+
+public class JSONObject {
+ public static final NullNode NULL = NullNode.getInstance();
+
+ private ObjectNode node;
+
+ public JSONObject(){
+ ObjectMapper mapper = new ObjectMapper();
+ // create a JSON object
+ this.node = mapper.createObjectNode();
+ }
+
+ public JSONObject(String content){
+ ObjectMapper mapper = new ObjectMapper();
+ // create a JSON object
+ try {
+ this.node = (ObjectNode) mapper.readTree(content);
+ } catch (JsonProcessingException e) {
+ throw new WayangException(e);
+ }
+ }
+
+ public JSONObject(Map<String, Object> map){
+ ObjectMapper mapper = new ObjectMapper();
+ this.node = mapper.createObjectNode();
+ // create a JSON object
+ map.entrySet()
+ .stream()
+ .forEach(
+ entry -> {
+ insertType(entry.getValue()).accept(entry.getKey(), entry.getValue());
+ }
+ );
+
+ }
+
+ JSONObject(ObjectNode node){
+ this.node = node;
+ }
+
+ public ObjectNode getNode(){
+ return this.node;
+ }
+
+ public boolean has(String key){
+ return this.node.has(key);
+ }
+
+ public String get(String key){
+ return this.node.get(key).asText();
+ }
+
+ public String getString(String key){
+ return this.node.get(key).asText();
+ }
+
+ public double getDouble(String key){
+ return this.node.get(key).asDouble();
+ }
+
+ public long getLong(String key){
+ return this.node.get(key).asLong();
+ }
+
+ public int getInt(String key){
+ return this.node.get(key).asInt();
+ }
+
+ public JSONObject getJSONObject(String key){
+ JsonNode value = this.getNode().get(key);
+ if(value == null){
+ return null;
+ }
+ if(!value.isObject()) {
+ throw new WayangException("the key does not exist on the component");
+ }
+ return new JSONObject((ObjectNode) value);
+ }
+
+ public JSONArray getJSONArray(String key){
+ JsonNode value = this.getNode().get(key);
+ if(value == null){
+ return null;
+ }
+ if(!value.isArray()) {
+ throw new WayangException("the key does not exist on the component");
+ }
+ return new JSONArray((ArrayNode) value);
+ }
+
+ public JSONObject put(String key, String value){
+ this.getNode().put(key, value);
+ return this;
+ }
+ public JSONObject put(String key, int value){
+ this.getNode().put(key, value);
+ return this;
+ }
+
+ public JSONObject put(String key, double value){
+ this.getNode().put(key, value);
+ return this;
+ }
+
+ public JSONObject put(String key, long value){
+ this.getNode().put(key, value);
+ return this;
+ }
+
+ public JSONObject put(String key, Object value){
+ this.insertType(value).accept(key, value);
+ return this;
+ }
+
+ public void write(BufferedWriter writter) throws IOException {
+ writter.write(this.getNode().toString());
+ }
+
+ public JSONObject put(String key, JSONObject value){
+ if(this.getNode().has(key)){
+ this.getNode().replace(key, (value == null)? NULL:value.getNode());
+ }else {
+ this.getNode().set(key, (value == null)? NULL: value.getNode());
+ }
+ return this;
+ }
+
+ public JSONObject put(String key, JSONArray value){
+ if(this.getNode().has(key)){
+ this.getNode().replace(key, value.getNode());
+ }else {
+ this.getNode().set(key, value.getNode());
+ }
+ return this;
+ }
+
+ public JSONObject putOpt(String key, Object value){
+ if(value == null){
+ return this;
+ }
+ this.put(key, value);
+ return this;
+ }
+
+ public JSONObject optJSONObject(String key){
+ try {
+ return this.getJSONObject(key);
+ }catch (WayangException ex){
+ return null;
+ }
+ }
+
+ public JSONArray optJSONArray(String key){
+ try {
+ return this.getJSONArray(key);
+ }catch (WayangException ex){
+ return null;
+ }
+ }
+
+ public double optDouble(String key){
+ return this.optDouble(key, Double.NaN);
+ }
+
+ public double optDouble(String key, double value){
+ try {
+ return this.getDouble(key);
+ }catch (WayangException ex){
+ return value;
+ }
+ }
+
+ public Set<String> keySet(){
+ return Sets.newHashSet(Iterators.filter(this.node.fieldNames(), String.class));
+ }
+
+ public int length(){
+ return this.node.size();
+ }
+
+ BiConsumer<String, Object> insertType(Object value){
+ ObjectNode node = this.getNode();
+ if(value == null){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, NULL);
+ }else{
+ node.set(k, NULL);
+ }
+ };
+ }else if(value instanceof Integer){
+ return (k, v) -> node.put(k, (Integer) v);
+ }else if(value instanceof Long){
+ return (k, v) -> node.put(k, (Long) v);
+ }else if(value instanceof Float){
+ return (k, v) -> node.put(k, (Float) v);
+ }else if(value instanceof String){
+ return (k, v) -> node.put(k, (String) v);
+ }else if(value instanceof Double){
+ return (k, v) -> node.put(k, (Double) v);
+ }else if(value instanceof ObjectNode){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, (ObjectNode)v);
+ }else{
+ node.set(k, (ObjectNode)v);
+ }
+ };
+ }else if(value instanceof ArrayNode){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, (ArrayNode)v);
+ }else{
+ node.set(k, (ArrayNode)v);
+ }
+ };
+ }else if(value instanceof JSONArray){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, ((JSONArray)v).getNode());
+ }else{
+ node.set(k, ((JSONArray)v).getNode());
+ }
+ };
+ }else if(value instanceof JSONObject){
+ return (k, v) -> {
+ if(node.has(k)){
+ node.replace(k, ((JSONObject)v).getNode());
+ }else{
+ node.set(k, ((JSONObject)v).getNode());
+ }
+ };
+ }
+ throw new WayangException("The type is not recognizable "+ value.getClass());
+ }
+
+ @Override
+ public String toString() {
+ return this.getNode().toString();
+ }
+}
diff --git a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/platform/PartialExecutionTest.java b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/platform/PartialExecutionTest.java
index 0fc5f64..f39d331 100644
--- a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/platform/PartialExecutionTest.java
+++ b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/platform/PartialExecutionTest.java
@@ -18,7 +18,7 @@
package org.apache.wayang.core.platform;
-import org.json.JSONObject;
+import org.apache.wayang.core.util.json.JSONObject;
import org.junit.Assert;
import org.junit.Test;
import org.apache.wayang.core.api.Configuration;
diff --git a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/test/SerializableDummyExecutionOperator.java b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/test/SerializableDummyExecutionOperator.java
index 4b60f15..81a1ef7 100644
--- a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/test/SerializableDummyExecutionOperator.java
+++ b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/test/SerializableDummyExecutionOperator.java
@@ -18,9 +18,9 @@
package org.apache.wayang.core.test;
-import org.json.JSONObject;
import org.apache.wayang.core.plan.wayangplan.ExecutionOperator;
import org.apache.wayang.core.util.JsonSerializable;
+import org.apache.wayang.core.util.json.JSONObject;
/**
* Dummy {@link ExecutionOperator} for test purposes.
diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/SqlToStreamOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/SqlToStreamOperator.java
index f6f4a34..12d77f0 100644
--- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/SqlToStreamOperator.java
+++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/SqlToStreamOperator.java
@@ -18,7 +18,6 @@
package org.apache.wayang.jdbc.operators;
-import org.json.JSONObject;
import org.apache.wayang.basic.data.Record;
import org.apache.wayang.basic.types.RecordType;
import org.apache.wayang.core.api.exception.WayangException;
@@ -33,6 +32,7 @@ import org.apache.wayang.core.types.DataSetType;
import org.apache.wayang.core.util.JsonSerializable;
import org.apache.wayang.core.util.ReflectionUtils;
import org.apache.wayang.core.util.Tuple;
+import org.apache.wayang.core.util.json.JSONObject;
import org.apache.wayang.java.channels.StreamChannel;
import org.apache.wayang.java.execution.JavaExecutor;
import org.apache.wayang.java.operators.JavaExecutionOperator;
diff --git a/wayang-platforms/wayang-spark/pom.xml b/wayang-platforms/wayang-spark/pom.xml
index 1a0d786..7266a7b 100644
--- a/wayang-platforms/wayang-spark/pom.xml
+++ b/wayang-platforms/wayang-spark/pom.xml
@@ -37,6 +37,7 @@
<properties>
<java-module-name>org.apache.wayang.platform.spark</java-module-name>
<spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
</properties>
diff --git a/wayang-profiler/pom.xml b/wayang-profiler/pom.xml
index 0d9d15e..95de519 100644
--- a/wayang-profiler/pom.xml
+++ b/wayang-profiler/pom.xml
@@ -35,6 +35,7 @@
<properties>
<java-module-name>org.apache.wayang.profiler</java-module-name>
<spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
</properties>
<dependencyManagement>
diff --git a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/DynamicLoadProfileEstimators.java b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/DynamicLoadProfileEstimators.java
index d5a209b..c487766 100644
--- a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/DynamicLoadProfileEstimators.java
+++ b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/DynamicLoadProfileEstimators.java
@@ -18,7 +18,8 @@
package org.apache.wayang.profiler.log;
-import org.json.JSONObject;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.optimizer.costs.EstimationContext;
@@ -650,10 +651,11 @@ public class DynamicLoadProfileEstimators {
String specification,
OptimizationSpace optimizationSpace) {
try {
- final JSONObject spec = new JSONObject(specification);
- if (!spec.has("type") || "mathex".equalsIgnoreCase(spec.getString("type"))) {
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode spec = mapper.readTree(specification);
+ if (!spec.has("type") || "mathex".equalsIgnoreCase(spec.get("type").asText())) {
return createFromMathExTemplate(configKey, spec, optimizationSpace);
- } else if ("juel".equalsIgnoreCase(spec.getString("type"))) {
+ } else if ("juel".equalsIgnoreCase(spec.get("type").asText())) {
throw new IllegalStateException("JUEL templates not supported");
} else {
throw new WayangException(String.format("Unknown specification type: %s", spec.get("type")));
@@ -672,21 +674,21 @@ public class DynamicLoadProfileEstimators {
* @return the {@link DynamicLoadEstimator}
*/
private static DynamicLoadProfileEstimator createFromMathExTemplate(String configKey,
- JSONObject spec,
+ JsonNode spec,
OptimizationSpace optimizationSpace) {
- int numInputs = spec.getInt("in");
- int numOutputs = spec.getInt("out");
+ int numInputs = spec.get("in").asInt();
+ int numOutputs = spec.get("out").asInt();
DynamicLoadEstimator cpuEstimator =
- DynamicLoadEstimator.createFor(configKey, "cpu", spec.getString("cpu"), optimizationSpace);
+ DynamicLoadEstimator.createFor(configKey, "cpu", spec.get("cpu").textValue(), optimizationSpace);
// DynamicLoadEstimator ramEstimator =
// DynamicLoadEstimator.createFor(configKey, "ram", spec.getString("ram"), optimizationSpace);
DynamicLoadEstimator diskEstimator = !spec.has("disk") ?
DynamicLoadEstimator.zeroLoad :
- DynamicLoadEstimator.createFor(configKey, "disk", spec.getString("disk"), optimizationSpace);
+ DynamicLoadEstimator.createFor(configKey, "disk", spec.get("disk").textValue(), optimizationSpace);
DynamicLoadEstimator networkEstimator = !spec.has("network") ?
DynamicLoadEstimator.zeroLoad :
- DynamicLoadEstimator.createFor(configKey, "network", spec.getString("network"), optimizationSpace);
+ DynamicLoadEstimator.createFor(configKey, "network", spec.get("network").textValue(), optimizationSpace);
if (spec.has("overhead")) {
logger.warn("Overhead specification in {} will be ignored.", configKey);
diff --git a/wayang-tests-integration/pom.xml b/wayang-tests-integration/pom.xml
index 71a0957..583886c 100644
--- a/wayang-tests-integration/pom.xml
+++ b/wayang-tests-integration/pom.xml
@@ -36,6 +36,7 @@
<java-module-name>org.apache.wayang.test.integration</java-module-name>
<graphchi.version>0.2.2</graphchi.version>
<spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
<flink.version>1.7.1</flink.version>
<giraph.version>1.2.0-hadoop2</giraph.version>
</properties>