You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2022/07/05 12:01:01 UTC
[incubator-wayang] 01/01: [WAYANG-S3] corrections
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch s3
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit f5c9cdd900edae982acf7b53eb5333002cdb35c8
Author: Bertty Contreras-Rojas <be...@databloom.ai>
AuthorDate: Tue Jul 5 11:57:51 2022 +0000
[WAYANG-S3] corrections
Signed-off-by: bertty <be...@apache.org>
---
.gitignore | 1 +
.../main/java/org/apache/wayang/apps/sgd/SGDImpl.java | 7 ++++---
wayang-benchmark/pom.xml | 10 ++++++++++
.../org/apache/wayang/core/util/fs/S3FileSystem.java | 17 +++++++++++++++++
.../apache/wayang/spark/execution/SparkExecutor.java | 1 +
5 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/.gitignore b/.gitignore
index 3a9f55c4..210c8e3d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
.idea
.idea/
**/*.iml
+**/maven-wrapper.jar
# Maven
**/target/
diff --git a/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java b/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java
index d5331c64..7585e735 100644
--- a/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java
+++ b/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java
@@ -119,15 +119,16 @@ class Transform implements FunctionDescriptor.SerializableFunction<String, doubl
@Override
public double[] apply(String line) {
- String[] pointStr = line.split(" ");
+ String[] pointStr = line.split(",");
double[] point = new double[features + 1];
point[0] = Double.parseDouble(pointStr[0]);
for (int i = 1; i < pointStr.length; i++) {
- if (pointStr[i].equals("")) {
+/* if (pointStr[i].equals("")) {
continue;
}
String kv[] = pointStr[i].split(":", 2);
- point[Integer.parseInt(kv[0]) - 1] = Double.parseDouble(kv[1]);
+ point[Integer.parseInt(kv[0]) - 1] = Double.parseDouble(kv[1]);*/
+ point[i] = Double.parseDouble(pointStr[i]);
}
return point;
}
diff --git a/wayang-benchmark/pom.xml b/wayang-benchmark/pom.xml
index 7df47adf..5cace142 100644
--- a/wayang-benchmark/pom.xml
+++ b/wayang-benchmark/pom.xml
@@ -64,6 +64,16 @@
<artifactId>hadoop-aws</artifactId>
<version>3.1.2</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>3.2.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>3.1.2</version>
+ </dependency>
</dependencies>
<modules>
diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java
index 99fe3cb8..d175e632 100644
--- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java
+++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.wayang.core.util.fs;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
diff --git a/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java b/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java
index 9aab425a..7100de44 100644
--- a/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java
+++ b/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java
@@ -195,6 +195,7 @@ public class SparkExecutor extends PushExecutorTemplate {
@Override
public void dispose() {
super.dispose();
+ this.sparkContextReference.get().stop();
this.sparkContextReference.noteDiscardedReference(true);
}