You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by rp...@apache.org on 2022/07/08 17:25:22 UTC
[incubator-wayang] branch main updated: Correction minnors (#262)
This is an automated email from the ASF dual-hosted git repository.
rpardomeza pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
The following commit(s) were added to refs/heads/main by this push:
new 458a7383 Correction minnors (#262)
458a7383 is described below
commit 458a7383db5e58cc8ce4cb93660e732461bceb66
Author: Bertty Contreras-Rojas <be...@databloom.ai>
AuthorDate: Fri Jul 8 19:25:19 2022 +0200
Correction minnors (#262)
* [WAYANG-assembly] correction in the folder structure
Signed-off-by: bertty <be...@apache.org>
* [WAYANG-assembly] add missing link and minnor
Signed-off-by: bertty <be...@apache.org>
* [WAYANG-assembly] mirror issues
Signed-off-by: bertty <be...@apache.org>
---
README.md | 2 +-
bin/wayang-submit | 22 +++++++++++++++++++---
wayang-assembly/README.md | 1 +
3 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index 34c92412..592eb0a4 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ You can download wayang from [here](https://github.com/apache/incubator-wayang/r
```shell
tar -xvf wayang-0.6.1-snapshot.tar.gz
-cd wayang-0.6.1-snapshot
+cd wayang-0.6.1-SNAPSHOT
```
In linux
diff --git a/bin/wayang-submit b/bin/wayang-submit
index 9cb4d48b..aa85c86b 100755
--- a/bin/wayang-submit
+++ b/bin/wayang-submit
@@ -71,22 +71,37 @@ fi
# Find Spark jars.
if [ -d "${SPARK_HOME}" ]; then
SPARK_JARS_DIR="${SPARK_HOME}/jars"
+ ls ${SPARK_JARS_DIR}/
fi
# Find Hadoop jars.
if [ -d "${HADOOP_HOME}" ]; then
HADOOP_JARS_DIR="${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*"
+ ls ${HADOOP_HOME}/share/hadoop/common/
+ ls ${HADOOP_HOME}/share/hadoop/common/lib/
+fi
+
+if [ "$(ls ${SPARK_JARS_DIR} | grep ^hadoop | wc -l)" == "0" ]; then
+
+ HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-common | grep -v tests | head -n 1)"
+ HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-core | grep -v tests | head -n 1)"
+ HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-jobclient | grep -v tests | head -n 1)"
+ HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/$(ls ${HADOOP_HOME}/share/hadoop/hdfs | grep ^hadoop-hdfs-client | grep -v tests | head -n 1)"
+ HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-annotations | grep -v tests | head -n 1)"
+ HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-auth | grep -v tests | head -n 1)"
+
fi
WAYANG_CODE="${WAYANG_HOME}/jars"
+ls ${WAYANG_CODE}
WAYANG_LIBS="${WAYANG_HOME}/libs"
+ls ${WAYANG_LIBS}
WAYANG_CONF="${WAYANG_HOME}/conf"
-
+ls ${WAYANG_CONF}
# Bootstrap the classpath.
WAYANG_CLASSPATH="${WAYANG_CONF}/*:${WAYANG_CODE}/*:${WAYANG_LIBS}/*"
-
-WAYANG_CLASSPATH="${WAYANG_CLASSPATH}:${SPARK_JARS_DIR}/*:${HADOOP_JARS_DIR}"
+WAYANG_CLASSPATH="${SPARK_JARS_DIR}/*:${WAYANG_CLASSPATH}:${HADOOP_JARS_DIR}"
FLAGS=""
if [ "${FLAG_LOG}" = "true" ]; then
@@ -101,6 +116,7 @@ if [ -n "${OTHER_FLAGS}" ]; then
FLAGS="${FLAGS} ${OTHER_FLAGS}"
fi
+echo "lala"
echo "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${@:2}"
eval "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${@:2}"
diff --git a/wayang-assembly/README.md b/wayang-assembly/README.md
index 4f78e1db..c29cc46b 100644
--- a/wayang-assembly/README.md
+++ b/wayang-assembly/README.md
@@ -19,5 +19,6 @@ except for the jars in the list
To execute the Wayang Assembly you need to execute the following command in the project root
```shell
+./mvnw clean install -DskipTests
./mvnw clean package -pl :wayang-assembly -Pdistribution
```
\ No newline at end of file