[incubator-wayang] branch assembly created (now b889d40e)

[be...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

bertty pushed a change to branch assembly
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git


      at b889d40e [WAYANG-assembly] mirror issues

This branch includes the following new commits:

     new b889d40e [WAYANG-assembly] mirror issues

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[incubator-wayang] 01/01: [WAYANG-assembly] mirror issues

[be...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

bertty pushed a commit to branch assembly
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git

commit b889d40e6681deeeb310796416b0b1e535871ddd
Author: Bertty Contreras-Rojas <be...@databloom.ai>
AuthorDate: Fri Jul 8 19:19:48 2022 +0200

    [WAYANG-assembly] mirror issues
    
    Signed-off-by: bertty <be...@apache.org>
---
 README.md                 |  2 +-
 bin/wayang-submit         | 22 +++++++++++++++++++---
 wayang-assembly/README.md |  1 +
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 314d39ec..b8b11b36 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ You can download wayang from [here](https://github.com/apache/incubator-wayang/r
 
 ```shell
 tar -xvf wayang-0.6.1-snapshot.tar.gz
-cd wayang-0.6.1-snapshot
+cd wayang-0.6.1-SNAPSHOT
 ```
 
 In linux
diff --git a/bin/wayang-submit b/bin/wayang-submit
index 9cb4d48b..aa85c86b 100755
--- a/bin/wayang-submit
+++ b/bin/wayang-submit
@@ -71,22 +71,37 @@ fi
 # Find Spark jars.
 if [ -d "${SPARK_HOME}" ]; then
   SPARK_JARS_DIR="${SPARK_HOME}/jars"
+  ls ${SPARK_JARS_DIR}/
 fi
 
 # Find Hadoop jars.
 if [ -d "${HADOOP_HOME}" ]; then
   HADOOP_JARS_DIR="${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*"
+  ls ${HADOOP_HOME}/share/hadoop/common/
+  ls ${HADOOP_HOME}/share/hadoop/common/lib/
+fi
+
+if [ "$(ls ${SPARK_JARS_DIR} | grep ^hadoop | wc -l)" == "0" ]; then
+
+  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-common | grep -v tests | head -n 1)"
+  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-core | grep -v tests | head -n 1)"
+  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-jobclient | grep -v tests | head -n 1)"
+  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/$(ls ${HADOOP_HOME}/share/hadoop/hdfs | grep ^hadoop-hdfs-client | grep -v tests | head -n 1)"
+  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-annotations | grep -v tests | head -n 1)"
+  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-auth | grep -v tests | head -n 1)"
+
 fi
 
 
 WAYANG_CODE="${WAYANG_HOME}/jars"
+ls ${WAYANG_CODE}
 WAYANG_LIBS="${WAYANG_HOME}/libs"
+ls ${WAYANG_LIBS}
 WAYANG_CONF="${WAYANG_HOME}/conf"
-
+ls ${WAYANG_CONF}
 # Bootstrap the classpath.
 WAYANG_CLASSPATH="${WAYANG_CONF}/*:${WAYANG_CODE}/*:${WAYANG_LIBS}/*"
-
-WAYANG_CLASSPATH="${WAYANG_CLASSPATH}:${SPARK_JARS_DIR}/*:${HADOOP_JARS_DIR}"
+WAYANG_CLASSPATH="${SPARK_JARS_DIR}/*:${WAYANG_CLASSPATH}:${HADOOP_JARS_DIR}"
 
 FLAGS=""
 if [ "${FLAG_LOG}" = "true" ]; then
@@ -101,6 +116,7 @@ if [ -n "${OTHER_FLAGS}" ]; then
 	FLAGS="${FLAGS} ${OTHER_FLAGS}"
 fi
 
+echo "lala"
 echo "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${@:2}"
 eval "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${@:2}"
 
diff --git a/wayang-assembly/README.md b/wayang-assembly/README.md
index 4f78e1db..c29cc46b 100644
--- a/wayang-assembly/README.md
+++ b/wayang-assembly/README.md
@@ -19,5 +19,6 @@ except for the jars in the list
 To execute the Wayang Assembly you need to execute the following command in the project root
 
 ```shell
+./mvnw clean install -DskipTests 
 ./mvnw clean package -pl :wayang-assembly -Pdistribution
 ```
\ No newline at end of file