You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrql.apache.org by fe...@apache.org on 2015/07/26 23:48:27 UTC
incubator-mrql git commit: [MRQL-78] Turn off Flink log INFO messages
Repository: incubator-mrql
Updated Branches:
refs/heads/master 05feb9f65 -> a5330f211
[MRQL-78] Turn off Flink log INFO messages
Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/a5330f21
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/a5330f21
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/a5330f21
Branch: refs/heads/master
Commit: a5330f211f057c620e2edc89430b7cafb6a8125a
Parents: 05feb9f
Author: fegaras <fe...@cse.uta.edu>
Authored: Sun Jul 26 13:07:15 2015 -0500
Committer: fegaras <fe...@cse.uta.edu>
Committed: Sun Jul 26 13:07:15 2015 -0500
----------------------------------------------------------------------
bin/mrql.flink | 10 +++++-----
bin/mrql.spark | 2 +-
flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen | 2 ++
3 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/a5330f21/bin/mrql.flink
----------------------------------------------------------------------
diff --git a/bin/mrql.flink b/bin/mrql.flink
index 135a9de..48c960e 100755
--- a/bin/mrql.flink
+++ b/bin/mrql.flink
@@ -60,7 +60,7 @@ if [ "$1" == "-dist" ]; then
fi
if ! type nc > /dev/null; then
jobManager=
- elif [ "$host" == "" ] || [ "$port" == "" ] || [ "`nc -z $host $port`" == "" ]; then
+ elif [ "$host" == "" ] || [ "$port" == "" ] || [ "`nc -z -w5 $host $port; echo $?`" == "1" ]; then
# there is no Flink cluster running (host:port is closed)
jobManager=
fi
@@ -71,15 +71,15 @@ if [ "$1" == "-dist" ]; then
ARGS=($*)
for (( i = 0; i < $#; i++ )); do
if [ "${ARGS[i]}" = "-nodes" ]; then
- yn=$(( ${ARGS[i+1]} / $FLINK_SLOTS ))
+ yn=$(( (${ARGS[i+1]} + $FLINK_SLOTS -1) / $FLINK_SLOTS ))
fi
done
- $FLINK_HOME/bin/flink run -m yarn-cluster -yn $yn -ys $FLINK_SLOTS -ytm $FLINK_TASK_MANAGER_MEMORY -c org.apache.mrql.Main $FULL_JAR args $args | grep -v " switched to "
+ $FLINK_HOME/bin/flink run -m yarn-cluster -yn $yn -ys $FLINK_SLOTS -ytm $FLINK_TASK_MANAGER_MEMORY -c org.apache.mrql.Main $FULL_JAR args $args
else
# a long-running Flink cluster has already been started on YARN
# (using $FLINK_HOME/bin/yarn-session.sh on a separate window)
- $FLINK_HOME/bin/flink run -c org.apache.mrql.Main $FULL_JAR args $args | grep -v " switched to "
+ $FLINK_HOME/bin/flink run -c org.apache.mrql.Main $FULL_JAR args $args
fi
else
- $JAVA_HOME/bin/java -classpath "$FULL_JAR:$FLINK_JARS" org.apache.mrql.Main -flink $* | grep -v " switched to "
+ $JAVA_HOME/bin/java -classpath "$FULL_JAR:$FLINK_JARS" org.apache.mrql.Main -flink $*
fi
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/a5330f21/bin/mrql.spark
----------------------------------------------------------------------
diff --git a/bin/mrql.spark b/bin/mrql.spark
index 5c06943..65ff2ce 100755
--- a/bin/mrql.spark
+++ b/bin/mrql.spark
@@ -43,7 +43,7 @@ if [ "$1" == "-local" ] || [ "$1" == "-dist" ]; then
for (( i = 0; i < $#; i++ )); do
if [ "${ARGS[i]}" = "-nodes" ]; then
# calculate the number of Yarn containers from the number of requested nodes
- export SPARK_WORKER_INSTANCES=$(( ${ARGS[i+1]} / $SPARK_WORKER_CORES ))
+ export SPARK_WORKER_INSTANCES=$(( (${ARGS[i+1]} + $SPARK_WORKER_CORES - 1) / $SPARK_WORKER_CORES ))
fi
done
$SPARK_HOME/bin/spark-submit --class org.apache.mrql.Main --jars $CUP_JAR,$JLINE_JAR,$GEN_JAR,$CORE_JAR --driver-class-path $CUP_JAR:$JLINE_JAR:$GEN_JAR:$CORE_JAR:$MRQL_JAR $MRQL_JAR -spark $*
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/a5330f21/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
----------------------------------------------------------------------
diff --git a/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
index e3f6e9d..24cac23 100644
--- a/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
+++ b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen
@@ -112,6 +112,8 @@ public class FlinkEvaluator extends Evaluator implements Serializable {
stream_env = StreamExecutionEnvironment.createLocalEnvironment(Config.nodes)
.setBufferTimeout(Config.stream_window);
};
+ if (!Config.info)
+ flink_env.getConfig().disableSysoutLogging();
} catch (Exception ex) {
throw new Error("Cannot initialize the Flink evaluator: "+ex);
}