You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2015/06/18 22:22:06 UTC
[02/41] vxquery git commit: MRQL script for FLINK.
MRQL script for FLINK.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/2ff5293d
Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/2ff5293d
Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/2ff5293d
Branch: refs/heads/master
Commit: 2ff5293d564e73c7051cd158d8072c3de4b57bdd
Parents: 9abe3e9
Author: Preston Carman <pr...@apache.org>
Authored: Wed Oct 22 13:50:54 2014 -0700
Committer: Preston Carman <pr...@apache.org>
Committed: Wed Feb 25 10:37:24 2015 -0800
----------------------------------------------------------------------
.../other_systems/mrql_scripts/clear.sh | 24 -------------
.../other_systems/mrql_scripts/clear_hadoop.sh | 24 +++++++++++++
.../other_systems/mrql_scripts/clear_hadoop2.sh | 22 ++++++++++++
.../mrql_scripts/run_group_test.sh | 36 +++++++++++++-------
.../mrql_scripts/run_mrql_tests.sh | 7 ++--
5 files changed, 71 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/vxquery/blob/2ff5293d/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
deleted file mode 100755
index 9a912b2..0000000
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Remove data
-rm -rf disk1/hadoop/data
-rm -rf disk2/hadoop/data
-rm -rf disk1/hadoop/tmp
-rm -rf disk1/hadoop/logs
http://git-wip-us.apache.org/repos/asf/vxquery/blob/2ff5293d/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop.sh
new file mode 100755
index 0000000..9a912b2
--- /dev/null
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Remove data
+rm -rf disk1/hadoop/data
+rm -rf disk2/hadoop/data
+rm -rf disk1/hadoop/tmp
+rm -rf disk1/hadoop/logs
http://git-wip-us.apache.org/repos/asf/vxquery/blob/2ff5293d/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop2.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop2.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop2.sh
new file mode 100755
index 0000000..fbc15b5
--- /dev/null
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop2.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Remove data
+rm -rf disk1/hadoop2/hdfs
+rm -rf disk2/hadoop2/hdfs
http://git-wip-us.apache.org/repos/asf/vxquery/blob/2ff5293d/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
index 84028e5..9c2f3ad 100755
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
@@ -33,13 +33,21 @@ DATASET=${1}
NODES=${2}
REPEAT=1
DATA_FILES=${NODES}
-#DATA_FILES=8
# Start Hadoop
-sh saved/hadoop/hadoop-1.2.1/bin/start-all.sh
+# sh saved/hadoop/hadoop-1.2.1/bin/start-all.sh
+sh saved/hadoop/hadoop-2.5.1/sbin/hadoop-daemon.sh start namenode
+sh saved/hadoop/hadoop-2.5.1/sbin/hadoop-daemons.sh start datanode
+sh saved/hadoop/hadoop-2.5.1/sbin/yarn-daemon.sh start resourcemanager
+sh saved/hadoop/hadoop-2.5.1/sbin/yarn-daemons.sh start nodemanager
+sh saved/hadoop/hadoop-2.5.1/sbin/mr-jobhistory-daemon.sh start historyserver
sleep 10
+# Start Flink
+sh saved/flink/flink-yarn-0.6.1-incubating/bin/yarn-session.sh -n $((4*${NODES})) -tm 1024 &
+FLINK_PID=$!
+
# Prepare hadoop file system
hadoop fs -mkdir ${DATASET}
hadoop fs -ls
@@ -47,19 +55,12 @@ hadoop fs -mkdir ${DATASET}/sensors
hadoop fs -mkdir ${DATASET}/stations
hadoop fs -ls ${DATASET}
-# Prepare hadoop file system 2
-hadoop fs -mkdir ${DATASET}2
-hadoop fs -ls
-hadoop fs -mkdir ${DATASET}2/sensors
-hadoop fs -mkdir ${DATASET}2/stations
-hadoop fs -ls ${DATASET}2
-
hadoop balancer
# Upload test data
COUNTER=0
-while [ ${COUNTER} -lt ${NODES} ];
+while [ ${COUNTER} -lt ${DATA_FILES} ];
do
sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ${DATASET} ${COUNTER}
let COUNTER=COUNTER+1
@@ -67,9 +68,18 @@ done
# Start test
-sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh \
- vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT} ${DATASET}
+sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT} ${DATASET}
+
+# Stop Flink
+kill ${FLINK_PID}
+jobs -p
+kill $(jobs -p)
# Stop Hadoop
-sh saved/hadoop/hadoop-1.2.1/bin/stop-all.sh
+# sh saved/hadoop/hadoop-1.2.1/bin/stop-all.sh
+sh saved/hadoop/hadoop-2.5.1/sbin/mr-jobhistory-daemon.sh stop historyserver
+sh saved/hadoop/hadoop-2.5.1/sbin/yarn-daemons.sh stop nodemanager
+sh saved/hadoop/hadoop-2.5.1/sbin/yarn-daemon.sh stop resourcemanager
+sh saved/hadoop/hadoop-2.5.1/sbin/hadoop-daemons.sh stop datanode
+sh saved/hadoop/hadoop-2.5.1/sbin/hadoop-daemon.sh stop namenode
http://git-wip-us.apache.org/repos/asf/vxquery/blob/2ff5293d/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh
index 912cd3b..c000727 100755
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh
@@ -23,19 +23,16 @@
NODES=${2}
REPEAT=${3}
DATASET=${4}
-THREADS=$((4*${NODES}))
# Make log folder
mkdir -p ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/
-for j in $(find ${1} -name '*q?7.mrql')
+for j in $(find ${1} -name '*q??.mrql')
do
date
echo "Running MRQL query: ${j}"
- time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes ${THREADS} ${j} \
- ${DATASET}/sensors/ ${DATASET}/stations/ ${DATASET}2/sensors/ ${DATASET}2/stations/ \
- >> ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/$(basename "${j}").log 2>&1; done;
+ time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql.flink -dist -nodes ${NODES} ${j} ${DATASET}/sensors/ ${DATASET}/stations/ >> ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/$(basename "${j}").log 2>&1; done;
done