You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/10/21 21:36:11 UTC
[10/14] git commit: Getting MRQL scripts ready for prime time.
Getting MRQL scripts ready for prime time.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/eefadb25
Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/eefadb25
Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/eefadb25
Branch: refs/heads/master
Commit: eefadb254a1a30a191d01778fa62c2d7d9582593
Parents: 17bedfa
Author: Preston Carman <pr...@apache.org>
Authored: Wed Oct 8 15:11:40 2014 -0700
Committer: Preston Carman <pr...@apache.org>
Committed: Wed Oct 8 15:11:40 2014 -0700
----------------------------------------------------------------------
.../noaa-ghcn-daily/other_systems/mrql/q06.mrql | 2 +-
.../other_systems/mrql_scripts/clear.sh | 9 ++++
.../mrql_scripts/run_group_test.sh | 55 ++++++++++++++++++++
.../mrql_scripts/run_mrql_tests.sh | 5 +-
.../noaa-ghcn-daily/scripts/run_mrql_tests.sh | 42 ---------------
5 files changed, 69 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql
index 583a5b9..a50dfe2 100644
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/q06.mrql
@@ -19,7 +19,7 @@ from sensors in source(xml, args[0], {"data"}),
d in sensors.date,
v in sensors.value,
stations in source(xml, args[1], {"station"}),
- n in stations.displayName,
+ n in stations.displayName
where text(stations.id) = text(sensors.station)
and toInt(substring(text(d), 0, 4)) = 2000
and text(sensors.dataType) = "TMAX"
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
index da7cabe..b775de2 100755
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
@@ -16,4 +16,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
+# Hadoop data reset
hadoop namenode -format
+
+# Remove data
+rm -rf disk1/hadoop/data
+rm -rf disk2/hadoop/data
+rm -rf disk1/hadoop/tmp
+rm -rf disk1/hadoop/logs
+
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
new file mode 100755
index 0000000..c34ec95
--- /dev/null
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+NODES=2
+REPEAT=1
+
+# Start Hadoop
+sh saved/hadoop/hadoop-1.2.1/bin/start-all.sh
+
+# Prepare hadoop file system
+hadoop fs -mkdir all
+hadoop fs -mkdir all/sensors
+hadoop fs -mkdir all/stations
+
+
+# Upload test data
+n=0
+while [ ${n} -lt ${NODES} ];
+do
+ # Add each sensor block
+ cp saved/backups/mr/all_sensors_${n}.xml.gz disk1/hadoop/upload/
+ gunzip disk1/hadoop/upload/all_sensors_${n}.xml.gz
+ hadoop fs -copyFromLocal disk1/hadoop/upload/all_sensors_${n}.xml all/sensors
+ rm -f disk1/hadoop/upload/all_sensors_${n}.xml
+
+ # Add each station block
+ cp saved/backups/mr/all_stations_${n}.xml.gz disk1/hadoop/upload/
+ gunzip disk1/hadoop/upload/all_stations_${n}.xml.gz
+ hadoop fs -copyFromLocal disk1/hadoop/upload/all_stations_${n}.xml all/stations
+ rm -f disk1/hadoop/upload/all_stations_${n}.xml
+done
+
+
+# Start test
+sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT}
+
+
+# Stop Hadoop
+sh saved/hadoop/hadoop-1.2.1/bin/stop-all.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh
index 10ab4d9..1e512e1 100755
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh
@@ -25,11 +25,14 @@ REPEAT=${3}
DATASET="all"
+# Make log folder
+mkdir -p ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/
+
for j in $(find ${1} -name '*q??.mrql')
do
date
echo "Running MRQL query: ${j}"
- time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes ${NODES} ${j} ${DATASET}/sensors.xml ${DATASET}/stations.xml >> ~/disk1/weather_data/mrql/query_logs/$(basename "${j}").log 2>&1; done;
+ time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes ${NODES} ${j} ${DATASET}/sensors/ ${DATASET}/stations/ >> ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/$(basename "${j}").log 2>&1; done;
done
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eefadb25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_mrql_tests.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_mrql_tests.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_mrql_tests.sh
deleted file mode 100755
index a6788be..0000000
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_mrql_tests.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export JAVA_HOME=/home/ecarm002/java/jdk1.6.0_45
-REPEAT=${1}
-DATASET="hcn"
-
-for n in `seq 0 7`
-#for n in 0
-do
- date
- echo "Running q0${n} on ${DATASET} for MRQL."
- time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes 5 ~/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_${DATASET}/q0${n}.mrql >> weather_data/mrql/query_logs/${DATASET}/q0${n}.mrql.log 2>&1; done;
-done
-
-if which programname >/dev/null;
-then
- echo "Sending out e-mail notification."
- SUBJECT="MRQL Tests Finished (${DATASET})"
- EMAIL="ecarm002@ucr.edu"
- /bin/mail -s "${SUBJECT}" "${EMAIL}" <<EOM
- Completed all MRQL tests on ${DATASET}.
- EOM
-else
- echo "No mail command to use."
-fi;
\ No newline at end of file