You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/10/21 21:36:13 UTC
[12/14] git commit: More tweaking to get the MRQL benchmark to work
efficiently.
More tweaking to get the MRQL benchmark to work efficiently.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/7f06298f
Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/7f06298f
Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/7f06298f
Branch: refs/heads/master
Commit: 7f06298fa350ca8b0a81bb9ffb78aa29b26368c1
Parents: 31b3f4d
Author: Preston Carman <pr...@apache.org>
Authored: Thu Oct 9 15:29:16 2014 -0700
Committer: Preston Carman <pr...@apache.org>
Committed: Thu Oct 9 15:29:16 2014 -0700
----------------------------------------------------------------------
.../other_systems/mrql_scripts/README.md | 23 ++++++++++++++++++++
.../other_systems/mrql_scripts/clear.sh | 4 ----
.../mrql_scripts/load_node_file.sh | 17 ++++++++-------
.../mrql_scripts/run_group_test.sh | 22 +++++++++++++------
.../other_systems/mrql_scripts/start.sh | 20 -----------------
.../other_systems/mrql_scripts/stop.sh | 20 -----------------
6 files changed, 47 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/README.md
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/README.md b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/README.md
new file mode 100644
index 0000000..53a7ecf
--- /dev/null
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/README.md
@@ -0,0 +1,23 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+clear.sh
+
+hadoop namenode -format
+
+run_group_test.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
index b775de2..9a912b2 100755
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh
@@ -17,12 +17,8 @@
# limitations under the License.
#
-# Hadoop data reset
-hadoop namenode -format
-
# Remove data
rm -rf disk1/hadoop/data
rm -rf disk2/hadoop/data
rm -rf disk1/hadoop/tmp
rm -rf disk1/hadoop/logs
-
http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh
index 048274f..ead0902 100755
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh
@@ -23,15 +23,16 @@ then
exit
fi
+echo "Loading node ${1} data file in to cluster."
# Add each sensor block
-cp saved/backups/mr/all_sensors_${1}.xml.gz disk1/hadoop/upload/
-gunzip disk1/hadoop/upload/all_sensors_${1}.xml.gz
-hadoop fs -copyFromLocal disk1/hadoop/upload/all_sensors_${1}.xml all/sensors
-rm -f disk1/hadoop/upload/all_sensors_${1}.xml
+cp saved/backups/mr/all_sensors_${1}.xml.gz disk1/hadoop/
+gunzip disk1/hadoop/all_sensors_${1}.xml.gz
+hadoop fs -copyFromLocal disk1/hadoop/all_sensors_${1}.xml all/sensors
+rm -f disk1/hadoop/all_sensors_${1}.xml
# Add each station block
-cp saved/backups/mr/all_stations_${1}.xml.gz disk1/hadoop/upload/
-gunzip disk1/hadoop/upload/all_stations_${1}.xml.gz
-hadoop fs -copyFromLocal disk1/hadoop/upload/all_stations_${1}.xml all/stations
-rm -f disk1/hadoop/upload/all_stations_${1}.xml
+cp saved/backups/mr/all_stations_${1}.xml.gz disk1/hadoop/
+gunzip disk1/hadoop/all_stations_${1}.xml.gz
+hadoop fs -copyFromLocal disk1/hadoop/all_stations_${1}.xml all/stations
+rm -f disk1/hadoop/all_stations_${1}.xml
http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
index 60dc255..f42a451 100755
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
@@ -17,28 +17,36 @@
# limitations under the License.
#
-NODES=2
+if [ -z "${1}" ]
+then
+ echo "Please enter the number of nodes."
+ exit
+fi
+
+NODES=${1}
REPEAT=1
# Start Hadoop
sh saved/hadoop/hadoop-1.2.1/bin/start-all.sh
+sleep 10
+
# Prepare hadoop file system
hadoop fs -mkdir all
+hadoop fs -ls
hadoop fs -mkdir all/sensors
hadoop fs -mkdir all/stations
+hadoop fs -ls all
# Upload test data
-n=0
-while [ ${n} -lt ${NODES} ];
+COUNTER=0
+while [ ${COUNTER} -lt ${NODES} ];
do
- sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ${n} &
+ sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ${COUNTER}
+ let COUNTER=COUNTER+1
done
-# After all files have been uploaded, continue.
-wait
-
# Start test
sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh
deleted file mode 100755
index a1766c9..0000000
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-bin/start-all.sh
http://git-wip-us.apache.org/repos/asf/vxquery/blob/7f06298f/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh
deleted file mode 100755
index e49d818..0000000
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-bin/stop-all.sh