You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/10/21 21:36:12 UTC
[11/14] git commit: Added a background processes to help with
parallelizing the data loading.
Added a background processes to help with parallelizing the data loading.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/31b3f4d9
Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/31b3f4d9
Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/31b3f4d9
Branch: refs/heads/master
Commit: 31b3f4d985c6344d1ccc15ce57fb57a1cb2e3011
Parents: eefadb2
Author: Preston Carman <pr...@apache.org>
Authored: Thu Oct 9 12:19:27 2014 -0700
Committer: Preston Carman <pr...@apache.org>
Committed: Thu Oct 9 12:19:27 2014 -0700
----------------------------------------------------------------------
.../mrql_scripts/load_node_file.sh | 37 ++++++++++++++++++++
.../mrql_scripts/run_group_test.sh | 15 +++-----
2 files changed, 41 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/vxquery/blob/31b3f4d9/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh
new file mode 100755
index 0000000..048274f
--- /dev/null
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z "${1}" ]
+then
+ echo "Please enter the node number."
+ exit
+fi
+
+
+# Add each sensor block
+cp saved/backups/mr/all_sensors_${1}.xml.gz disk1/hadoop/upload/
+gunzip disk1/hadoop/upload/all_sensors_${1}.xml.gz
+hadoop fs -copyFromLocal disk1/hadoop/upload/all_sensors_${1}.xml all/sensors
+rm -f disk1/hadoop/upload/all_sensors_${1}.xml
+
+# Add each station block
+cp saved/backups/mr/all_stations_${1}.xml.gz disk1/hadoop/upload/
+gunzip disk1/hadoop/upload/all_stations_${1}.xml.gz
+hadoop fs -copyFromLocal disk1/hadoop/upload/all_stations_${1}.xml all/stations
+rm -f disk1/hadoop/upload/all_stations_${1}.xml
http://git-wip-us.apache.org/repos/asf/vxquery/blob/31b3f4d9/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
index c34ec95..60dc255 100755
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh
@@ -33,19 +33,12 @@ hadoop fs -mkdir all/stations
n=0
while [ ${n} -lt ${NODES} ];
do
- # Add each sensor block
- cp saved/backups/mr/all_sensors_${n}.xml.gz disk1/hadoop/upload/
- gunzip disk1/hadoop/upload/all_sensors_${n}.xml.gz
- hadoop fs -copyFromLocal disk1/hadoop/upload/all_sensors_${n}.xml all/sensors
- rm -f disk1/hadoop/upload/all_sensors_${n}.xml
-
- # Add each station block
- cp saved/backups/mr/all_stations_${n}.xml.gz disk1/hadoop/upload/
- gunzip disk1/hadoop/upload/all_stations_${n}.xml.gz
- hadoop fs -copyFromLocal disk1/hadoop/upload/all_stations_${n}.xml all/stations
- rm -f disk1/hadoop/upload/all_stations_${n}.xml
+ sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ${n} &
done
+# After all files have been uploaded, continue.
+wait
+
# Start test
sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT}