You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/04/02 06:11:48 UTC
[08/50] [abbrv] git commit: Clean up items for the weather scripts.
Clean up items for the weather scripts.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/b7061722
Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/b7061722
Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/b7061722
Branch: refs/heads/prestonc/hash_join
Commit: b706172249fbcb50e92ce8f96d09c13281737e70
Parents: 735140b
Author: Preston Carman <pr...@apache.org>
Authored: Tue Mar 4 17:39:53 2014 -0800
Committer: Preston Carman <pr...@apache.org>
Committed: Tue Apr 1 20:56:23 2014 -0700
----------------------------------------------------------------------
.../resources/noaa-ghcn-daily/scripts/weather_benchmark.py | 8 ++++----
.../main/resources/noaa-ghcn-daily/scripts/weather_cli.py | 2 +-
.../noaa-ghcn-daily/scripts/weather_convert_to_xml.py | 2 +-
3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/b7061722/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
index a812a1f..8a032c2 100644
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
@@ -14,7 +14,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import glob
import os.path
import linecache
import distutils.core
@@ -135,9 +134,6 @@ class WeatherBenchmark:
if index >= 0:
os.symlink(real_path + collection + "/", collection_path + "index" + str(index))
- def get_partition_folders(self, base_path):
- glob.glob(base_path + "partitions/d*_p*_i*")
-
def copy_query_files(self):
for test in self.dataset.get_tests():
if test in self.BENCHMARK_LOCAL_TESTS:
@@ -149,6 +145,8 @@ class WeatherBenchmark:
exit()
def copy_cluster_query_files(self, test):
+ '''Determine the data_link path for cluster query files and copy with
+ new location for collection.'''
partitions = self.dataset.get_partitions()[0]
for i in range(len(self.nodes)):
query_path = get_cluster_query_path(self.base_paths, test, i)
@@ -161,6 +159,8 @@ class WeatherBenchmark:
self.copy_and_replace_query(query_path, partition_paths)
def copy_local_query_files(self, test):
+ '''Determine the data_link path for local query files and copy with
+ new location for collection.'''
for i in self.partitions:
query_path = get_local_query_path(self.base_paths, test, i)
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/b7061722/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_cli.py
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_cli.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_cli.py
index 0f529f2..103c0d1 100644
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_cli.py
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_cli.py
@@ -119,7 +119,7 @@ def main(argv):
# Required fields to run the script.
if config.get_save_path() == "" or not os.path.exists(config.get_save_path()):
- print 'Error: The save directory option must be supplied: --save_directory (-d).'
+ print 'Error: The save directory option must be supplied in the config file.'
sys.exit()
# Set up downloads folder.
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/b7061722/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_convert_to_xml.py
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_convert_to_xml.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_convert_to_xml.py
index 514f5f9..36aff16 100644
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_convert_to_xml.py
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_convert_to_xml.py
@@ -472,7 +472,7 @@ class WeatherWebServiceMonthlyXMLFile(WeatherConvertToXML):
# Station data
def download_station_data(self, station_id, token, reset = False):
import time
- time.sleep(5)
+ time.sleep(10)
# Make sure the station folder is available.
ghcnd_xml_station_path = self.get_base_folder(station_id, "stations")
if not os.path.isdir(ghcnd_xml_station_path):