You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/04/02 06:11:44 UTC
[04/50] [abbrv] git commit: Preparing for local speed up tests.
Preparing for local speed up tests.
I removed the queries that are currently not working. Also fixed a bug referencing files directly in the query instead of through the data links. Removed some debug code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/c10a8e9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/c10a8e9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/c10a8e9e
Branch: refs/heads/prestonc/hash_join
Commit: c10a8e9e81becf4bd44e842352cfd05f9918cfad
Parents: fda0991
Author: Preston Carman <pr...@apache.org>
Authored: Thu Feb 27 14:21:00 2014 -0800
Committer: Preston Carman <pr...@apache.org>
Committed: Tue Apr 1 20:56:23 2014 -0700
----------------------------------------------------------------------
.../noaa-ghcn-daily/scripts/weather_benchmark.py | 16 +++++++---------
.../noaa-ghcn-daily/scripts/weather_example.xml | 2 ++
2 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/c10a8e9e/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
index 6d9301e..f0d93f8 100644
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
@@ -35,8 +35,9 @@ class WeatherBenchmark:
QUERY_REPLACEMENT_KEY = "/tmp/1.0_partition_ghcnd_all_xml/"
QUERY_MASTER_FOLDER = "../queries/"
- QUERY_FILE_LIST = ["q00.xq", "q01.xq", "q02.xq", "q03.xq", "q04.xq", "q05.xq"]
- BENCHMARK_LOCAL_TESTS = ["local_speed_up"]
+ QUERY_FILE_LIST = ["q00.xq", "q01.xq", "q02.xq", "q03.xq"]
+# QUERY_FILE_LIST = ["q00.xq", "q01.xq", "q02.xq", "q03.xq", "q04.xq", "q05.xq"]
+ BENCHMARK_LOCAL_TESTS = ["local_speed_up", "local_batch_scale_out"]
BENCHMARK_CLUSTER_TESTS = ["speed_up", "batch_scale_out"]
QUERY_COLLECTIONS = ["sensors", "stations"]
@@ -68,12 +69,8 @@ class WeatherBenchmark:
# Match link paths to real data paths.
offset = 0
group_size = len(data_paths) / len(link_base_paths)
- print "g " + str(group_size)
- print link_base_paths
- print data_paths
for link_index, link_path in enumerate(link_base_paths):
for data_index, data_path in enumerate(data_paths):
- print index, offset, group_size, link_index, data_index
if offset <= data_index and data_index < offset + group_size:
self.add_collection_links_for(data_path, link_path, data_index)
offset += group_size
@@ -161,8 +158,8 @@ class WeatherBenchmark:
os.makedirs(query_path)
# Copy query files.
- node_partitions = get_partition_paths(partitions, self.base_paths, "data_links/" + test + "/" + str(i) + "nodes")
- self.copy_and_replace_query(query_path, node_partitions)
+ partition_paths = get_partition_paths(partitions, self.base_paths, "data_links/" + test + "/" + str(i) + "nodes")
+ self.copy_and_replace_query(query_path, partition_paths)
def copy_local_query_files(self, test):
for i in self.partitions:
@@ -172,7 +169,8 @@ class WeatherBenchmark:
os.makedirs(query_path)
# Copy query files.
- self.copy_and_replace_query(query_path, get_partition_paths(i, self.base_paths))
+ partition_paths = get_partition_paths(i, self.base_paths, "data_links/" + test)
+ self.copy_and_replace_query(query_path, partition_paths)
def copy_and_replace_query(self, query_path, replacement_list):
'''Copy the query files over to the query_path and replace the path
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/c10a8e9e/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
index 4f31dff..daf8f20 100644
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
@@ -35,6 +35,8 @@
<name>small</name>
<test>local_speed_up</test>
<save_path>/data</save_path>
+ <partitions_per_path>1</partitions_per_path>
+ <partitions_per_path>2</partitions_per_path>
<partitions_per_path>4</partitions_per_path>
<partitions_per_path>8</partitions_per_path>
</dataset>