You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/04/02 06:11:47 UTC
[07/50] [abbrv] git commit: More tweaks to make the benchmark cluster
queries.
More tweaks to make the benchmark cluster queries.
* Updated the data links to real partitions.
* Updated the query copy process.
* Simplified the example config file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/06b20cfe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/06b20cfe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/06b20cfe
Branch: refs/heads/prestonc/hash_join
Commit: 06b20cfe65b70a1702f324fe92504f39077c65b7
Parents: 25c38f4
Author: Preston Carman <pr...@apache.org>
Authored: Sat Mar 1 23:15:26 2014 -0800
Committer: Preston Carman <pr...@apache.org>
Committed: Tue Apr 1 20:56:23 2014 -0700
----------------------------------------------------------------------
.../noaa-ghcn-daily/scripts/weather_benchmark.py | 18 +++++++++++-------
.../noaa-ghcn-daily/scripts/weather_example.xml | 11 +----------
.../metadata/VXQueryCollectionDataSource.java | 1 -
3 files changed, 12 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/06b20cfe/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
index f0d93f8..4813bd9 100644
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py
@@ -130,11 +130,11 @@ class WeatherBenchmark:
def add_collection_links_for(self, real_path, link_path, index):
for collection in self.QUERY_COLLECTIONS:
- collection_path = link_path + "/" + collection
+ collection_path = link_path + collection + "/"
if not os.path.isdir(collection_path):
os.makedirs(collection_path)
if index >= 0:
- os.symlink(real_path + "/" + collection, collection_path + "/index" + str(index))
+ os.symlink(real_path + collection + "/", collection_path + "index" + str(index))
def get_partition_folders(self, base_path):
glob.glob(base_path + "partitions/d*_p*_i*")
@@ -158,7 +158,7 @@ class WeatherBenchmark:
os.makedirs(query_path)
# Copy query files.
- partition_paths = get_partition_paths(partitions, self.base_paths, "data_links/" + test + "/" + str(i) + "nodes")
+ partition_paths = get_cluster_link_paths_for_node(i, self.base_paths, "data_links/" + test)
self.copy_and_replace_query(query_path, partition_paths)
def copy_local_query_files(self, test):
@@ -205,12 +205,16 @@ class WeatherBenchmark:
def get_cluster_link_paths(nodes, base_paths, key="partitions"):
link_paths = []
for i in range(0, nodes):
- for j in range(0, len(base_paths)):
- new_link_path = base_paths[j] + key + "/" + str(i) + "nodes/"
- link_paths.append(new_link_path)
+ new_link_path = get_cluster_link_paths_for_node(i, base_paths, key)
+ link_paths.extend(new_link_path)
return link_paths
-
+def get_cluster_link_paths_for_node(node_id, base_paths, key="partitions"):
+ link_paths = []
+ for j in range(0, len(base_paths)):
+ new_link_path = base_paths[j] + key + "/" + str(node_id) + "nodes/"
+ link_paths.append(new_link_path)
+ return link_paths
def get_local_query_path(base_paths, test, partition):
return base_paths[0] + "queries/" + test + "/" + get_local_query_folder(len(base_paths), partition) + "/"
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/06b20cfe/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
index daf8f20..bc9f07f 100644
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
@@ -23,16 +23,7 @@
<cluster_ip>127.0.0.1</cluster_ip>
</node>
<dataset>
- <name>tiny</name>
- <test>local_speed_up</test>
- <save_path>/data</save_path>
- <partitions_per_path>1</partitions_per_path>
- <partitions_per_path>2</partitions_per_path>
- <partitions_per_path>4</partitions_per_path>
- <partitions_per_path>8</partitions_per_path>
- </dataset>
- <dataset>
- <name>small</name>
+ <name>tiny-example</name>
<test>local_speed_up</test>
<save_path>/data</save_path>
<partitions_per_path>1</partitions_per_path>
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/06b20cfe/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
index 1f9f0a7..e48d4ab 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
@@ -16,7 +16,6 @@
*/
package org.apache.vxquery.metadata;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;