You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2013/11/19 07:59:21 UTC
svn commit: r1543321 -
/incubator/vxquery/trunk/vxquery/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py
Author: prestonc
Date: Tue Nov 19 06:59:20 2013
New Revision: 1543321
URL: http://svn.apache.org/r1543321
Log:
Added a check to only copy directories that exists. The progress file may include stations without data.
Modified:
incubator/vxquery/trunk/vxquery/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py
Modified: incubator/vxquery/trunk/vxquery/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py?rev=1543321&r1=1543320&r2=1543321&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py Tue Nov 19 06:59:20 2013
@@ -107,14 +107,16 @@ class WeatherDataFiles:
current_partition = 0
csv_sorted = self.get_csv_in_partition_order()
for item in csv_sorted:
- file_name = item.rsplit(self.SEPERATOR)[self.INDEX_DATA_FILE_NAME]
+ row_contents = self.progress_data[row].rsplit(self.SEPERATOR)
+ file_name = row_contents[self.INDEX_DATA_FILE_NAME]
station_id = os.path.basename(file_name).split('.')[0]
# Copy station files
for type in ("sensors", "stations"):
file_path = build_base_save_folder(save_path, station_id, type)
new_file_path = build_base_save_folder(partition_paths[current_partition], station_id, type)
- distutils.dir_util.copy_tree(file_path, new_file_path)
+ if os.path.isdir(file_path):
+ distutils.dir_util.copy_tree(file_path, new_file_path)
# Update partition
current_partition += 1