You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/12/06 02:03:49 UTC
svn commit: r1548345 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/builtin/JsonMetadata.java
Author: rohini
Date: Fri Dec 6 01:03:49 2013
New Revision: 1548345
URL: http://svn.apache.org/r1548345
Log:
PIG-3612: Storing schema does not work cross cluster with PigStorage and JsonStorage (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/builtin/JsonMetadata.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1548345&r1=1548344&r2=1548345&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Dec 6 01:03:49 2013
@@ -56,6 +56,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3612: Storing schema does not work cross cluster with PigStorage and JsonStorage (rohini)
+
PIG-3607: PigRecordReader should report progress for each inputsplit processed (rohini)
PIG-3566: Cannot set useMatches of REGEX_EXTRACT_ALL and REGEX_EXTRACT (nezihyigitbasi via cheolsoo)
Modified: pig/trunk/src/org/apache/pig/builtin/JsonMetadata.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/JsonMetadata.java?rev=1548345&r1=1548344&r2=1548345&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/JsonMetadata.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/JsonMetadata.java Fri Dec 6 01:03:49 2013
@@ -105,9 +105,9 @@ public class JsonMetadata implements Loa
String[] locations = LoadFunc.getPathStrings(path);
for (String loc : locations) {
DataStorage storage;
-
+
storage = new HDataStorage(new Path(loc).toUri(), ConfigurationUtil.toProperties(conf));
-
+
String fullPath = FileLocalizer.fullPath(loc, storage);
if(storage.isContainer(fullPath)) {
@@ -119,7 +119,7 @@ public class JsonMetadata implements Loa
ElementDescriptor[] descriptors = storage.asCollection(loc);
for(ElementDescriptor descriptor : descriptors) {
ContainerDescriptor container = null;
-
+
if (descriptor instanceof HFile) {
Path descriptorPath = ((HPath) descriptor).getPath();
String fileName = descriptorPath.getName();
@@ -275,7 +275,8 @@ public class JsonMetadata implements Loa
@Override
public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
Configuration conf = job.getConfiguration();
- DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
+ DataStorage storage = new HDataStorage(new Path(location).toUri(),
+ ConfigurationUtil.toProperties(conf));
ElementDescriptor statFilePath = storage.asElement(location, statFileName);
if(!statFilePath.exists() && stats != null) {
try {
@@ -293,7 +294,8 @@ public class JsonMetadata implements Loa
@Override
public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
Configuration conf = job.getConfiguration();
- DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
+ DataStorage storage = new HDataStorage(new Path(location).toUri(),
+ ConfigurationUtil.toProperties(conf));
ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
if(!schemaFilePath.exists() && schema != null) {
try {