You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2017/01/18 06:06:37 UTC

[1/2] incubator-carbondata git commit: Added support file:// for using local file

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 3a596f611 -> 7f8401d7f


Added support file:// for using local file

Added support file:// for using local file with fs.defaultFS is configured

Fixed testcase


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/95875151
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/95875151
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/95875151

Branch: refs/heads/master
Commit: 9587515136c373896c0c748777dc693f4ee2984c
Parents: 3a596f6
Author: ravipesala <ra...@gmail.com>
Authored: Tue Jan 17 18:57:44 2017 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 18 14:05:14 2017 +0800

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  2 +
 .../core/datastore/impl/DFSFileHolderImpl.java  |  2 +-
 .../core/datastore/impl/FileFactory.java        | 51 +++++++++++++++-----
 .../core/metadata/AbsoluteTableIdentifier.java  | 12 +++++
 .../carbondata/hadoop/CarbonInputFormat.java    | 11 +++--
 .../carbondata/spark/rdd/CarbonScanRDD.scala    |  2 +-
 6 files changed, 61 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 6a30bd4..c134f3b 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1127,6 +1127,8 @@ public final class CarbonCommonConstants {
 
   public static final String MAJOR = "major";
 
+  public static final String LOCAL_FILE_PREFIX = "file://";
+
   private CarbonCommonConstants() {
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
index c657fc9..8a97cbf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
@@ -58,7 +58,7 @@ public class DFSFileHolderImpl implements FileHolder {
     FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
     if (null == fileChannel) {
       Path pt = new Path(filePath);
-      FileSystem fs = FileSystem.get(FileFactory.getConfiguration());
+      FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
       fileChannel = fs.open(pt);
       fileNameAndStreamCache.put(filePath, fileChannel);
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index b4a5d0d..27931fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -29,11 +29,7 @@ import java.io.InputStream;
 import java.util.zip.GZIPInputStream;
 
 import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.filesystem.AlluxioCarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.ViewFSCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.*;
 import org.apache.carbondata.core.util.CarbonUtil;
 
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -91,7 +87,7 @@ public final class FileFactory {
   public static CarbonFile getCarbonFile(String path, FileType fileType) {
     switch (fileType) {
       case LOCAL:
-        return new LocalCarbonFile(path);
+        return new LocalCarbonFile(getUpdatedFilePath(path, fileType));
       case HDFS:
         return new HDFSCarbonFile(path);
       case ALLUXIO:
@@ -99,7 +95,7 @@ public final class FileFactory {
       case VIEWFS:
         return new ViewFSCarbonFile(path);
       default:
-        return new LocalCarbonFile(path);
+        return new LocalCarbonFile(getUpdatedFilePath(path, fileType));
     }
   }
 
@@ -116,6 +112,7 @@ public final class FileFactory {
     InputStream stream;
     switch (fileType) {
       case LOCAL:
+        path = getUpdatedFilePath(path, fileType);
         if (gzip) {
           stream = new GZIPInputStream(new FileInputStream(path));
         } else if (bzip2) {
@@ -171,6 +168,7 @@ public final class FileFactory {
         stream.seek(offset);
         return new DataInputStream(new BufferedInputStream(stream));
       default:
+        path = getUpdatedFilePath(path, fileType);
         FileInputStream fis = new FileInputStream(path);
         long actualSkipSize = 0;
         long skipSize = offset;
@@ -205,6 +203,7 @@ public final class FileFactory {
     path = path.replace("\\", "/");
     switch (fileType) {
       case LOCAL:
+        path = getUpdatedFilePath(path, fileType);
         return new DataOutputStream(
             new BufferedOutputStream(new FileOutputStream(path, append), bufferSize));
       case HDFS:
@@ -226,6 +225,7 @@ public final class FileFactory {
         }
         return stream;
       default:
+        path = getUpdatedFilePath(path, fileType);
         return new DataOutputStream(
             new BufferedOutputStream(new FileOutputStream(path), bufferSize));
     }
@@ -236,6 +236,7 @@ public final class FileFactory {
     path = path.replace("\\", "/");
     switch (fileType) {
       case LOCAL:
+        path = getUpdatedFilePath(path, fileType);
         return new DataOutputStream(
             new BufferedOutputStream(new FileOutputStream(path), bufferSize));
       case HDFS:
@@ -247,6 +248,7 @@ public final class FileFactory {
             fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
         return stream;
       default:
+        path = getUpdatedFilePath(path, fileType);
         return new DataOutputStream(
             new BufferedOutputStream(new FileOutputStream(path), bufferSize));
     }
@@ -277,6 +279,7 @@ public final class FileFactory {
 
       case LOCAL:
       default:
+        filePath = getUpdatedFilePath(filePath, fileType);
         File defaultFile = new File(filePath);
 
         if (performFileCheck) {
@@ -306,6 +309,7 @@ public final class FileFactory {
 
       case LOCAL:
       default:
+        filePath = getUpdatedFilePath(filePath, fileType);
         File defaultFile = new File(filePath);
         return defaultFile.exists();
     }
@@ -323,6 +327,7 @@ public final class FileFactory {
 
       case LOCAL:
       default:
+        filePath = getUpdatedFilePath(filePath, fileType);
         File file = new File(filePath);
         return file.createNewFile();
     }
@@ -340,6 +345,7 @@ public final class FileFactory {
 
       case LOCAL:
       default:
+        filePath = getUpdatedFilePath(filePath, fileType);
         File file = new File(filePath);
         return deleteAllFilesOfDir(file);
     }
@@ -370,6 +376,7 @@ public final class FileFactory {
         return fs.mkdirs(path);
       case LOCAL:
       default:
+        filePath = getUpdatedFilePath(filePath, fileType);
         File file = new File(filePath);
         return file.mkdirs();
     }
@@ -388,6 +395,7 @@ public final class FileFactory {
     path = path.replace("\\", "/");
     switch (fileType) {
       case LOCAL:
+        path = getUpdatedFilePath(path, fileType);
         return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path, true)));
       case HDFS:
       case ALLUXIO:
@@ -425,6 +433,7 @@ public final class FileFactory {
         return false;
       case LOCAL:
       default:
+        filePath = getUpdatedFilePath(filePath, fileType);
         File file = new File(filePath);
         return file.createNewFile();
     }
@@ -440,10 +449,10 @@ public final class FileFactory {
    * it removes the file:/ from the path
    *
    * @param filePath
+   * @param fileType
    * @return updated file path without url for local
    */
-  public static String getUpdatedFilePath(String filePath) {
-    FileType fileType = getFileType(filePath);
+  private static String getUpdatedFilePath(String filePath, FileType fileType) {
     switch (fileType) {
       case HDFS:
       case ALLUXIO:
@@ -451,13 +460,30 @@ public final class FileFactory {
         return filePath;
       case LOCAL:
       default:
-        Path pathWithoutSchemeAndAuthority =
-            Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
-        return pathWithoutSchemeAndAuthority.toString();
+        if (filePath != null && !filePath.isEmpty()) {
+          Path pathWithoutSchemeAndAuthority =
+              Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
+          return pathWithoutSchemeAndAuthority.toString();
+        } else {
+          return filePath;
+        }
     }
   }
 
   /**
+   * below method will be used to update the file path
+   * for local type
+   * it removes the file:/ from the path
+   *
+   * @param filePath
+   * @return updated file path without url for local
+   */
+  public static String getUpdatedFilePath(String filePath) {
+    FileType fileType = getFileType(filePath);
+    return getUpdatedFilePath(filePath, fileType);
+  }
+
+  /**
    * It computes size of directory
    *
    * @param filePath
@@ -475,6 +501,7 @@ public final class FileFactory {
         return fs.getContentSummary(path).getLength();
       case LOCAL:
       default:
+        filePath = getUpdatedFilePath(filePath, fileType);
         File file = new File(filePath);
         return FileUtils.sizeOfDirectory(file);
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
index f6263f4..aed248c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -38,6 +38,9 @@ public class AbsoluteTableIdentifier implements Serializable {
    */
   private String storePath;
 
+
+  private boolean isLocalPath;
+
   /**
    * carbon table identifier which will have table name and table database
    * name
@@ -47,6 +50,7 @@ public class AbsoluteTableIdentifier implements Serializable {
   public AbsoluteTableIdentifier(String storePath, CarbonTableIdentifier carbonTableIdentifier) {
     //TODO this should be moved to common place where path handling will be handled
     this.storePath = FileFactory.getUpdatedFilePath(storePath);
+    isLocalPath = storePath.startsWith(CarbonCommonConstants.LOCAL_FILE_PREFIX);
     this.carbonTableIdentifier = carbonTableIdentifier;
   }
 
@@ -91,6 +95,14 @@ public class AbsoluteTableIdentifier implements Serializable {
         File.separator + getCarbonTableIdentifier().getTableName();
   }
 
+  public String appendWithLocalPrefix(String path) {
+    if (isLocalPath) {
+      return CarbonCommonConstants.LOCAL_FILE_PREFIX + path;
+    } else {
+      return path;
+    }
+  }
+
   /**
    * to get the hash code
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index b320204..5acd1d5 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -175,8 +175,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
     }
   }
 
-  public static CarbonTablePath getTablePath(Configuration configuration) {
-    AbsoluteTableIdentifier absIdentifier = getAbsoluteTableIdentifier(configuration);
+  private static CarbonTablePath getTablePath(AbsoluteTableIdentifier absIdentifier) {
     return CarbonStorePath.getCarbonTablePath(absIdentifier);
   }
 
@@ -677,7 +676,8 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
     }
 
     PathFilter inputFilter = getDataFileFilter();
-    CarbonTablePath tablePath = getTablePath(job.getConfiguration());
+    AbsoluteTableIdentifier absIdentifier = getAbsoluteTableIdentifier(job.getConfiguration());
+    CarbonTablePath tablePath = getTablePath(absIdentifier);
 
     // get tokens for all the required FileSystem for table path
     TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { tablePath },
@@ -689,9 +689,10 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
 
       for (int j = 0; j < segmentsToConsider.length; ++j) {
         String segmentId = segmentsToConsider[j];
-        Path segmentPath = new Path(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
+        String dataDirectoryPath = absIdentifier
+            .appendWithLocalPrefix(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
+        Path segmentPath = new Path(dataDirectoryPath);
         FileSystem fs = segmentPath.getFileSystem(job.getConfiguration());
-
         RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(segmentPath);
         while (iter.hasNext()) {
           LocatedFileStatus stat = iter.next();

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index b06fe14..3415a81 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -250,7 +250,7 @@ class CarbonScanRDD(
 
   private def createInputFormat(conf: Configuration): CarbonInputFormat[Object] = {
     val format = new CarbonInputFormat[Object]
-    CarbonInputFormat.setTablePath(conf, identifier.getTablePath)
+    CarbonInputFormat.setTablePath(conf, identifier.appendWithLocalPrefix(identifier.getTablePath))
     CarbonInputFormat.setFilterPredicates(conf, filterExpression)
     CarbonInputFormat.setColumnProjection(conf, columnProjection)
     format


[2/2] incubator-carbondata git commit: [CARBONDATA-476][CARBONDATA-644]Added support file:// for using local file with fs.defaultFS configured This closes #545

Posted by ch...@apache.org.
[CARBONDATA-476][CARBONDATA-644]Added support file:// for using local file with fs.defaultFS configured This closes #545


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7f8401d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7f8401d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7f8401d7

Branch: refs/heads/master
Commit: 7f8401d7fbbcb1d0e81158c9bad998e1fb1c22c2
Parents: 3a596f6 9587515
Author: chenliang613 <ch...@huawei.com>
Authored: Wed Jan 18 14:06:15 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 18 14:06:15 2017 +0800

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  2 +
 .../core/datastore/impl/DFSFileHolderImpl.java  |  2 +-
 .../core/datastore/impl/FileFactory.java        | 51 +++++++++++++++-----
 .../core/metadata/AbsoluteTableIdentifier.java  | 12 +++++
 .../carbondata/hadoop/CarbonInputFormat.java    | 11 +++--
 .../carbondata/spark/rdd/CarbonScanRDD.scala    |  2 +-
 6 files changed, 61 insertions(+), 19 deletions(-)
----------------------------------------------------------------------