You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2017/01/18 06:06:37 UTC
[1/2] incubator-carbondata git commit: Added support file:// for
using local file
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 3a596f611 -> 7f8401d7f
Added support file:// for using local file
Added support file:// for using local file with fs.defaultFS is configured
Fixed testcase
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/95875151
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/95875151
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/95875151
Branch: refs/heads/master
Commit: 9587515136c373896c0c748777dc693f4ee2984c
Parents: 3a596f6
Author: ravipesala <ra...@gmail.com>
Authored: Tue Jan 17 18:57:44 2017 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 18 14:05:14 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 +
.../core/datastore/impl/DFSFileHolderImpl.java | 2 +-
.../core/datastore/impl/FileFactory.java | 51 +++++++++++++++-----
.../core/metadata/AbsoluteTableIdentifier.java | 12 +++++
.../carbondata/hadoop/CarbonInputFormat.java | 11 +++--
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
6 files changed, 61 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 6a30bd4..c134f3b 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1127,6 +1127,8 @@ public final class CarbonCommonConstants {
public static final String MAJOR = "major";
+ public static final String LOCAL_FILE_PREFIX = "file://";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
index c657fc9..8a97cbf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
@@ -58,7 +58,7 @@ public class DFSFileHolderImpl implements FileHolder {
FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
if (null == fileChannel) {
Path pt = new Path(filePath);
- FileSystem fs = FileSystem.get(FileFactory.getConfiguration());
+ FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
fileChannel = fs.open(pt);
fileNameAndStreamCache.put(filePath, fileChannel);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index b4a5d0d..27931fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -29,11 +29,7 @@ import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.filesystem.AlluxioCarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.ViewFSCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.*;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -91,7 +87,7 @@ public final class FileFactory {
public static CarbonFile getCarbonFile(String path, FileType fileType) {
switch (fileType) {
case LOCAL:
- return new LocalCarbonFile(path);
+ return new LocalCarbonFile(getUpdatedFilePath(path, fileType));
case HDFS:
return new HDFSCarbonFile(path);
case ALLUXIO:
@@ -99,7 +95,7 @@ public final class FileFactory {
case VIEWFS:
return new ViewFSCarbonFile(path);
default:
- return new LocalCarbonFile(path);
+ return new LocalCarbonFile(getUpdatedFilePath(path, fileType));
}
}
@@ -116,6 +112,7 @@ public final class FileFactory {
InputStream stream;
switch (fileType) {
case LOCAL:
+ path = getUpdatedFilePath(path, fileType);
if (gzip) {
stream = new GZIPInputStream(new FileInputStream(path));
} else if (bzip2) {
@@ -171,6 +168,7 @@ public final class FileFactory {
stream.seek(offset);
return new DataInputStream(new BufferedInputStream(stream));
default:
+ path = getUpdatedFilePath(path, fileType);
FileInputStream fis = new FileInputStream(path);
long actualSkipSize = 0;
long skipSize = offset;
@@ -205,6 +203,7 @@ public final class FileFactory {
path = path.replace("\\", "/");
switch (fileType) {
case LOCAL:
+ path = getUpdatedFilePath(path, fileType);
return new DataOutputStream(
new BufferedOutputStream(new FileOutputStream(path, append), bufferSize));
case HDFS:
@@ -226,6 +225,7 @@ public final class FileFactory {
}
return stream;
default:
+ path = getUpdatedFilePath(path, fileType);
return new DataOutputStream(
new BufferedOutputStream(new FileOutputStream(path), bufferSize));
}
@@ -236,6 +236,7 @@ public final class FileFactory {
path = path.replace("\\", "/");
switch (fileType) {
case LOCAL:
+ path = getUpdatedFilePath(path, fileType);
return new DataOutputStream(
new BufferedOutputStream(new FileOutputStream(path), bufferSize));
case HDFS:
@@ -247,6 +248,7 @@ public final class FileFactory {
fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
return stream;
default:
+ path = getUpdatedFilePath(path, fileType);
return new DataOutputStream(
new BufferedOutputStream(new FileOutputStream(path), bufferSize));
}
@@ -277,6 +279,7 @@ public final class FileFactory {
case LOCAL:
default:
+ filePath = getUpdatedFilePath(filePath, fileType);
File defaultFile = new File(filePath);
if (performFileCheck) {
@@ -306,6 +309,7 @@ public final class FileFactory {
case LOCAL:
default:
+ filePath = getUpdatedFilePath(filePath, fileType);
File defaultFile = new File(filePath);
return defaultFile.exists();
}
@@ -323,6 +327,7 @@ public final class FileFactory {
case LOCAL:
default:
+ filePath = getUpdatedFilePath(filePath, fileType);
File file = new File(filePath);
return file.createNewFile();
}
@@ -340,6 +345,7 @@ public final class FileFactory {
case LOCAL:
default:
+ filePath = getUpdatedFilePath(filePath, fileType);
File file = new File(filePath);
return deleteAllFilesOfDir(file);
}
@@ -370,6 +376,7 @@ public final class FileFactory {
return fs.mkdirs(path);
case LOCAL:
default:
+ filePath = getUpdatedFilePath(filePath, fileType);
File file = new File(filePath);
return file.mkdirs();
}
@@ -388,6 +395,7 @@ public final class FileFactory {
path = path.replace("\\", "/");
switch (fileType) {
case LOCAL:
+ path = getUpdatedFilePath(path, fileType);
return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path, true)));
case HDFS:
case ALLUXIO:
@@ -425,6 +433,7 @@ public final class FileFactory {
return false;
case LOCAL:
default:
+ filePath = getUpdatedFilePath(filePath, fileType);
File file = new File(filePath);
return file.createNewFile();
}
@@ -440,10 +449,10 @@ public final class FileFactory {
* it removes the file:/ from the path
*
* @param filePath
+ * @param fileType
* @return updated file path without url for local
*/
- public static String getUpdatedFilePath(String filePath) {
- FileType fileType = getFileType(filePath);
+ private static String getUpdatedFilePath(String filePath, FileType fileType) {
switch (fileType) {
case HDFS:
case ALLUXIO:
@@ -451,13 +460,30 @@ public final class FileFactory {
return filePath;
case LOCAL:
default:
- Path pathWithoutSchemeAndAuthority =
- Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
- return pathWithoutSchemeAndAuthority.toString();
+ if (filePath != null && !filePath.isEmpty()) {
+ Path pathWithoutSchemeAndAuthority =
+ Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
+ return pathWithoutSchemeAndAuthority.toString();
+ } else {
+ return filePath;
+ }
}
}
/**
+ * below method will be used to update the file path
+ * for local type
+ * it removes the file:/ from the path
+ *
+ * @param filePath
+ * @return updated file path without url for local
+ */
+ public static String getUpdatedFilePath(String filePath) {
+ FileType fileType = getFileType(filePath);
+ return getUpdatedFilePath(filePath, fileType);
+ }
+
+ /**
* It computes size of directory
*
* @param filePath
@@ -475,6 +501,7 @@ public final class FileFactory {
return fs.getContentSummary(path).getLength();
case LOCAL:
default:
+ filePath = getUpdatedFilePath(filePath, fileType);
File file = new File(filePath);
return FileUtils.sizeOfDirectory(file);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
index f6263f4..aed248c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -38,6 +38,9 @@ public class AbsoluteTableIdentifier implements Serializable {
*/
private String storePath;
+
+ private boolean isLocalPath;
+
/**
* carbon table identifier which will have table name and table database
* name
@@ -47,6 +50,7 @@ public class AbsoluteTableIdentifier implements Serializable {
public AbsoluteTableIdentifier(String storePath, CarbonTableIdentifier carbonTableIdentifier) {
//TODO this should be moved to common place where path handling will be handled
this.storePath = FileFactory.getUpdatedFilePath(storePath);
+ isLocalPath = storePath.startsWith(CarbonCommonConstants.LOCAL_FILE_PREFIX);
this.carbonTableIdentifier = carbonTableIdentifier;
}
@@ -91,6 +95,14 @@ public class AbsoluteTableIdentifier implements Serializable {
File.separator + getCarbonTableIdentifier().getTableName();
}
+ public String appendWithLocalPrefix(String path) {
+ if (isLocalPath) {
+ return CarbonCommonConstants.LOCAL_FILE_PREFIX + path;
+ } else {
+ return path;
+ }
+ }
+
/**
* to get the hash code
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index b320204..5acd1d5 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -175,8 +175,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
}
}
- public static CarbonTablePath getTablePath(Configuration configuration) {
- AbsoluteTableIdentifier absIdentifier = getAbsoluteTableIdentifier(configuration);
+ private static CarbonTablePath getTablePath(AbsoluteTableIdentifier absIdentifier) {
return CarbonStorePath.getCarbonTablePath(absIdentifier);
}
@@ -677,7 +676,8 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
}
PathFilter inputFilter = getDataFileFilter();
- CarbonTablePath tablePath = getTablePath(job.getConfiguration());
+ AbsoluteTableIdentifier absIdentifier = getAbsoluteTableIdentifier(job.getConfiguration());
+ CarbonTablePath tablePath = getTablePath(absIdentifier);
// get tokens for all the required FileSystem for table path
TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { tablePath },
@@ -689,9 +689,10 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
for (int j = 0; j < segmentsToConsider.length; ++j) {
String segmentId = segmentsToConsider[j];
- Path segmentPath = new Path(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
+ String dataDirectoryPath = absIdentifier
+ .appendWithLocalPrefix(tablePath.getCarbonDataDirectoryPath(partition, segmentId));
+ Path segmentPath = new Path(dataDirectoryPath);
FileSystem fs = segmentPath.getFileSystem(job.getConfiguration());
-
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(segmentPath);
while (iter.hasNext()) {
LocatedFileStatus stat = iter.next();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/95875151/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index b06fe14..3415a81 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -250,7 +250,7 @@ class CarbonScanRDD(
private def createInputFormat(conf: Configuration): CarbonInputFormat[Object] = {
val format = new CarbonInputFormat[Object]
- CarbonInputFormat.setTablePath(conf, identifier.getTablePath)
+ CarbonInputFormat.setTablePath(conf, identifier.appendWithLocalPrefix(identifier.getTablePath))
CarbonInputFormat.setFilterPredicates(conf, filterExpression)
CarbonInputFormat.setColumnProjection(conf, columnProjection)
format
[2/2] incubator-carbondata git commit:
[CARBONDATA-476][CARBONDATA-644]Added support file:// for using local file
with fs.defaultFS configured This closes #545
Posted by ch...@apache.org.
[CARBONDATA-476][CARBONDATA-644]Added support file:// for using local file with fs.defaultFS configured This closes #545
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7f8401d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7f8401d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7f8401d7
Branch: refs/heads/master
Commit: 7f8401d7fbbcb1d0e81158c9bad998e1fb1c22c2
Parents: 3a596f6 9587515
Author: chenliang613 <ch...@huawei.com>
Authored: Wed Jan 18 14:06:15 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 18 14:06:15 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 +
.../core/datastore/impl/DFSFileHolderImpl.java | 2 +-
.../core/datastore/impl/FileFactory.java | 51 +++++++++++++++-----
.../core/metadata/AbsoluteTableIdentifier.java | 12 +++++
.../carbondata/hadoop/CarbonInputFormat.java | 11 +++--
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
6 files changed, 61 insertions(+), 19 deletions(-)
----------------------------------------------------------------------