You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/07/12 15:04:43 UTC
[07/50] [abbrv] carbondata git commit: [CARBONDATA-1236] Support
absolute path without scheme in loading - change in logic
[CARBONDATA-1236] Support absolute path without scheme in loading - change in logic
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c671c5b6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c671c5b6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c671c5b6
Branch: refs/heads/datamap
Commit: c671c5b60ad4989643e9b6f92540b60c2c540145
Parents: 126a041
Author: dhatchayani <dh...@gmail.com>
Authored: Thu Jun 29 17:59:33 2017 +0530
Committer: dhatchayani <dh...@gmail.com>
Committed: Thu Jun 29 18:02:50 2017 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 50 +++++------
.../carbondata/core/util/CarbonUtilTest.java | 89 +++++++++++++++++++-
2 files changed, 112 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c671c5b6/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 6e0348b..8298600 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -697,32 +697,32 @@ public final class CarbonUtil {
*/
public static String checkAndAppendHDFSUrl(String filePath) {
String currentPath = filePath;
- if (null != filePath && filePath.length() != 0
- && FileFactory.getFileType(filePath) != FileFactory.FileType.HDFS
- && FileFactory.getFileType(filePath) != FileFactory.FileType.VIEWFS) {
- if (!filePath.startsWith("/")) {
- filePath = "/" + filePath;
- }
- String baseDFSUrl = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_DDL_BASE_HDFS_URL);
- String dfsUrl = conf.get(FS_DEFAULT_FS);
- if (null != baseDFSUrl) {
- if (!baseDFSUrl.startsWith("/")) {
- baseDFSUrl = "/" + baseDFSUrl;
- }
- if (null != dfsUrl && (dfsUrl.startsWith(HDFS_PREFIX) || dfsUrl
- .startsWith(VIEWFS_PREFIX))) {
- baseDFSUrl = dfsUrl + baseDFSUrl;
- }
- if (baseDFSUrl.endsWith("/")) {
- baseDFSUrl = baseDFSUrl.substring(0, baseDFSUrl.length() - 1);
- }
- currentPath = baseDFSUrl + filePath;
- } else {
- currentPath = dfsUrl + filePath;
- }
+ String defaultFsUrl = conf.get(FS_DEFAULT_FS);
+ String baseDFSUrl = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_DDL_BASE_HDFS_URL, "");
+ if (checkIfPrefixExists(filePath)) {
+ return currentPath;
}
- return currentPath;
+ if (baseDFSUrl.endsWith("/")) {
+ baseDFSUrl = baseDFSUrl.substring(0, baseDFSUrl.length() - 1);
+ }
+ if (!filePath.startsWith("/")) {
+ filePath = "/" + filePath;
+ }
+ currentPath = baseDFSUrl + filePath;
+ if (checkIfPrefixExists(currentPath)) {
+ return currentPath;
+ }
+ if (defaultFsUrl == null) {
+ return currentPath;
+ }
+ return defaultFsUrl + currentPath;
+ }
+
+ private static boolean checkIfPrefixExists(String path) {
+ final String lowerPath = path.toLowerCase();
+ return lowerPath.startsWith(HDFS_PREFIX) || lowerPath.startsWith(VIEWFS_PREFIX) || lowerPath
+ .startsWith("file://") || lowerPath.startsWith(ALLUXIO_PREFIX);
}
public static String getCarbonStorePath() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c671c5b6/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index b42c57e..776059f 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -342,7 +342,7 @@ public class CarbonUtilTest {
}
};
String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase");
- assertEquals(hdfsURL, "/BASE_URL/../core/src/test/resources/testDatabase");
+ assertEquals(hdfsURL, "file:///BASE_URL/../core/src/test/resources/testDatabase");
}
@Test public void testToCheckAndAppendHDFSUrlWithBlackSlash() {
@@ -357,7 +357,7 @@ public class CarbonUtilTest {
}
};
String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase");
- assertEquals(hdfsURL, "/BASE_URL/../core/src/test/resources/testDatabase");
+ assertEquals(hdfsURL, "file:///BASE_URL/../core/src/test/resources/testDatabase");
}
@Test public void testToCheckAndAppendHDFSUrlWithNull() {
@@ -375,6 +375,91 @@ public class CarbonUtilTest {
assertEquals(hdfsURL, "file:////../core/src/test/resources/testDatabase");
}
+ @Test public void testToCheckAndAppendHDFSUrlWithHdfs() {
+ new MockUp<FileFactory>() {
+ @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
+ return FileFactory.FileType.HDFS;
+ }
+ };
+ new MockUp<org.apache.hadoop.conf.Configuration>() {
+ @SuppressWarnings("unused") @Mock public String get(String name) {
+ return "hdfs://";
+ }
+ };
+ String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("hdfs://ha/core/src/test/resources/testDatabase");
+ assertEquals(hdfsURL, "hdfs://ha/core/src/test/resources/testDatabase");
+ }
+
+ @Test public void testToCheckAndAppendHDFSUrlWithDoubleSlashLocal() {
+ new MockUp<FileFactory>() {
+ @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
+ return FileFactory.FileType.LOCAL;
+ }
+ };
+ new MockUp<CarbonProperties>() {
+ @SuppressWarnings("unused") @Mock public String getProperty(String key) {
+ return "/opt/";
+ }
+ };
+ String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
+ assertEquals(hdfsURL, "file:////opt/core/src/test/resources/testDatabase");
+ }
+
+ @Test public void testToCheckAndAppendHDFSUrlWithDoubleSlashHDFS() {
+ new MockUp<FileFactory>() {
+ @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
+ return FileFactory.FileType.HDFS;
+ }
+ };
+ new MockUp<org.apache.hadoop.conf.Configuration>() {
+ @SuppressWarnings("unused") @Mock public String get(String name) {
+ return "hdfs://";
+ }
+ };
+ new MockUp<CarbonProperties>() {
+ @SuppressWarnings("unused") @Mock public String getProperty(String key) {
+ return "/opt/";
+ }
+ };
+ String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
+ assertEquals(hdfsURL, "hdfs:///opt/core/src/test/resources/testDatabase");
+ }
+
+ @Test public void testToCheckAndAppendHDFSUrlWithBaseURLPrefix() {
+ new MockUp<FileFactory>() {
+ @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
+ return FileFactory.FileType.HDFS;
+ }
+ };
+ new MockUp<CarbonProperties>() {
+ @SuppressWarnings("unused") @Mock public String getProperty(String key) {
+ return "hdfs://ha/opt/";
+ }
+ };
+ String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
+ assertEquals(hdfsURL, "hdfs://ha/opt/core/src/test/resources/testDatabase");
+ }
+
+ @Test public void testToCheckAndAppendHDFSUrlWithBaseURLFile() {
+ new MockUp<FileFactory>() {
+ @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) {
+ return FileFactory.FileType.HDFS;
+ }
+ };
+ new MockUp<CarbonProperties>() {
+ @SuppressWarnings("unused") @Mock public String getProperty(String key) {
+ return "file:///";
+ }
+ };
+ String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
+ assertEquals(hdfsURL, "file:///core/src/test/resources/testDatabase");
+ }
+
+ @Test public void testToCheckAndAppendHDFSUrlWithFilepathPrefix() {
+ String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("file:///core/src/test/resources/testDatabase");
+ assertEquals(hdfsURL, "file:///core/src/test/resources/testDatabase");
+ }
+
@Test public void testForisFileExists() {
assertTrue(CarbonUtil.isFileExists("../core/src/test/resources/testFile.txt"));
}