You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by me...@apache.org on 2015/12/22 23:48:05 UTC
drill git commit: DRILL-4192: Fix selection root when there is only a
single top level partition closes #307
Repository: drill
Updated Branches:
refs/heads/master 1ea3d6c3f -> ed0369b0a
DRILL-4192: Fix selection root when there is only a single top level partition
closes #307
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ed0369b0
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ed0369b0
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ed0369b0
Branch: refs/heads/master
Commit: ed0369b0ac4f4576d9cc9639184232618982e36b
Parents: 1ea3d6c
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Dec 15 17:39:03 2015 -0800
Committer: Mehant Baid <me...@gmail.com>
Committed: Tue Dec 22 13:20:33 2015 -0800
----------------------------------------------------------------------
.../drill/exec/store/dfs/FileSelection.java | 20 ++++++++++++-----
.../java/org/apache/drill/TestBugFixes.java | 23 ++++++++++++++++++++
.../nested_partition_1/simpledata.json | 1 +
.../nested_partition_2/simpledata.json | 1 +
4 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/ed0369b0/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
index e6cb653..6df3ffc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -28,6 +28,7 @@ import com.google.common.base.Predicate;
import com.google.common.base.Strings;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@@ -37,6 +38,7 @@ import org.apache.hadoop.fs.Path;
public class FileSelection {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSelection.class);
private static final String PATH_SEPARATOR = System.getProperty("file.separator");
+ private static final String WILD_CARD = "*";
private List<FileStatus> statuses;
@@ -213,14 +215,20 @@ public class FileSelection {
if (statuses == null || statuses.isEmpty()) {
selectionRoot = commonPathForFiles(files);
} else {
- if (statuses.size() == 1 && !Strings.isNullOrEmpty(root)) {
- final Path rootPath = new Path(root);
- final URI uri = statuses.get(0).getPath().toUri();
- final Path path = new Path(uri.getScheme(), uri.getAuthority(), rootPath.toUri().getPath());
- selectionRoot = path.toString();
+ if (Strings.isNullOrEmpty(root)) {
+ throw new DrillRuntimeException("Selection root is null or empty" + root);
+ }
+ // Handle wild card
+ final Path rootPath;
+ if (root.contains(WILD_CARD)) {
+ final String newRoot = root.substring(0, root.indexOf(WILD_CARD));
+ rootPath = new Path(newRoot);
} else {
- selectionRoot = commonPath(statuses);
+ rootPath = new Path(root);
}
+ final URI uri = statuses.get(0).getPath().toUri();
+ final Path path = new Path(uri.getScheme(), uri.getAuthority(), rootPath.toUri().getPath());
+ selectionRoot = path.toString();
}
return new FileSelection(statuses, files, selectionRoot);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/ed0369b0/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
index 202ed2a..c5062b2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
@@ -18,12 +18,15 @@
package org.apache.drill;
import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.junit.Ignore;
import org.junit.Test;
public class TestBugFixes extends BaseTestQuery {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBugFixes.class);
+ private static final String WORKING_PATH = TestTools.getWorkingPath();
+ private static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
@Test
public void leak1() throws Exception {
@@ -144,4 +147,24 @@ public class TestBugFixes extends BaseTestQuery {
.build().run();
}
+ @Test
+ public void testDRILL4192() throws Exception {
+ String query = (String.format("select dir0, dir1 from dfs_test.`%s/bugs/DRILL-4192` order by dir1", TEST_RES_PATH));
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("dir0", "dir1")
+ .baselineValues("single_top_partition", "nested_partition_1")
+ .baselineValues("single_top_partition", "nested_partition_2")
+ .go();
+
+ query = (String.format("select dir0, dir1 from dfs_test.`%s/bugs/DRILL-4192/*/nested_partition_1` order by dir1", TEST_RES_PATH));
+
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("dir0", "dir1")
+ .baselineValues("single_top_partition", "nested_partition_1")
+ .go();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/ed0369b0/exec/java-exec/src/test/resources/bugs/DRILL-4192/single_top_partition/nested_partition_1/simpledata.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/bugs/DRILL-4192/single_top_partition/nested_partition_1/simpledata.json b/exec/java-exec/src/test/resources/bugs/DRILL-4192/single_top_partition/nested_partition_1/simpledata.json
new file mode 100644
index 0000000..7a8d569
--- /dev/null
+++ b/exec/java-exec/src/test/resources/bugs/DRILL-4192/single_top_partition/nested_partition_1/simpledata.json
@@ -0,0 +1 @@
+{"col":1}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/ed0369b0/exec/java-exec/src/test/resources/bugs/DRILL-4192/single_top_partition/nested_partition_2/simpledata.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/bugs/DRILL-4192/single_top_partition/nested_partition_2/simpledata.json b/exec/java-exec/src/test/resources/bugs/DRILL-4192/single_top_partition/nested_partition_2/simpledata.json
new file mode 100644
index 0000000..7a8d569
--- /dev/null
+++ b/exec/java-exec/src/test/resources/bugs/DRILL-4192/single_top_partition/nested_partition_2/simpledata.json
@@ -0,0 +1 @@
+{"col":1}
\ No newline at end of file