You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by sm...@apache.org on 2015/05/10 00:15:39 UTC
drill git commit: DRILL-2532: Fix glob implementation
Repository: drill
Updated Branches:
refs/heads/master 4844b0e61 -> 76cf72969
Updated Tags: refs/tags/0.9.0 [created] 78fd658d2
refs/tags/drill-root-0.4.0-incubating [created] caa15e262
DRILL-2532: Fix glob implementation
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/76cf7296
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/76cf7296
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/76cf7296
Branch: refs/heads/master
Commit: 76cf72969014788a882204afd88fbaf944a981f9
Parents: 4844b0e
Author: AdamPD <ad...@pharmadata.net.au>
Authored: Tue Mar 24 12:33:14 2015 +1000
Committer: Steven Phillips <sm...@apache.org>
Committed: Sat May 9 13:45:01 2015 -0700
----------------------------------------------------------------------
.../drill/exec/store/dfs/FileSelection.java | 54 +++++++++-------
.../apache/drill/exec/store/dfs/TestGlob.java | 67 ++++++++++++++++++++
2 files changed, 97 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/76cf7296/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
index b9ae303..be9784e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -127,35 +127,41 @@ public class FileSelection {
return statuses;
}
- public static FileSelection create(DrillFileSystem fs, String parent, String path) throws IOException {
- if ( !(path.contains("*") || path.contains("?")) ) {
- Path p = new Path(parent, removeLeadingSlash(path));
- FileStatus status = fs.getFileStatus(p);
- return new FileSelection(Collections.singletonList(status), p.toUri().getPath());
- } else {
- Path p = new Path(parent,removeLeadingSlash(path));
- FileStatus[] status = fs.globStatus(p);
- if (status == null || status.length == 0) {
- return null;
- }
- String[] s = p.toUri().getPath().split("/");
- int i = 0;
-
- // get a selection root based on the portions of the selection path that don't contain a wildcard.
- for(; i < s.length; i++){
- if(s[i].contains("*") || s[i].contains("?")){
- break;
- }
+ public static String commonPath(FileStatus... paths){
+ String commonPath = "";
+ String[][] folders = new String[paths.length][];
+ for(int i = 0; i < paths.length; i++){
+ folders[i] = Path.getPathWithoutSchemeAndAuthority(paths[i].getPath()).toString().split("/");
+ }
+ for(int j = 0; j < folders[0].length; j++){
+ String thisFolder = folders[0][j];
+ boolean allMatched = true;
+ for(int i = 1; i < folders.length && allMatched; i++){
+ if(folders[i].length < j){
+ allMatched = false;
+ break;
+ }
+ allMatched &= folders[i][j].equals(thisFolder);
}
- String newPath;
- if(i > 0){
- newPath = StringUtils.join(ArrayUtils.subarray(s, 0, i), "/");
+ if(allMatched){
+ commonPath += thisFolder + "/";
}else{
- newPath = "/";
+ break;
}
+ }
+ return commonPath;
+ }
- return new FileSelection(Lists.newArrayList(status), newPath);
+ public static FileSelection create(DrillFileSystem fs, String parent, String path) throws IOException {
+ Path p = new Path(parent,removeLeadingSlash(path));
+ FileStatus[] status = fs.globStatus(p);
+ if (status == null || status.length == 0) {
+ return null;
+ }
+ if (status.length == 1) {
+ return new FileSelection(Collections.singletonList(status[0]), p.toUri().getPath());
}
+ return new FileSelection(Lists.newArrayList(status), commonPath(status));
}
private static String removeLeadingSlash(String path) {
http://git-wip-us.apache.org/repos/asf/drill/blob/76cf7296/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
new file mode 100644
index 0000000..8b4f7e7
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.dfs;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.TestTools;
+import org.junit.Test;
+
+public class TestGlob extends BaseTestQuery {
+
+ String MULTILEVEL = TestTools.getWorkingPath() + "/../java-exec/src/test/resources/multilevel";
+
+ @Test
+ public void testGlobSet() throws Exception {
+ testBuilder()
+ .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/{1994,1995}`", MULTILEVEL))
+ .unOrdered()
+ .baselineColumns("EXPR$0")
+ .baselineValues(80L)
+ .build().run();
+ }
+
+ @Test
+ public void testGlobWildcard() throws Exception {
+ testBuilder()
+ .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/1994/*`", MULTILEVEL))
+ .unOrdered()
+ .baselineColumns("EXPR$0")
+ .baselineValues(40L)
+ .build().run();
+ }
+
+ @Test
+ public void testGlobSingleCharacter() throws Exception {
+ testBuilder()
+ .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/199?/*`", MULTILEVEL))
+ .unOrdered()
+ .baselineColumns("EXPR$0")
+ .baselineValues(120L)
+ .build().run();
+ }
+
+ @Test
+ public void testGlobSingleCharacterRange() throws Exception {
+ testBuilder()
+ .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/199[4-5]/*`", MULTILEVEL))
+ .unOrdered()
+ .baselineColumns("EXPR$0")
+ .baselineValues(80L)
+ .build().run();
+ }
+}