You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pr...@apache.org on 2017/11/15 01:46:47 UTC
[01/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Repository: drill
Updated Branches:
refs/heads/master c56de2f13 -> acc5ed927
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index ae1f0b1..6b29258 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -54,7 +54,6 @@ import com.google.common.collect.Maps;
public class RepeatedMapVector extends AbstractMapVector
implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
public final static MajorType TYPE = MajorType.newBuilder().setMinorType(MinorType.MAP).setMode(DataMode.REPEATED).build();
@@ -425,9 +424,6 @@ public class RepeatedMapVector extends AbstractMapVector
@Override
public DrillBuf[] getBuffers(boolean clear) {
- //final int expectedBufferSize = getBufferSize();
- //final int actualBufferSize = super.getBufferSize();
- //Preconditions.checkArgument(expectedBufferSize == actualBufferSize + offsets.getBufferSize());
return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
----------------------------------------------------------------------
diff --git a/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java b/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
index 090b8f4..c628783 100644
--- a/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
+++ b/logical/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
@@ -27,7 +27,7 @@ import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.common.scanner.persistence.ScanResult;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.test.DrillTest;
import org.junit.Test;
@@ -46,6 +46,6 @@ public class CheckStorageConfig extends DrillTest {
public void checkPlanParsing() throws Exception{
DrillConfig config = DrillConfig.create();
ScanResult scan = ClassPathScanner.fromPrescan(config);
- LogicalPlan plan = LogicalPlan.parse(new LogicalPlanPersistence(config, scan), FileUtils.getResourceAsString("/storage_engine_plan.json"));
+ LogicalPlan plan = LogicalPlan.parse(new LogicalPlanPersistence(config, scan), DrillFileUtils.getResourceAsString("/storage_engine_plan.json"));
}
}
[06/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java
index d2f2590..94c8ebf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestResourceLeak.java
@@ -26,7 +26,7 @@ import java.util.Properties;
import javax.inject.Inject;
-import org.apache.drill.QueryTestUtil;
+import org.apache.drill.test.QueryTestUtil;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.ExecConstants;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java
index 5bb6815..8427d49 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/TestEmptyPopulation.java
@@ -17,10 +17,10 @@
*/
package org.apache.drill.exec.vector.complex;
-import static org.apache.drill.TestBuilder.listOf;
-import static org.apache.drill.TestBuilder.mapOf;
+import static org.apache.drill.test.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.RootAllocatorFactory;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
index 808d98c..fafabeb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertEquals;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.util.JsonStringArrayList;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java
index 656429e..db3907d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexToJson.java
@@ -22,12 +22,13 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestComplexToJson extends BaseTestQuery {
@@ -42,7 +43,7 @@ public class TestComplexToJson extends BaseTestQuery {
client = new DrillClient(config, serviceSet.getCoordinator());
client.setSupportComplexTypes(false);
client.connect();
- results = testSqlWithResults("select * from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/regions.csv`");
+ results = testSqlWithResults("select * from cp.`store/text/data/regions.csv`");
loader.load(results.get(0).getHeader().getDef(), results.get(0).getData());
RecordBatchDef def = results.get(0).getHeader().getDef();
// the entire row is returned as a single column
@@ -58,7 +59,7 @@ public class TestComplexToJson extends BaseTestQuery {
client = new DrillClient(config, serviceSet.getCoordinator());
client.setSupportComplexTypes(true);
client.connect();
- results = testSqlWithResults("select * from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/regions.csv`");
+ results = testSqlWithResults("select * from cp.`store/text/data/regions.csv`");
loader.load(results.get(0).getHeader().getDef(), results.get(0).getData());
def = results.get(0).getHeader().getDef();
// the entire row is returned as a single column
@@ -73,5 +74,4 @@ public class TestComplexToJson extends BaseTestQuery {
client = parent_client;
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
index 67b2e5a..5badbdb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
@@ -18,8 +18,8 @@
package org.apache.drill.exec.vector.complex.writer;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestBuilder;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -30,13 +30,10 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
-import static org.apache.drill.TestBuilder.listOf;
-
-public class TestComplexTypeReader extends BaseTestQuery{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class);
+import static org.apache.drill.test.TestBuilder.listOf;
+public class TestComplexTypeReader extends BaseTestQuery {
@BeforeClass
public static void init() throws Exception {
testNoResult("alter session set `exec.enable_union_type` = true");
@@ -243,13 +240,11 @@ public class TestComplexTypeReader extends BaseTestQuery{
@Test //DRILL-2872.
public void testRepeatedJson() throws Exception {
-
- final String query="select cast(convert_to(interests, 'JSON') as varchar(0)) as interests from cp.`complex_student.json`";
testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .jsonBaselineFile("DRILL-2872-result.json")
- .go();
+ .sqlQuery("select cast(convert_to(interests, 'JSON') as varchar(0)) as interests from cp.`complex_student.json`")
+ .unOrdered()
+ .jsonBaselineFile("DRILL-2872-result.json")
+ .go();
}
@Test // DRILL-4410
@@ -258,19 +253,18 @@ public class TestComplexTypeReader extends BaseTestQuery{
long numRecords = 100000;
- String tempDir = BaseTestQuery.getTempDir("ComplexTypeWriter");
- String file1 = tempDir + TestComplexTypeReader.class.getName() + "arrays1.json";
- String file2 = tempDir + TestComplexTypeReader.class.getName() + "arrays2.json";
- Path path1 = Paths.get(file1);
- Path path2 = Paths.get(file2);
+ final String file1 = "arrays1.json";
+ final String file2 = "arrays2.json";
+
+ final Path path1 = dirTestWatcher.getRootDir().toPath().resolve(file1);
+ final Path path2 = dirTestWatcher.getRootDir().toPath().resolve(file2);
- String arrayString = "[ \"abcdef\", \"ghijkl\", \"mnopqr\", \"stuvwx\", \"yz1234\", \"567890\" ] ";
- Files.deleteIfExists(path1);
- Files.deleteIfExists(path2);
Files.createFile(path1);
Files.createFile(path2);
- try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file1, true)))) {
+ final String arrayString = "[ \"abcdef\", \"ghijkl\", \"mnopqr\", \"stuvwx\", \"yz1234\", \"567890\" ] ";
+
+ try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(path1.toFile(), true)))) {
for (long i = 0; i < numRecords; i++) {
out.println("{ \"id\" : " + i + ", \"array\" : " + arrayString + "}");
}
@@ -278,7 +272,7 @@ public class TestComplexTypeReader extends BaseTestQuery{
throw e;
}
- try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file2, true)))) {
+ try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(path2.toFile(), true)))) {
for (long i = 0; i < numRecords; i++) {
out.println("{ \"id\" : " + i + ", \"array\" : " + arrayString + "}");
}
@@ -286,27 +280,24 @@ public class TestComplexTypeReader extends BaseTestQuery{
throw e;
}
- String queryString = "select * from dfs.`" + file1 + "` `arrays1` INNER JOIN dfs.`" + file2 + "` `arrays2` ON "
- + "(`arrays1`.id = `arrays2`.id)";
- TestBuilder testBuilder = testBuilder().sqlQuery(queryString).unOrdered();
- testBuilder.baselineColumns("id", "id0", "array", "array0");
+ TestBuilder testBuilder = testBuilder()
+ .sqlQuery("select * from dfs.`%s` `arrays1` INNER JOIN dfs.`%s` `arrays2` ON "
+ + "(`arrays1`.id = `arrays2`.id)", file1, file2)
+ .unOrdered()
+ .baselineColumns("id", "id0", "array", "array0");
+
for (long i = 0; i < numRecords; i++) {
testBuilder.baselineValues(i, i, listOf("abcdef", "ghijkl", "mnopqr", "stuvwx", "yz1234", "567890"),
listOf("abcdef", "ghijkl", "mnopqr", "stuvwx", "yz1234", "567890"));
}
testBuilder.go();
-
- Files.deleteIfExists(path1);
- Files.deleteIfExists(path2);
}
@Test
public void testNonExistentFieldConverting() throws Exception {
- String query = "select employee_id, convert_to(`complex_field`, 'JSON') as complex_field from cp.`employee.json` " +
- "where employee_id = 1";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select employee_id, convert_to(`complex_field`, 'JSON') as complex_field from cp.`employee.json` " +
+ "where employee_id = 1")
.unOrdered()
.baselineColumns("employee_id", "complex_field")
.baselineValues(1L, null)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java
index cb7bef2..3c0e1db 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeWriter.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.vector.complex.writer;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
public class TestComplexTypeWriter extends BaseTestQuery{
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
index 58793a2..b6562bb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
@@ -18,47 +18,46 @@
package org.apache.drill.exec.vector.complex.writer;
import mockit.integration.junit4.JMockit;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.util.TestTools;
+import static org.junit.Assert.assertEquals;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
-import org.apache.drill.test.OperatorFixture;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import static org.junit.Assert.assertEquals;
@RunWith(JMockit.class)
public class TestExtendedTypes extends BaseTestQuery {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExtendedTypes.class);
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("vector", "complex"));
+ }
@Test
public void checkReadWriteExtended() throws Exception {
mockUtcDateTimeZone();
- final String originalFile = "${WORKING_PATH}/src/test/resources/vector/complex/extended.json".replaceAll(
- Pattern.quote("${WORKING_PATH}"),
- Matcher.quoteReplacement(TestTools.getWorkingPath()));
-
+ final String originalFile = "vector/complex/extended.json";
final String newTable = "TestExtendedTypes/newjson";
+
try {
testNoResult(String.format("ALTER SESSION SET `%s` = 'json'", ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName()));
testNoResult(String.format("ALTER SESSION SET `%s` = true", ExecConstants.JSON_EXTENDED_TYPES.getOptionName()));
// create table
- test("create table dfs_test.tmp.`%s` as select * from dfs.`%s`", newTable, originalFile);
-
+ test("create table dfs.tmp.`%s` as select * from cp.`%s`", newTable, originalFile);
// check query of table.
- test("select * from dfs_test.tmp.`%s`", newTable);
+ test("select * from dfs.tmp.`%s`", newTable);
+
// check that original file and new file match.
- final byte[] originalData = Files.readAllBytes(Paths.get(originalFile));
- final byte[] newData = Files.readAllBytes(Paths.get(BaseTestQuery.getDfsTestTmpSchemaLocation() + '/' + newTable
- + "/0_0_0.json"));
+ final byte[] originalData = Files.readAllBytes(dirTestWatcher.getRootDir().toPath().resolve(originalFile));
+ final byte[] newData = Files.readAllBytes(dirTestWatcher.getDfsTestTmpDir().toPath().resolve(Paths.get(newTable, "0_0_0.json")));
assertEquals(new String(originalData), new String(newData));
} finally {
resetSessionOption(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName());
@@ -68,16 +67,13 @@ public class TestExtendedTypes extends BaseTestQuery {
@Test
public void testMongoExtendedTypes() throws Exception {
-
- final String originalFile = "${WORKING_PATH}/src/test/resources/vector/complex/mongo_extended.json".replaceAll(
- Pattern.quote("${WORKING_PATH}"),
- Matcher.quoteReplacement(TestTools.getWorkingPath()));
+ final String originalFile = "vector/complex/mongo_extended.json";
try {
testNoResult(String.format("ALTER SESSION SET `%s` = 'json'", ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName()));
testNoResult(String.format("ALTER SESSION SET `%s` = true", ExecConstants.JSON_EXTENDED_TYPES.getOptionName()));
- int actualRecordCount = testSql(String.format("select * from dfs.`%s`", originalFile));
+ int actualRecordCount = testSql(String.format("select * from cp.`%s`", originalFile));
assertEquals(
String.format(
"Received unexpected number of rows in output: expected=%d, received=%s",
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index 9064c5c..3e3580f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.vector.complex.writer;
-import static org.apache.drill.TestBuilder.listOf;
-import static org.apache.drill.TestBuilder.mapOf;
+import static org.apache.drill.test.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -31,12 +31,13 @@ import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
+import java.nio.file.Paths;
import java.util.List;
import java.util.zip.GZIPOutputStream;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -45,30 +46,28 @@ import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.store.easy.json.JSONRecordReader;
import org.apache.drill.exec.vector.IntVector;
import org.apache.drill.exec.vector.RepeatedBigIntVector;
+import org.junit.BeforeClass;
import org.junit.Ignore;
-import org.junit.Rule;
import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import org.junit.rules.TemporaryFolder;
-
public class TestJsonReader extends BaseTestQuery {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJsonReader.class);
-
private static final boolean VERBOSE_DEBUG = false;
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("store", "json"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("vector","complex", "writer"));
+ }
@Test
public void testEmptyList() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/json/emptyLists").toURI().toString();
- String query = String.format("select count(a[0]) as ct from dfs_test.`%s`", root, root);
+ final String root = "store/json/emptyLists";
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select count(a[0]) as ct from dfs.`%s`", root, root)
.ordered()
.baselineColumns("ct")
.baselineValues(6l)
@@ -78,7 +77,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void schemaChange() throws Exception {
- test("select b from dfs.`${WORKING_PATH}/src/test/resources/vector/complex/writer/schemaChange/`");
+ test("select b from dfs.`vector/complex/writer/schemaChange/`");
}
@Test
@@ -116,16 +115,16 @@ public class TestJsonReader extends BaseTestQuery {
public void testSplitAndTransferFailure() throws Exception {
final String testVal = "a string";
testBuilder()
- .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list.json`")
+ .sqlQuery("select flatten(config) as flat from cp.`store/json/null_list.json`")
.ordered()
.baselineColumns("flat")
.baselineValues(listOf())
.baselineValues(listOf(testVal))
.go();
- test("select flatten(config) as flat from cp.`/store/json/null_list_v2.json`");
+ test("select flatten(config) as flat from cp.`store/json/null_list_v2.json`");
testBuilder()
- .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list_v2.json`")
+ .sqlQuery("select flatten(config) as flat from cp.`store/json/null_list_v2.json`")
.ordered()
.baselineColumns("flat")
.baselineValues(mapOf("repeated_varchar", listOf()))
@@ -133,7 +132,7 @@ public class TestJsonReader extends BaseTestQuery {
.go();
testBuilder()
- .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list_v3.json`")
+ .sqlQuery("select flatten(config) as flat from cp.`store/json/null_list_v3.json`")
.ordered()
.baselineColumns("flat")
.baselineValues(mapOf("repeated_map", listOf(mapOf("repeated_varchar", listOf()))))
@@ -144,10 +143,10 @@ public class TestJsonReader extends BaseTestQuery {
@Test
@Ignore("DRILL-1824")
public void schemaChangeValidate() throws Exception {
- testBuilder() //
- .sqlQuery("select b from dfs.`${WORKING_PATH}/src/test/resources/vector/complex/writer/schemaChange/`") //
- .unOrdered() //
- .jsonBaselineFile("/vector/complex/writer/expected.json") //
+ testBuilder()
+ .sqlQuery("select b from dfs.`vector/complex/writer/schemaChange/`")
+ .unOrdered()
+ .jsonBaselineFile("/vector/complex/writer/expected.json")
.build()
.run();
}
@@ -157,7 +156,7 @@ public class TestJsonReader extends BaseTestQuery {
System.out.println("===================");
System.out.println("source data in json");
System.out.println("===================");
- System.out.println(Files.toString(FileUtils.getResourceAsFile(filename), Charsets.UTF_8));
+ System.out.println(Files.toString(DrillFileUtils.getResourceAsFile(filename), Charsets.UTF_8));
}
int i = 0;
@@ -181,14 +180,14 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testReadCompressed() throws Exception {
String filepath = "compressed_json.json";
- File f = folder.newFile(filepath);
+ File f = new File(dirTestWatcher.getRootDir(), filepath);
PrintWriter out = new PrintWriter(f);
out.println("{\"a\" :5}");
out.close();
gzipIt(f);
testBuilder()
- .sqlQuery("select * from dfs.`" + f.getPath() + ".gz" + "`")
+ .sqlQuery("select * from dfs.`%s.gz`", filepath)
.unOrdered()
.baselineColumns("a")
.baselineValues(5l)
@@ -196,7 +195,7 @@ public class TestJsonReader extends BaseTestQuery {
// test reading the uncompressed version as well
testBuilder()
- .sqlQuery("select * from dfs.`" + f.getPath() + "`")
+ .sqlQuery("select * from dfs.`%s`", filepath)
.unOrdered()
.baselineColumns("a")
.baselineValues(5l)
@@ -224,7 +223,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testDrill_1419() throws Exception {
- String[] queries = {"select t.trans_id, t.trans_info.prod_id[0],t.trans_info.prod_id[1] from cp.`/store/json/clicks.json` t limit 5"};
+ String[] queries = {"select t.trans_id, t.trans_info.prod_id[0],t.trans_info.prod_id[1] from cp.`store/json/clicks.json` t limit 5"};
long[] rowCounts = {5};
String filename = "/store/json/clicks.json";
runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
@@ -232,26 +231,26 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testRepeatedCount() throws Exception {
- test("select repeated_count(str_list) from cp.`/store/json/json_basic_repeated_varchar.json`");
- test("select repeated_count(INT_col) from cp.`/parquet/alltypes_repeated.json`");
- test("select repeated_count(FLOAT4_col) from cp.`/parquet/alltypes_repeated.json`");
- test("select repeated_count(VARCHAR_col) from cp.`/parquet/alltypes_repeated.json`");
- test("select repeated_count(BIT_col) from cp.`/parquet/alltypes_repeated.json`");
+ test("select repeated_count(str_list) from cp.`store/json/json_basic_repeated_varchar.json`");
+ test("select repeated_count(INT_col) from cp.`parquet/alltypes_repeated.json`");
+ test("select repeated_count(FLOAT4_col) from cp.`parquet/alltypes_repeated.json`");
+ test("select repeated_count(VARCHAR_col) from cp.`parquet/alltypes_repeated.json`");
+ test("select repeated_count(BIT_col) from cp.`parquet/alltypes_repeated.json`");
}
@Test
public void testRepeatedContains() throws Exception {
- test("select repeated_contains(str_list, 'asdf') from cp.`/store/json/json_basic_repeated_varchar.json`");
- test("select repeated_contains(INT_col, -2147483648) from cp.`/parquet/alltypes_repeated.json`");
- test("select repeated_contains(FLOAT4_col, -1000000000000.0) from cp.`/parquet/alltypes_repeated.json`");
- test("select repeated_contains(VARCHAR_col, 'qwerty' ) from cp.`/parquet/alltypes_repeated.json`");
- test("select repeated_contains(BIT_col, true) from cp.`/parquet/alltypes_repeated.json`");
- test("select repeated_contains(BIT_col, false) from cp.`/parquet/alltypes_repeated.json`");
+ test("select repeated_contains(str_list, 'asdf') from cp.`store/json/json_basic_repeated_varchar.json`");
+ test("select repeated_contains(INT_col, -2147483648) from cp.`parquet/alltypes_repeated.json`");
+ test("select repeated_contains(FLOAT4_col, -1000000000000.0) from cp.`parquet/alltypes_repeated.json`");
+ test("select repeated_contains(VARCHAR_col, 'qwerty' ) from cp.`parquet/alltypes_repeated.json`");
+ test("select repeated_contains(BIT_col, true) from cp.`parquet/alltypes_repeated.json`");
+ test("select repeated_contains(BIT_col, false) from cp.`parquet/alltypes_repeated.json`");
}
@Test
public void testSingleColumnRead_vector_fill_bug() throws Exception {
- String[] queries = {"select * from cp.`/store/json/single_column_long_file.json`"};
+ String[] queries = {"select * from cp.`store/json/single_column_long_file.json`"};
long[] rowCounts = {13512};
String filename = "/store/json/single_column_long_file.json";
runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
@@ -259,7 +258,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testNonExistentColumnReadAlone() throws Exception {
- String[] queries = {"select non_existent_column from cp.`/store/json/single_column_long_file.json`"};
+ String[] queries = {"select non_existent_column from cp.`store/json/single_column_long_file.json`"};
long[] rowCounts = {13512};
String filename = "/store/json/single_column_long_file.json";
runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
@@ -268,7 +267,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testAllTextMode() throws Exception {
test("alter system set `store.json.all_text_mode` = true");
- String[] queries = {"select * from cp.`/store/json/schema_change_int_to_string.json`"};
+ String[] queries = {"select * from cp.`store/json/schema_change_int_to_string.json`"};
long[] rowCounts = {3};
String filename = "/store/json/schema_change_int_to_string.json";
runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
@@ -277,7 +276,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void readComplexWithStar() throws Exception {
- List<QueryDataBatch> results = testSqlWithResults("select * from cp.`/store/json/test_complex_read_with_star.json`");
+ List<QueryDataBatch> results = testSqlWithResults("select * from cp.`store/json/test_complex_read_with_star.json`");
assertEquals(1, results.size());
RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
@@ -294,7 +293,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testNullWhereListExpected() throws Exception {
test("alter system set `store.json.all_text_mode` = true");
- String[] queries = {"select * from cp.`/store/json/null_where_list_expected.json`"};
+ String[] queries = {"select * from cp.`store/json/null_where_list_expected.json`"};
long[] rowCounts = {3};
String filename = "/store/json/null_where_list_expected.json";
runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
@@ -304,7 +303,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testNullWhereMapExpected() throws Exception {
test("alter system set `store.json.all_text_mode` = true");
- String[] queries = {"select * from cp.`/store/json/null_where_map_expected.json`"};
+ String[] queries = {"select * from cp.`store/json/null_where_map_expected.json`"};
long[] rowCounts = {3};
String filename = "/store/json/null_where_map_expected.json";
runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
@@ -324,7 +323,7 @@ public class TestJsonReader extends BaseTestQuery {
// ensure that the project is filtering out the correct data in the scan alone
@Test
public void testProjectPushdown() throws Exception {
- String[] queries = {Files.toString(FileUtils.getResourceAsFile("/store/json/project_pushdown_json_physical_plan.json"), Charsets.UTF_8)};
+ String[] queries = {Files.toString(DrillFileUtils.getResourceAsFile("/store/json/project_pushdown_json_physical_plan.json"), Charsets.UTF_8)};
long[] rowCounts = {3};
String filename = "/store/json/schema_change_int_to_string.json";
test("alter system set `store.json.all_text_mode` = false");
@@ -349,13 +348,8 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testJsonDirectoryWithEmptyFile() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/json/jsonDirectoryWithEmpyFile").toURI().toString();
-
- String queryRightEmpty = String.format(
- "select * from dfs_test.`%s`", root);
-
testBuilder()
- .sqlQuery(queryRightEmpty)
+ .sqlQuery("select * from dfs.`store/json/jsonDirectoryWithEmpyFile`")
.unOrdered()
.baselineColumns("a")
.baselineValues(1l)
@@ -400,9 +394,8 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testSelectStarWithUnionType() throws Exception {
try {
- String query = "select * from cp.`jsoninput/union/a.json`";
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from cp.`jsoninput/union/a.json`")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.baselineColumns("field1", "field2")
@@ -452,12 +445,11 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testSelectFromListWithCase() throws Exception {
- String query = "select a, typeOf(a) `type` from " +
- "(select case when is_list(field2) then field2[4][1].inner7 end a " +
- "from cp.`jsoninput/union/a.json`) where a is not null";
try {
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select a, typeOf(a) `type` from " +
+ "(select case when is_list(field2) then field2[4][1].inner7 end a " +
+ "from cp.`jsoninput/union/a.json`) where a is not null")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.baselineColumns("a", "type")
@@ -470,12 +462,11 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testTypeCase() throws Exception {
- String query = "select case when is_bigint(field1) " +
- "then field1 when is_list(field1) then field1[0] " +
- "when is_map(field1) then t.field1.inner1 end f1 from cp.`jsoninput/union/a.json` t";
try {
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select case when is_bigint(field1) " +
+ "then field1 when is_list(field1) then field1[0] " +
+ "when is_map(field1) then t.field1.inner1 end f1 from cp.`jsoninput/union/a.json` t")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.baselineColumns("f1")
@@ -491,13 +482,12 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testSumWithTypeCase() throws Exception {
- String query = "select sum(cast(f1 as bigint)) sum_f1 from " +
- "(select case when is_bigint(field1) then field1 " +
- "when is_list(field1) then field1[0] when is_map(field1) then t.field1.inner1 end f1 " +
- "from cp.`jsoninput/union/a.json` t)";
try {
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select sum(cast(f1 as bigint)) sum_f1 from " +
+ "(select case when is_bigint(field1) then field1 " +
+ "when is_list(field1) then field1[0] when is_map(field1) then t.field1.inner1 end f1 " +
+ "from cp.`jsoninput/union/a.json` t)")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.baselineColumns("sum_f1")
@@ -510,10 +500,9 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testUnionExpressionMaterialization() throws Exception {
- String query = "select a + b c from cp.`jsoninput/union/b.json`";
try {
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select a + b c from cp.`jsoninput/union/b.json`")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.baselineColumns("c")
@@ -528,9 +517,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testSumMultipleBatches() throws Exception {
- String dfs_temp = getDfsTestTmpSchemaLocation();
- File table_dir = new File(dfs_temp, "multi_batch");
- table_dir.mkdir();
+ File table_dir = dirTestWatcher.makeTestTmpSubDir(Paths.get("multi_batch"));
BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")));
for (int i = 0; i < 10000; i++) {
os.write("{ type : \"map\", data : { a : 1 } }\n".getBytes());
@@ -538,10 +525,10 @@ public class TestJsonReader extends BaseTestQuery {
}
os.flush();
os.close();
- String query = "select sum(cast(case when `type` = 'map' then t.data.a else data end as bigint)) `sum` from dfs_test.tmp.multi_batch t";
+
try {
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select sum(cast(case when `type` = 'map' then t.data.a else data end as bigint)) `sum` from dfs.tmp.multi_batch t")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.baselineColumns("sum")
@@ -554,9 +541,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testSumFilesWithDifferentSchema() throws Exception {
- String dfs_temp = getDfsTestTmpSchemaLocation();
- File table_dir = new File(dfs_temp, "multi_file");
- table_dir.mkdir();
+ File table_dir = dirTestWatcher.makeTestTmpSubDir(Paths.get("multi_file"));
BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")));
for (int i = 0; i < 10000; i++) {
os.write("{ type : \"map\", data : { a : 1 } }\n".getBytes());
@@ -569,10 +554,10 @@ public class TestJsonReader extends BaseTestQuery {
}
os.flush();
os.close();
- String query = "select sum(cast(case when `type` = 'map' then t.data.a else data end as bigint)) `sum` from dfs_test.tmp.multi_file t";
+
try {
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select sum(cast(case when `type` = 'map' then t.data.a else data end as bigint)) `sum` from dfs.tmp.multi_file t")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.baselineColumns("sum")
@@ -585,8 +570,7 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void drill_4032() throws Exception {
- String dfs_temp = getDfsTestTmpSchemaLocation();
- File table_dir = new File(dfs_temp, "drill_4032");
+ File table_dir = dirTestWatcher.makeTestTmpSubDir(Paths.get("drill_4032"));
table_dir.mkdir();
BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")));
os.write("{\"col1\": \"val1\",\"col2\": null}".getBytes());
@@ -598,14 +582,13 @@ public class TestJsonReader extends BaseTestQuery {
os.write("{\"col1\": \"val1\",\"col2\": null}".getBytes());
os.flush();
os.close();
- testNoResult("select t.col2.col3 from dfs_test.tmp.drill_4032 t");
+ testNoResult("select t.col2.col3 from dfs.tmp.drill_4032 t");
}
@Test
public void drill_4479() throws Exception {
try {
- String dfs_temp = getDfsTestTmpSchemaLocation();
- File table_dir = new File(dfs_temp, "drill_4479");
+ File table_dir = dirTestWatcher.makeTestTmpSubDir(Paths.get("drill_4479"));
table_dir.mkdir();
BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "mostlynulls.json")));
// Create an entire batch of null values for 3 columns
@@ -617,12 +600,8 @@ public class TestJsonReader extends BaseTestQuery {
os.flush();
os.close();
- String query1 = "select c, count(*) as cnt from dfs_test.tmp.drill_4479 t group by c";
- String query2 = "select a, b, c, count(*) as cnt from dfs_test.tmp.drill_4479 t group by a, b, c";
- String query3 = "select max(a) as x, max(b) as y, max(c) as z from dfs_test.tmp.drill_4479 t";
-
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select c, count(*) as cnt from dfs.tmp.drill_4479 t group by c")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `store.json.all_text_mode` = true")
.baselineColumns("c", "cnt")
@@ -631,7 +610,7 @@ public class TestJsonReader extends BaseTestQuery {
.go();
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select a, b, c, count(*) as cnt from dfs.tmp.drill_4479 t group by a, b, c")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `store.json.all_text_mode` = true")
.baselineColumns("a", "b", "c", "cnt")
@@ -640,7 +619,7 @@ public class TestJsonReader extends BaseTestQuery {
.go();
testBuilder()
- .sqlQuery(query3)
+ .sqlQuery("select max(a) as x, max(b) as y, max(c) as z from dfs.tmp.drill_4479 t")
.ordered()
.optionSettingQueriesForTestQuery("alter session set `store.json.all_text_mode` = true")
.baselineColumns("x", "y", "z")
@@ -654,18 +633,12 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testFlattenEmptyArrayWithAllTextMode() throws Exception {
- File path = new File(BaseTestQuery.getTempDir("json/input"));
- path.mkdirs();
- path.deleteOnExit();
- String pathString = path.toPath().toString();
-
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(path, "empty_array_all_text_mode.json")))) {
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "empty_array_all_text_mode.json")))) {
writer.write("{ \"a\": { \"b\": { \"c\": [] }, \"c\": [] } }");
}
try {
- String query = String.format("select flatten(t.a.b.c) as c from dfs_test.`%s/empty_array_all_text_mode.json` t",
- pathString);
+ String query = "select flatten(t.a.b.c) as c from dfs.`empty_array_all_text_mode.json` t";
testBuilder()
.sqlQuery(query)
@@ -688,18 +661,12 @@ public class TestJsonReader extends BaseTestQuery {
@Test
public void testFlattenEmptyArrayWithUnionType() throws Exception {
- File path = new File(BaseTestQuery.getTempDir("json/input"));
- path.mkdirs();
- path.deleteOnExit();
- String pathString = path.toPath().toString();
-
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(path, "empty_array.json")))) {
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "empty_array.json")))) {
writer.write("{ \"a\": { \"b\": { \"c\": [] }, \"c\": [] } }");
}
try {
- String query = String.format("select flatten(t.a.b.c) as c from dfs_test.`%s/empty_array.json` t",
- pathString);
+ String query = "select flatten(t.a.b.c) as c from dfs.`empty_array.json` t";
testBuilder()
.sqlQuery(query)
@@ -724,49 +691,33 @@ public class TestJsonReader extends BaseTestQuery {
@Test // DRILL-5521
public void testKvgenWithUnionAll() throws Exception {
- File directory = new File(BaseTestQuery.getTempDir("json/input"));
- try {
- directory.mkdirs();
- String fileName = "map.json";
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
- writer.write("{\"rk\": \"a\", \"m\": {\"a\":\"1\"}}");
- }
-
- String query = String.format("select kvgen(m) as res from (select m from dfs_test.`%s/%s` union all " +
- "select convert_from('{\"a\" : null}' ,'json') as m from (values(1)))", directory.toPath().toString(), fileName);
- assertEquals("Row count should match", 2, testSql(query));
-
- } finally {
- org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ String fileName = "map.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), fileName)))) {
+ writer.write("{\"rk\": \"a\", \"m\": {\"a\":\"1\"}}");
}
+
+ String query = String.format("select kvgen(m) as res from (select m from dfs.`%s` union all " +
+ "select convert_from('{\"a\" : null}' ,'json') as m from (values(1)))", fileName);
+ assertEquals("Row count should match", 2, testSql(query));
}
@Test // DRILL-4264
public void testFieldWithDots() throws Exception {
- File directory = new File(BaseTestQuery.getTempDir("json/input"));
- try {
- directory.mkdirs();
- String fileName = "table.json";
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
- writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
- }
-
- String query = String.format("select t.m.`a.b` as a,\n" +
- "t.m.a.b as b,\n" +
- "t.m['a.b'] as c,\n" +
- "t.rk.q as d,\n" +
- "t.`rk.q` as e\n" +
- "from dfs_test.`%s/%s` t",
- directory.toPath().toString(), fileName);
- testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .baselineColumns("a", "b", "c", "d", "e")
- .baselineValues("1", "2", "1", null, "a")
- .go();
-
- } finally {
- org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ String fileName = "table.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), fileName)))) {
+ writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
}
+
+ testBuilder()
+ .sqlQuery("select t.m.`a.b` as a,\n" +
+ "t.m.a.b as b,\n" +
+ "t.m['a.b'] as c,\n" +
+ "t.rk.q as d,\n" +
+ "t.`rk.q` as e\n" +
+ "from dfs.`%s` t", fileName)
+ .unOrdered()
+ .baselineColumns("a", "b", "c", "d", "e")
+ .baselineValues("1", "2", "1", null, "a")
+ .go();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java
index 389bb80..0f18fba 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/TestSpoolingBuffer.java
@@ -21,9 +21,9 @@ import static org.junit.Assert.assertEquals;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.server.Drillbit;
@@ -47,7 +47,7 @@ public class TestSpoolingBuffer extends BaseTestQuery {
bit1.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/work/batch/multiple_exchange.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/work/batch/multiple_exchange.json"),
Charsets.UTF_8));
int count = 0;
for(QueryDataBatch b : results) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
index cdb703f..d2181b4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/fragment/TestFragmentExecutorCancel.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.work.fragment;
import static org.junit.Assert.fail;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.CoordinationProtos;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
index 4eabfe3..c30cb09 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
@@ -27,7 +27,7 @@ import static org.junit.Assert.fail;
import java.util.Arrays;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OptionsTest;
import org.apache.drill.exec.proto.UserProtos.CatalogMetadata;
import org.apache.drill.exec.proto.UserProtos.ColumnMetadata;
@@ -103,16 +103,13 @@ public class TestMetadataProvider extends BaseTestQuery {
assertEquals(RequestStatus.OK, resp.getStatus());
List<SchemaMetadata> schemas = resp.getSchemasList();
- assertEquals(9, schemas.size());
+ assertEquals(6, schemas.size());
verifySchema("INFORMATION_SCHEMA", schemas);
verifySchema("cp.default", schemas);
verifySchema("dfs.default", schemas);
verifySchema("dfs.root", schemas);
verifySchema("dfs.tmp", schemas);
- verifySchema("dfs_test.default", schemas);
- verifySchema("dfs_test.home", schemas);
- verifySchema("dfs_test.tmp", schemas);
verifySchema("sys", schemas);
}
@@ -131,21 +128,17 @@ public class TestMetadataProvider extends BaseTestQuery {
@Test
public void schemasWithCatalogNameFilterAndSchemaNameFilter() throws Exception {
-
- // test("SELECT * FROM INFORMATION_SCHEMA.SCHEMATA " +
- // "WHERE CATALOG_NAME LIKE '%RI%' AND SCHEMA_NAME LIKE '%y%'"); // SQL equivalent
-
GetSchemasResp resp = client.getSchemas(
LikeFilter.newBuilder().setPattern("%RI%").build(),
- LikeFilter.newBuilder().setPattern("%dfs_test%").build()).get();
+ LikeFilter.newBuilder().setPattern("%dfs%").build()).get();
assertEquals(RequestStatus.OK, resp.getStatus());
List<SchemaMetadata> schemas = resp.getSchemasList();
assertEquals(3, schemas.size());
- verifySchema("dfs_test.default", schemas);
- verifySchema("dfs_test.home", schemas);
- verifySchema("dfs_test.tmp", schemas);
+ verifySchema("dfs.default", schemas);
+ verifySchema("dfs.root", schemas);
+ verifySchema("dfs.tmp", schemas);
}
@Test
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestServerMetaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestServerMetaProvider.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestServerMetaProvider.java
index c1fd38b..7099a37 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestServerMetaProvider.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestServerMetaProvider.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import org.apache.calcite.avatica.util.Quoting;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.proto.UserProtos.GetServerMetaResp;
import org.apache.drill.exec.proto.UserProtos.RequestStatus;
import org.apache.drill.exec.proto.UserProtos.ServerMeta;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/work/prepare/PreparedStatementTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/prepare/PreparedStatementTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/prepare/PreparedStatementTestBase.java
index 814414c..6802bc9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/prepare/PreparedStatementTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/prepare/PreparedStatementTestBase.java
@@ -16,7 +16,7 @@
*/
package org.apache.drill.exec.work.prepare;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.proto.UserProtos;
import org.apache.drill.exec.store.ischema.InfoSchemaConstants;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java b/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
new file mode 100644
index 0000000..fe7e6a6
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.test;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.FileUtils;
+import org.junit.runner.Description;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * <p>
+ * This is a {@link DirTestWatcher} which creates all the temporary directories required by a Drillbit and the various <b>dfs.*</b> storage workspaces. It also
+ * provides convenience methods that do the following:
+ * </p>
+ *
+ * <ol>
+ * <li>Copy project files to temp directories. This is useful for copying the sample data into a temp directory.</li>
+ * <li>Copy resource files to temp.</li>
+ * <li>Updating parquet metadata files.</li>
+ * </ol>
+ *
+ * <p>
+ * The {@link BaseDirTestWatcher} is used in {@link BaseTestQuery} and an example of how it is used in conjunction with the {@link ClusterFixture} can be found in
+ * {@link ExampleTest}.
+ * </p>
+ */
+public class BaseDirTestWatcher extends DirTestWatcher {
+ /**
+ * An enum used to represent the directories mapped to the <b>dfs.root</b> and <b>dfs.tmp</b> workspaces repectively.
+ */
+ public enum DirType {
+ ROOT, // Corresponds to the directory that should be mapped to dfs.root
+ TEST_TMP // Corresponds to the directory that should be mapped to dfs.tmp
+ }
+
+ private File tmpDir;
+ private File storeDir;
+ private File dfsTestTmpParentDir;
+ private File dfsTestTmpDir;
+ private File rootDir;
+
+ /**
+ * Creates a {@link BaseDirTestWatcher} which does not delete it's temp directories at the end of tests.
+ */
+ public BaseDirTestWatcher() {
+ super();
+ }
+
+ /**
+ * Creates a {@link BaseDirTestWatcher}.
+ * @param deleteDirAtEnd If true, temp directories are deleted at the end of tests. If fals, temp directories are not deleted at the end of tests.
+ */
+ public BaseDirTestWatcher(boolean deleteDirAtEnd) {
+ super(deleteDirAtEnd);
+ }
+
+ @Override
+ protected void starting(Description description) {
+ super.starting(description);
+
+ rootDir = makeSubDir(Paths.get("root"));
+ tmpDir = makeSubDir(Paths.get("tmp"));
+ storeDir = makeSubDir(Paths.get("store"));
+ dfsTestTmpParentDir = makeSubDir(Paths.get("dfsTestTmp"));
+
+ newDfsTestTmpDir();
+ }
+
+ /**
+ * Clear contents of cluster directories
+ */
+ public void clear() {
+ try {
+ FileUtils.cleanDirectory(rootDir);
+ FileUtils.cleanDirectory(tmpDir);
+ FileUtils.cleanDirectory(storeDir);
+ FileUtils.cleanDirectory(dfsTestTmpDir);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Gets the temp directory that should be used as a Drillbit's tmp directory.
+ * @return The temp directory that should be used as a Drillbit's tmp directory.
+ */
+ public File getTmpDir() {
+ return tmpDir;
+ }
+
+ /**
+ * Gets the temp directory that should be used by the {@link org.apache.drill.exec.store.sys.store.LocalPersistentStore}.
+ * @return The temp directory that should be used by the {@link org.apache.drill.exec.store.sys.store.LocalPersistentStore}.
+ */
+ public File getStoreDir() {
+ return storeDir;
+ }
+
+ /**
+ * Gets the temp directory that should be used by the <b>dfs.tmp</b> workspace.
+ * @return The temp directory that should be used by the <b>dfs.tmp</b> workspace.
+ */
+ public File getDfsTestTmpDir() {
+ return dfsTestTmpDir;
+ }
+
+ /**
+ * Gets the temp directory that should be used to hold the contents of the <b>dfs.root</b> workspace.
+ * @return The temp directory that should be used to hold the contents of the <b>dfs.root</b> workspace.
+ */
+ public File getRootDir() {
+ return rootDir;
+ }
+
+ /**
+ * This methods creates a new directory which can be mapped to <b>dfs.tmp</b>.
+ */
+ public void newDfsTestTmpDir() {
+ dfsTestTmpDir = DirTestWatcher.createTempDir(dfsTestTmpParentDir);
+ }
+
+ /**
+ * A helper method which returns the correct directory corresponding to the given {@link DirType}.
+ * @param type The directory to return.
+ * @return The directory corresponding to the given {@link DirType}.
+ */
+ private File getDir(DirType type) {
+ switch (type) {
+ case ROOT:
+ return rootDir;
+ case TEST_TMP:
+ return dfsTestTmpDir;
+ default:
+ throw new IllegalArgumentException(String.format("Unsupported type %s", type));
+ }
+ }
+
+ /**
+ * Creates a directory in the temp root directory (corresponding to <b>dfs.root</b>) at the given relative path.
+ * @param relPath The relative path in the temp root directory at which to create a directory.
+ * @return The {@link java.io.File} corresponding to the sub directory that was created.
+ */
+ public File makeRootSubDir(Path relPath) {
+ return makeSubDir(relPath, DirType.ROOT);
+ }
+
+ /**
+ * Creates a directory in the temp tmp directory (corresponding to <b>dfs.tmp</b>) at the given relative path.
+ * @param relPath The relative path in the temp tmp directory at which to create a directory.
+ * @return The {@link java.io.File} corresponding to the sub directory that was created.
+ */
+ public File makeTestTmpSubDir(Path relPath) {
+ return makeSubDir(relPath, DirType.TEST_TMP);
+ }
+
+ private File makeSubDir(Path relPath, DirType type) {
+ File subDir = getDir(type)
+ .toPath()
+ .resolve(relPath)
+ .toFile();
+ subDir.mkdirs();
+ return subDir;
+ }
+
+ /**
+ * This copies a file or directory from <b>src/test/resources</b> into the temp root directory (corresponding to <b>dfs.root</b>). The relative path of the file or
+ * directory in <b>src/test/resources</b> is preserved in the temp root directory.
+ * @param relPath The relative path of the file or directory in <b>src/test/resources</b> to copy into the root temp folder.
+ * @return The {@link java.io.File} corresponding to the copied file or directory in the temp root directory.
+ */
+ public File copyResourceToRoot(Path relPath) {
+ return copyTo(relPath, relPath, TestTools.FileSource.RESOURCE, DirType.ROOT);
+ }
+
+ /**
+ * This copies a filed or directory from the maven project into the temp root directory (corresponding to <b>dfs.root</b>). The relative path of the file or directory
+ * in the maven module is preserved in the temp root directory.
+ * @param relPath The relative path of the file or directory in the maven module to copy into the root temp folder.
+ * @return The {@link java.io.File} corresponding to the copied file or directory in the temp root directory.
+ */
+ public File copyFileToRoot(Path relPath) {
+ return copyTo(relPath, relPath, TestTools.FileSource.PROJECT, DirType.ROOT);
+ }
+
+ /**
+ * This copies a file or directory from <b>src/test/resources</b> into the temp root directory (corresponding to <b>dfs.root</b>). The file or directory is copied
+ * to the provided relative destPath in the temp root directory.
+ * @param relPath The source relative path of a file or directory from <b>src/test/resources</b> that will be copied.
+ * @param destPath The destination relative path of the file or directory in the temp root directory.
+ * @return The {@link java.io.File} corresponding to the final copied file or directory in the temp root directory.
+ */
+ public File copyResourceToRoot(Path relPath, Path destPath) {
+ return copyTo(relPath, destPath, TestTools.FileSource.RESOURCE, DirType.ROOT);
+ }
+
+ /**
+ * This copies a file or directory from <b>src/test/resources</b> into the temp root directory (corresponding to <b>dfs.root</b>). The file or directory is copied
+ * to the provided relative destPath in the temp root directory.
+ * @param relPath The source relative path of a file or directory from <b>src/test/resources</b> that will be copied.
+ * @param destPath The destination relative path of the file or directory in the temp root directory.
+ * @return The {@link java.io.File} corresponding to the final copied file or directory in the temp root directory.
+ */
+ public File copyResourceToTestTmp(Path relPath, Path destPath) {
+ return copyTo(relPath, destPath, TestTools.FileSource.RESOURCE, DirType.TEST_TMP);
+ }
+
+ private File copyTo(Path relPath, Path destPath, TestTools.FileSource fileSource, DirType dirType) {
+ File file = TestTools.getFile(relPath, fileSource);
+
+ if (file.isDirectory()) {
+ File subDir = makeSubDir(destPath, dirType);
+ TestTools.copyDirToDest(relPath, subDir, fileSource);
+ return subDir;
+ } else {
+ File baseDir = getDir(dirType);
+
+ baseDir.toPath()
+ .resolve(destPath)
+ .getParent()
+ .toFile()
+ .mkdirs();
+
+ File destFile = baseDir.toPath()
+ .resolve(destPath)
+ .toFile();
+
+ try {
+ destFile.createNewFile();
+ FileUtils.copyFile(file, destFile);
+ } catch (IOException e) {
+ throw new RuntimeException("This should not happen", e);
+ }
+
+ return destFile;
+ }
+ }
+
+ /**
+ * This is a convenience method that replaces placeholders in test parquet metadata files.
+ * @param metaDataFile The parquet metadata file to do string replacement on.
+ * @param replacePath The path to replace <b>REPLACED_IN_TEST</b> with in the parquet metadata file.
+ * @param customStringReplacement If this is provided a <b>CUSTOM_STRING_REPLACEMENT</b> is replaced in the parquet metadata file with this string.
+ */
+ public void replaceMetaDataContents(File metaDataFile, File replacePath, String customStringReplacement) {
+ try {
+ String metadataFileContents = FileUtils.readFileToString(metaDataFile, Charsets.UTF_8);
+
+ if (customStringReplacement != null) {
+ metadataFileContents = metadataFileContents.replace("CUSTOM_STRING_REPLACEMENT", customStringReplacement);
+ }
+
+ metadataFileContents = metadataFileContents.replace("REPLACED_IN_TEST", replacePath.getCanonicalPath());
+ FileUtils.write(metaDataFile, metadataFileContents, Charsets.UTF_8);
+ } catch (IOException e) {
+ throw new RuntimeException("This should not happen", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
new file mode 100644
index 0000000..802ce1b
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
@@ -0,0 +1,613 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test;
+
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DEFAULT_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.ROOT_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA;
+import static org.hamcrest.core.StringContains.containsString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.drill.test.DrillTestWrapper.TestServices;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.config.DrillProperties;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.scanner.ClassPathScanner;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ExecTest;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.RootAllocatorFactory;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
+import org.apache.drill.exec.proto.UserBitShared.QueryType;
+import org.apache.drill.exec.proto.UserProtos.PreparedStatementHandle;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.ConnectionThrottle;
+import org.apache.drill.exec.rpc.user.AwaitableUserResultsListener;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.util.StoragePluginTestUtils;
+import org.apache.drill.exec.util.VectorUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.io.Resources;
+
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.test.ClusterFixture;
+
+public class BaseTestQuery extends ExecTest {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class);
+
+ private static final int MAX_WIDTH_PER_NODE = 2;
+
+ @SuppressWarnings("serial")
+ private static final Properties TEST_CONFIGURATIONS = new Properties() {
+ {
+ put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false");
+ put(ExecConstants.HTTP_ENABLE, "false");
+ // Increasing retry attempts for testing
+ put(ExecConstants.UDF_RETRY_ATTEMPTS, "10");
+ put(ExecConstants.SSL_USE_HADOOP_CONF, "false");
+ }
+ };
+
+ protected static DrillClient client;
+ protected static Drillbit[] bits;
+ protected static RemoteServiceSet serviceSet;
+ protected static DrillConfig config;
+ protected static BufferAllocator allocator;
+
+ /**
+ * Number of Drillbits in test cluster. Default is 1.
+ *
+ * Tests can update the cluster size through {@link #updateTestCluster(int, DrillConfig)}
+ */
+ private static int drillbitCount = 1;
+
+ private int[] columnWidths = new int[] { 8 };
+
+ private static ScanResult classpathScan;
+
+ @ClassRule
+ public static final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
+ @BeforeClass
+ public static void setupDefaultTestCluster() throws Exception {
+ config = DrillConfig.create(cloneDefaultTestConfigProperties());
+ classpathScan = ClassPathScanner.fromPrescan(config);
+ openClient();
+ // turns on the verbose errors in tests
+ // sever side stacktraces are added to the message before sending back to the client
+ test("ALTER SESSION SET `exec.errors.verbose` = true");
+ }
+
+ protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig) {
+ updateTestCluster(newDrillbitCount, newConfig, cloneDefaultTestConfigProperties());
+ }
+
+ protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig, Properties properties) {
+ Preconditions.checkArgument(newDrillbitCount > 0, "Number of Drillbits must be at least one");
+ if (drillbitCount != newDrillbitCount || config != null) {
+ // TODO: Currently we have to shutdown the existing Drillbit cluster before starting a new one with the given
+ // Drillbit count. Revisit later to avoid stopping the cluster.
+ try {
+ closeClient();
+ drillbitCount = newDrillbitCount;
+ if (newConfig != null) {
+ // For next test class, updated DrillConfig will be replaced by default DrillConfig in BaseTestQuery as part
+ // of the @BeforeClass method of test class.
+ config = newConfig;
+ }
+ openClient(properties);
+ } catch(Exception e) {
+ throw new RuntimeException("Failure while updating the test Drillbit cluster.", e);
+ }
+ }
+ }
+
+ /**
+ * Useful for tests that require a DrillbitContext to get/add storage plugins, options etc.
+ *
+ * @return DrillbitContext of first Drillbit in the cluster.
+ */
+ protected static DrillbitContext getDrillbitContext() {
+ Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup.");
+ return bits[0].getContext();
+ }
+
+ protected static Properties cloneDefaultTestConfigProperties() {
+ final Properties props = new Properties();
+ for(String propName : TEST_CONFIGURATIONS.stringPropertyNames()) {
+ props.put(propName, TEST_CONFIGURATIONS.getProperty(propName));
+ }
+
+ props.setProperty(ExecConstants.DRILL_TMP_DIR, dirTestWatcher.getTmpDir().getAbsolutePath());
+ props.setProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, dirTestWatcher.getStoreDir().getAbsolutePath());
+
+ return props;
+ }
+
+ private static void openClient() throws Exception {
+ openClient(null);
+ }
+
+ private static void openClient(Properties properties) throws Exception {
+ if (properties == null) {
+ properties = new Properties();
+ }
+
+ allocator = RootAllocatorFactory.newRoot(config);
+ serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ dirTestWatcher.newDfsTestTmpDir();
+
+ bits = new Drillbit[drillbitCount];
+ for(int i = 0; i < drillbitCount; i++) {
+ bits[i] = new Drillbit(config, serviceSet, classpathScan);
+ bits[i].run();
+
+ @SuppressWarnings("resource")
+ final StoragePluginRegistry pluginRegistry = bits[i].getContext().getStorage();
+ StoragePluginTestUtils.configureFormatPlugins(pluginRegistry);
+
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry,
+ dirTestWatcher.getDfsTestTmpDir(), TMP_SCHEMA);
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry,
+ dirTestWatcher.getRootDir(), ROOT_SCHEMA);
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry,
+ dirTestWatcher.getRootDir(), DEFAULT_SCHEMA);
+ }
+
+ if (!properties.containsKey(DrillProperties.DRILLBIT_CONNECTION)) {
+ properties.setProperty(DrillProperties.DRILLBIT_CONNECTION,
+ String.format("localhost:%s", bits[0].getUserPort()));
+ }
+
+ DrillConfig clientConfig = DrillConfig.forClient();
+ client = QueryTestUtil.createClient(clientConfig, serviceSet, MAX_WIDTH_PER_NODE, properties);
+ }
+
+ /**
+ * Close the current <i>client</i> and open a new client using the given <i>properties</i>. All tests executed
+ * after this method call use the new <i>client</i>.
+ *
+ * @param properties
+ */
+ public static void updateClient(Properties properties) throws Exception {
+ Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup.");
+ if (client != null) {
+ client.close();
+ client = null;
+ }
+
+ DrillConfig clientConfig = DrillConfig.forClient();
+ client = QueryTestUtil.createClient(clientConfig, serviceSet, MAX_WIDTH_PER_NODE, properties);
+ }
+
+ /*
+ * Close the current <i>client</i> and open a new client for the given user. All tests executed
+ * after this method call use the new <i>client</i>.
+ * @param user
+ */
+ public static void updateClient(String user) throws Exception {
+ updateClient(user, null);
+ }
+
+ /*
+ * Close the current <i>client</i> and open a new client for the given user and password credentials. Tests
+ * executed after this method call use the new <i>client</i>.
+ * @param user
+ */
+ public static void updateClient(final String user, final String password) throws Exception {
+ final Properties props = new Properties();
+ props.setProperty(DrillProperties.USER, user);
+ if (password != null) {
+ props.setProperty(DrillProperties.PASSWORD, password);
+ }
+ updateClient(props);
+ }
+
+ protected static BufferAllocator getAllocator() {
+ return allocator;
+ }
+
+ public static int getUserPort() {
+ return bits[0].getUserPort();
+ }
+
+ public static TestBuilder newTest() {
+ return testBuilder();
+ }
+
+
+ public static class ClassicTestServices implements TestServices {
+ @Override
+ public BufferAllocator allocator() {
+ return allocator;
+ }
+
+ @Override
+ public void test(String query) throws Exception {
+ BaseTestQuery.test(query);
+ }
+
+ @Override
+ public List<QueryDataBatch> testRunAndReturn(final QueryType type, final Object query) throws Exception {
+ return BaseTestQuery.testRunAndReturn(type, query);
+ }
+ }
+
+ public static TestBuilder testBuilder() {
+ return new TestBuilder(new ClassicTestServices());
+ }
+
+ @AfterClass
+ public static void closeClient() throws Exception {
+ if (client != null) {
+ client.close();
+ }
+
+ if (bits != null) {
+ for(final Drillbit bit : bits) {
+ if (bit != null) {
+ bit.close();
+ }
+ }
+ }
+
+ if(serviceSet != null) {
+ serviceSet.close();
+ }
+ if (allocator != null) {
+ allocator.close();
+ }
+ }
+
+ @AfterClass
+ public static void resetDrillbitCount() {
+ // some test classes assume this value to be 1 and will fail if run along other tests that increase it
+ drillbitCount = 1;
+ }
+
+ protected static void runSQL(String sql) throws Exception {
+ final AwaitableUserResultsListener listener = new AwaitableUserResultsListener(new SilentListener());
+ testWithListener(QueryType.SQL, sql, listener);
+ listener.await();
+ }
+
+ protected static List<QueryDataBatch> testSqlWithResults(String sql) throws Exception{
+ return testRunAndReturn(QueryType.SQL, sql);
+ }
+
+ protected static List<QueryDataBatch> testLogicalWithResults(String logical) throws Exception{
+ return testRunAndReturn(QueryType.LOGICAL, logical);
+ }
+
+ protected static List<QueryDataBatch> testPhysicalWithResults(String physical) throws Exception{
+ return testRunAndReturn(QueryType.PHYSICAL, physical);
+ }
+
+ public static List<QueryDataBatch> testRunAndReturn(QueryType type, Object query) throws Exception{
+ if (type == QueryType.PREPARED_STATEMENT) {
+ Preconditions.checkArgument(query instanceof PreparedStatementHandle,
+ "Expected an instance of PreparedStatement as input query");
+ return testPreparedStatement((PreparedStatementHandle)query);
+ } else {
+ Preconditions.checkArgument(query instanceof String, "Expected a string as input query");
+ query = QueryTestUtil.normalizeQuery((String)query);
+ return client.runQuery(type, (String)query);
+ }
+ }
+
+ public static List<QueryDataBatch> testPreparedStatement(PreparedStatementHandle handle) throws Exception {
+ return client.executePreparedStatement(handle);
+ }
+
+ public static int testRunAndPrint(final QueryType type, final String query) throws Exception {
+ return QueryTestUtil.testRunAndPrint(client, type, query);
+ }
+
+ protected static void testWithListener(QueryType type, String query, UserResultsListener resultListener) {
+ QueryTestUtil.testWithListener(client, type, query, resultListener);
+ }
+
+ public static void testNoResult(String query, Object... args) throws Exception {
+ testNoResult(1, query, args);
+ }
+
+ public static void alterSession(String option, Object value) {
+ String valueStr = ClusterFixture.stringify(value);
+ try {
+ test("ALTER SESSION SET `%s` = %s", option, valueStr);
+ } catch(final Exception e) {
+ fail(String.format("Failed to set session option `%s` = %s, Error: %s",
+ option, valueStr, e.toString()));
+ }
+ }
+
+ public static void resetSessionOption(String option) {
+ try {
+ test("ALTER SESSION RESET `%s`", option);
+ } catch(final Exception e) {
+ fail(String.format("Failed to reset session option `%s`, Error: %s",
+ option, e.toString()));
+ }
+ }
+
+ protected static void testNoResult(int interation, String query, Object... args) throws Exception {
+ query = String.format(query, args);
+ logger.debug("Running query:\n--------------\n" + query);
+ for (int i = 0; i < interation; i++) {
+ final List<QueryDataBatch> results = client.runQuery(QueryType.SQL, query);
+ for (final QueryDataBatch queryDataBatch : results) {
+ queryDataBatch.release();
+ }
+ }
+ }
+
+ public static void test(String query, Object... args) throws Exception {
+ QueryTestUtil.test(client, String.format(query, args));
+ }
+
+ public static void test(final String query) throws Exception {
+ QueryTestUtil.test(client, query);
+ }
+
+ protected static int testPhysical(String query) throws Exception{
+ return testRunAndPrint(QueryType.PHYSICAL, query);
+ }
+
+ protected static int testSql(String query) throws Exception{
+ return testRunAndPrint(QueryType.SQL, query);
+ }
+
+ protected static void testPhysicalFromFile(String file) throws Exception{
+ testPhysical(getFile(file));
+ }
+
+ /**
+ * Utility method which tests given query produces a {@link UserException} and the exception message contains
+ * the given message.
+ * @param testSqlQuery Test query
+ * @param expectedErrorMsg Expected error message.
+ */
+ protected static void errorMsgTestHelper(final String testSqlQuery, final String expectedErrorMsg) throws Exception {
+ try {
+ test(testSqlQuery);
+ fail("Expected a UserException when running " + testSqlQuery);
+ } catch (final UserException actualException) {
+ try {
+ assertThat("message of UserException when running " + testSqlQuery, actualException.getMessage(), containsString(expectedErrorMsg));
+ } catch (AssertionError e) {
+ e.addSuppressed(actualException);
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Utility method which tests given query produces a {@link UserException}
+ * with {@link org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType} being DrillPBError.ErrorType.PARSE
+ * the given message.
+ * @param testSqlQuery Test query
+ */
+ protected static void parseErrorHelper(final String testSqlQuery) throws Exception {
+ errorMsgTestHelper(testSqlQuery, UserBitShared.DrillPBError.ErrorType.PARSE.name());
+ }
+
+ public static String getFile(String resource) throws IOException{
+ final URL url = Resources.getResource(resource);
+ if (url == null) {
+ throw new IOException(String.format("Unable to find path %s.", resource));
+ }
+ return Resources.toString(url, Charsets.UTF_8);
+ }
+
+ /**
+ * Copy the resource (ex. file on classpath) to a physical file on FileSystem.
+ * @param resource
+ * @return the file path
+ * @throws IOException
+ */
+ public static String getPhysicalFileFromResource(final String resource) throws IOException {
+ final File file = File.createTempFile("tempfile", ".txt");
+ file.deleteOnExit();
+ final PrintWriter printWriter = new PrintWriter(file);
+ printWriter.write(BaseTestQuery.getFile(resource));
+ printWriter.close();
+
+ return file.getPath();
+ }
+
+ protected static void setSessionOption(final String option, final boolean value) {
+ alterSession(option, value);
+ }
+
+ protected static void setSessionOption(final String option, final long value) {
+ alterSession(option, value);
+ }
+
+ protected static void setSessionOption(final String option, final double value) {
+ alterSession(option, value);
+ }
+
+ protected static void setSessionOption(final String option, final String value) {
+ alterSession(option, value);
+ }
+
+ public static class SilentListener implements UserResultsListener {
+ private final AtomicInteger count = new AtomicInteger();
+
+ @Override
+ public void submissionFailed(UserException ex) {
+ logger.debug("Query failed: " + ex.getMessage());
+ }
+
+ @Override
+ public void queryCompleted(QueryState state) {
+ logger.debug("Query completed successfully with row count: " + count.get());
+ }
+
+ @Override
+ public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) {
+ final int rows = result.getHeader().getRowCount();
+ if (result.getData() != null) {
+ count.addAndGet(rows);
+ }
+ result.release();
+ }
+
+ @Override
+ public void queryIdArrived(QueryId queryId) {}
+
+ }
+
+ protected void setColumnWidth(int columnWidth) {
+ this.columnWidths = new int[] { columnWidth };
+ }
+
+ protected void setColumnWidths(int[] columnWidths) {
+ this.columnWidths = columnWidths;
+ }
+
+ protected int printResult(List<QueryDataBatch> results) throws SchemaChangeException {
+ int rowCount = 0;
+ final RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+ for(final QueryDataBatch result : results) {
+ rowCount += result.getHeader().getRowCount();
+ loader.load(result.getHeader().getDef(), result.getData());
+ // TODO: Clean: DRILL-2933: That load(...) no longer throws
+ // SchemaChangeException, so check/clean throw clause above.
+ VectorUtil.showVectorAccessibleContent(loader, columnWidths);
+ loader.clear();
+ result.release();
+ }
+ System.out.println("Total record count: " + rowCount);
+ return rowCount;
+ }
+
+ protected static String getResultString(List<QueryDataBatch> results, String delimiter)
+ throws SchemaChangeException {
+ final StringBuilder formattedResults = new StringBuilder();
+ boolean includeHeader = true;
+ final RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+ for(final QueryDataBatch result : results) {
+ loader.load(result.getHeader().getDef(), result.getData());
+ if (loader.getRecordCount() <= 0) {
+ continue;
+ }
+ VectorUtil.appendVectorAccessibleContent(loader, formattedResults, delimiter, includeHeader);
+ if (!includeHeader) {
+ includeHeader = false;
+ }
+ loader.clear();
+ result.release();
+ }
+
+ return formattedResults.toString();
+ }
+
+ public class TestResultSet {
+
+ private final List<List<String>> rows;
+
+ public TestResultSet() {
+ rows = new ArrayList<>();
+ }
+
+ public TestResultSet(List<QueryDataBatch> batches) throws SchemaChangeException {
+ rows = new ArrayList<>();
+ convert(batches);
+ }
+
+ public void addRow(String... cells) {
+ List<String> newRow = Arrays.asList(cells);
+ rows.add(newRow);
+ }
+
+ public int size() {
+ return rows.size();
+ }
+
+ @Override public boolean equals(Object o) {
+ boolean result = false;
+
+ if (this == o) {
+ result = true;
+ } else if (o instanceof TestResultSet) {
+ TestResultSet that = (TestResultSet) o;
+ assertEquals(this.size(), that.size());
+ for (int i = 0; i < this.rows.size(); i++) {
+ assertEquals(this.rows.get(i).size(), that.rows.get(i).size());
+ for (int j = 0; j < this.rows.get(i).size(); ++j) {
+ assertEquals(this.rows.get(i).get(j), that.rows.get(i).get(j));
+ }
+ }
+ result = true;
+ }
+
+ return result;
+ }
+
+ private void convert(List<QueryDataBatch> batches) throws SchemaChangeException {
+ RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+ for (QueryDataBatch batch : batches) {
+ int rc = batch.getHeader().getRowCount();
+ if (batch.getData() != null) {
+ loader.load(batch.getHeader().getDef(), batch.getData());
+ for (int i = 0; i < rc; ++i) {
+ List<String> newRow = new ArrayList<>();
+ rows.add(newRow);
+ for (VectorWrapper<?> vw : loader) {
+ ValueVector.Accessor accessor = vw.getValueVector().getAccessor();
+ Object o = accessor.getObject(i);
+ newRow.add(o == null ? null : o.toString());
+ }
+ }
+ }
+ loader.clear();
+ batch.release();
+ }
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java
index a80d4f6..12be961 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java
@@ -27,8 +27,6 @@ import java.io.StringReader;
import java.util.List;
import java.util.Properties;
-import org.apache.drill.QueryTestUtil;
-import org.apache.drill.TestBuilder;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.memory.BufferAllocator;
@@ -341,7 +339,7 @@ public class ClientFixture implements AutoCloseable {
/**
* Execute a set of statements from a file.
- * @param stmts the set of statements, separated by semicolons
+ * @param source the set of statements, separated by semicolons
* @return the number of statements executed
*/
[03/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
index d11af95..f652a91 100644
--- a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
+++ b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
@@ -31,21 +31,97 @@ import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import java.util.UUID;
import java.util.Vector;
import java.util.concurrent.Semaphore;
-import org.apache.commons.io.FileUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
public class ITTestShadedJar {
-
private static DrillbitClassLoader drillbitLoader;
private static URLClassLoader rootClassLoader;
private static int userPort;
+ @ClassRule
+ public static final TestWatcher testWatcher = new TestWatcher() {
+
+ @Override
+ protected void starting(Description description) {
+ super.starting(description);
+
+ try {
+ drillbitLoader = new DrillbitClassLoader();
+ drillbitLoader.loadClass("org.apache.commons.io.FileUtils");
+ rootClassLoader = (URLClassLoader) Thread.currentThread().getContextClassLoader();
+
+ Class<?> clazz = drillbitLoader.loadClass("org.apache.drill.test.BaseTestQuery");
+ Class<?> watcherClazz = drillbitLoader.loadClass("org.apache.drill.test.BaseDirTestWatcher");
+
+ // Setup tmp dirs
+ runMethod("starting", description);
+
+ // Configure tmp dir
+ Object watcher = clazz.getField("dirTestWatcher").get(null);
+ Method method = watcherClazz.getDeclaredMethod("getTmpDir");
+ File tmpDir = (File) method.invoke(watcher);
+ System.setProperty("DRILL_CONF_DIR", tmpDir.getAbsolutePath());
+
+ // Start the drill cluster
+ try {
+ runWithLoader("DrillbitStartThread", drillbitLoader);
+ } catch (Exception e) {
+ printClassesLoaded("root", rootClassLoader);
+ throw e;
+ }
+
+ DrillbitStartThread.SEM.acquire();
+
+ // After starting the drill cluster get the client port
+ userPort = (Integer) clazz.getMethod("getUserPort").invoke(null);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected void finished(Description description) {
+ super.finished(description);
+ done();
+ runMethod("finished", description);
+ }
+
+ @Override
+ protected void failed(Throwable e, Description description) {
+ super.failed(e, description);
+ done();
+ runMethod("failed", description);
+ }
+
+ private void done() {
+ try {
+ runWithLoader("DrillbitStopThread", drillbitLoader);
+ DrillbitStopThread.SEM.acquire();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void runMethod(String name, Description description) {
+ try {
+ Class<?> clazz = drillbitLoader.loadClass("org.apache.drill.test.BaseTestQuery");
+ Class<?> watcherClazz = drillbitLoader.loadClass("org.junit.rules.TestWatcher");
+ Object watcher = clazz.getField("dirTestWatcher").get(null);
+ Method method = watcherClazz.getDeclaredMethod(name, Description.class);
+ method.setAccessible(true);
+ method.invoke(watcher, description);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
private static URL getJdbcUrl() throws MalformedURLException {
return new URL(
String.format("%s../../target/drill-jdbc-all-%s.jar",
@@ -55,22 +131,6 @@ public class ITTestShadedJar {
}
- static {
- String dirConfDir = "DRILL_CONF_DIR";
- if (System.getProperty(dirConfDir) == null) {
- final File condDir = new File(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
- condDir.mkdirs();
- condDir.deleteOnExit();
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- FileUtils.deleteQuietly(condDir);
- }
- });
- System.setProperty(dirConfDir, condDir.getAbsolutePath());
- }
- }
-
@Test
public void testDatabaseVersion() throws Exception {
@@ -90,7 +150,6 @@ public class ITTestShadedJar {
DatabaseMetaData metadata = c.getMetaData();
assertEquals("Apache Drill JDBC Driver", metadata.getDriverName());
assertEquals("Apache Drill Server", metadata.getDatabaseProductName());
- //assertEquals()
}
} catch (Exception ex) {
throw ex;
@@ -114,8 +173,7 @@ public class ITTestShadedJar {
try {
Driver driver = (Driver) clazz.newInstance();
try (Connection c = driver.connect("jdbc:drill:drillbit=localhost:" + userPort, null)) {
- String path = Paths.get("").toAbsolutePath().toString() + "/src/test/resources/types.json";
- printQuery(c, "select * from dfs.`" + path + "`");
+ printQuery(c, "select * from cp.`types.json`");
}
} catch (Exception ex) {
throw ex;
@@ -136,30 +194,6 @@ public class ITTestShadedJar {
}
}
- @BeforeClass
- public static void setupDefaultTestCluster() throws Exception {
- drillbitLoader = new DrillbitClassLoader();
- rootClassLoader = (URLClassLoader) Thread.currentThread().getContextClassLoader();
- try {
- runWithLoader("DrillbitStartThread", drillbitLoader);
- } catch (Exception e) {
- printClassesLoaded("root", rootClassLoader);
- throw e;
- }
-
- Class<?> clazz = drillbitLoader.loadClass("org.apache.drill.BaseTestQuery");
- userPort = (Integer) clazz.getMethod("getUserPort").invoke(null);
-
- DrillbitStartThread.SEM.acquire();
- }
-
- @AfterClass
- public static void closeClient() throws Exception {
- runWithLoader("DrillbitStopThread", drillbitLoader);
-
- DrillbitStopThread.SEM.acquire();
- }
-
private static int getClassesLoadedCount(ClassLoader classLoader) {
try {
Field f = ClassLoader.class.getDeclaredField("classes");
@@ -229,7 +263,7 @@ public class ITTestShadedJar {
@Override
protected void internalRun() throws Exception {
- Class<?> clazz = loader.loadClass("org.apache.drill.BaseTestQuery");
+ Class<?> clazz = loader.loadClass("org.apache.drill.test.BaseTestQuery");
clazz.getMethod("setupDefaultTestCluster").invoke(null);
// loader.loadClass("org.apache.drill.exec.exception.SchemaChangeException");
@@ -252,7 +286,7 @@ public class ITTestShadedJar {
@Override
protected void internalRun() throws Exception {
- Class<?> clazz = loader.loadClass("org.apache.drill.BaseTestQuery");
+ Class<?> clazz = loader.loadClass("org.apache.drill.test.BaseTestQuery");
clazz.getMethod("closeClient").invoke(null);
SEM.release();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
index c0804c5..a2b9211 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.jdbc.impl;
+import java.io.File;
import java.sql.Array;
import java.sql.Blob;
import java.sql.CallableStatement;
@@ -58,7 +59,6 @@ import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.store.StoragePluginRegistry;
-import org.apache.drill.exec.util.TestUtilities;
import org.apache.drill.jdbc.AlreadyClosedSqlException;
import org.apache.drill.jdbc.DrillConnection;
import org.apache.drill.jdbc.DrillConnectionConfig;
@@ -68,6 +68,16 @@ import org.slf4j.Logger;
import com.google.common.base.Throwables;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DEFAULT_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_PLUGIN_NAME;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.ROOT_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.UNIT_TEST_DFS_DEFAULT_PROP;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.UNIT_TEST_DFS_ROOT_PROP;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.UNIT_TEST_DFS_TMP_PROP;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.UNIT_TEST_PROP_PREFIX;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.updateSchemaLocation;
+
/**
* Drill's implementation of {@link Connection}.
*/
@@ -817,16 +827,36 @@ class DrillConnectionImpl extends AvaticaConnection
/**
* Test only code to make JDBC tests run concurrently. If the property <i>drillJDBCUnitTests</i> is set to
* <i>true</i> in connection properties:
- * - Update dfs_test.tmp workspace location with a temp directory. This temp is for exclusive use for test jvm.
+ * - Update dfs.tmp workspace location with a temp directory. This temp is for exclusive use for test jvm.
* - Update dfs.tmp workspace to immutable, so that test writer don't try to create views in dfs.tmp
* @param pluginRegistry
*/
private static void makeTmpSchemaLocationsUnique(StoragePluginRegistry pluginRegistry, Properties props) {
try {
- if (props != null && "true".equalsIgnoreCase(props.getProperty("drillJDBCUnitTests"))) {
- final String tmpDirPath = TestUtilities.createTempDir();
- TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, tmpDirPath);
- TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry);
+ if (props != null && "true".equalsIgnoreCase(props.getProperty(UNIT_TEST_PROP_PREFIX))) {
+ final String logMessage = "The {} property was not configured";
+
+ final String dfsTmpPath = props.getProperty(UNIT_TEST_DFS_TMP_PROP);
+ final String dfsRootPath = props.getProperty(UNIT_TEST_DFS_ROOT_PROP);
+ final String dfsDefaultPath = props.getProperty(UNIT_TEST_DFS_DEFAULT_PROP);
+
+ if (dfsTmpPath == null) {
+ logger.warn(logMessage, UNIT_TEST_DFS_TMP_PROP);
+ } else {
+ updateSchemaLocation(DFS_PLUGIN_NAME, pluginRegistry, new File(dfsTmpPath), TMP_SCHEMA);
+ }
+
+ if (dfsRootPath == null) {
+ logger.warn(logMessage, UNIT_TEST_DFS_ROOT_PROP);
+ } else {
+ updateSchemaLocation(DFS_PLUGIN_NAME, pluginRegistry, new File(dfsRootPath), ROOT_SCHEMA);
+ }
+
+ if (dfsDefaultPath == null) {
+ logger.warn(logMessage, UNIT_TEST_DFS_DEFAULT_PROP);
+ } else {
+ updateSchemaLocation(DFS_PLUGIN_NAME, pluginRegistry, new File(dfsDefaultPath), DEFAULT_SCHEMA);
+ }
}
} catch(Throwable e) {
// Reason for catching Throwable is to capture NoSuchMethodError etc which depend on certain classed to be
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/DatabaseMetaDataGetColumnsTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DatabaseMetaDataGetColumnsTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DatabaseMetaDataGetColumnsTest.java
index 894abd9..ed95162 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DatabaseMetaDataGetColumnsTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DatabaseMetaDataGetColumnsTest.java
@@ -20,6 +20,7 @@ package org.apache.drill.jdbc;
import static java.sql.ResultSetMetaData.columnNoNulls;
import static java.sql.ResultSetMetaData.columnNullable;
import static java.sql.ResultSetMetaData.columnNullableUnknown;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertNotNull;
@@ -35,7 +36,6 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
-import org.apache.drill.jdbc.test.JdbcAssert;
import org.apache.drill.categories.JdbcTest;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -85,8 +85,6 @@ import org.junit.experimental.categories.Category;
*/
@Category(JdbcTest.class)
public class DatabaseMetaDataGetColumnsTest extends JdbcTestBase {
-
- private static final String VIEW_SCHEMA = "dfs_test.tmp";
private static final String VIEW_NAME =
DatabaseMetaDataGetColumnsTest.class.getSimpleName() + "_View";
@@ -189,8 +187,7 @@ public class DatabaseMetaDataGetColumnsTest extends JdbcTestBase {
// (Note: Can't use JdbcTest's connect(...) because JdbcTest closes
// Connection--and other JDBC objects--on test method failure, but this test
// class uses some objects across methods.)
- connection = new Driver().connect( "jdbc:drill:zk=local",
- JdbcAssert.getDefaultProperties() );
+ connection = connect();
dbMetadata = connection.getMetaData();
setUpMetadataToCheck();
@@ -229,7 +226,7 @@ public class DatabaseMetaDataGetColumnsTest extends JdbcTestBase {
TODO(DRILL-3253)(end) */
// Create temporary test-columns view:
- util = stmt.executeQuery( "USE dfs_test.tmp" );
+ util = stmt.executeQuery( "USE dfs.tmp" );
assertTrue( util.next() );
assertTrue( "Error setting schema for test: " + util.getString( 2 ),
util.getBoolean( 1 ) );
@@ -290,48 +287,48 @@ public class DatabaseMetaDataGetColumnsTest extends JdbcTestBase {
// Set up result rows for temporary test view and Hivetest columns:
- mdrOptBOOLEAN = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptBOOLEAN" );
+ mdrOptBOOLEAN = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptBOOLEAN" );
// TODO(DRILL-2470): Uncomment when TINYINT is implemented:
//mdrReqTINYINT = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqTINYINT" );
// TODO(DRILL-2470): Uncomment when SMALLINT is implemented:
//mdrOptSMALLINT = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptSMALLINT" );
- mdrReqINTEGER = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTEGER" );
- mdrOptBIGINT = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptBIGINT" );
+ mdrReqINTEGER = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTEGER" );
+ mdrOptBIGINT = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptBIGINT" );
// TODO(DRILL-2683): Uncomment when REAL is implemented:
//mdrOptREAL = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptREAL" );
- mdrOptFLOAT = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptFLOAT" );
- mdrReqDOUBLE = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqDOUBLE" );
-
- mdrReqDECIMAL_5_3 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqDECIMAL_5_3" );
-
- mdrReqVARCHAR_10 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqVARCHAR_10" );
- mdrOptVARCHAR = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptVARCHAR" );
- mdrReqCHAR_5 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqCHAR_5" );
- mdrOptVARBINARY_16 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptVARBINARY_16" );
- mdrOptBINARY_1048576 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptBINARY_1048576" );
-
- mdrReqDATE = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqDATE" );
- mdrReqTIME = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqTIME" );
- mdrOptTIME_7 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptTIME_7" );
- mdrOptTIMESTAMP = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptTIMESTAMP" );
-
- mdrReqINTERVAL_Y = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Y" );
- mdrReqINTERVAL_3Y_Mo = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3Y_Mo" );
- mdrReqINTERVAL_Mo = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Mo" );
- mdrReqINTERVAL_D = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_D" );
- mdrReqINTERVAL_4D_H = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_4D_H" );
- mdrReqINTERVAL_3D_Mi = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3D_Mi" );
- mdrReqINTERVAL_2D_S5 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_2D_S5" );
- mdrReqINTERVAL_H = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_H" );
- mdrReqINTERVAL_1H_Mi = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_1H_Mi" );
- mdrReqINTERVAL_3H_S1 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3H_S1" );
- mdrReqINTERVAL_Mi = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Mi" );
- mdrReqINTERVAL_5Mi_S = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_5Mi_S" );
- mdrReqINTERVAL_S = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_S" );
- mdrReqINTERVAL_3S = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3S" );
- mdrReqINTERVAL_3S1 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3S1" );
+ mdrOptFLOAT = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptFLOAT" );
+ mdrReqDOUBLE = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqDOUBLE" );
+
+ mdrReqDECIMAL_5_3 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqDECIMAL_5_3" );
+
+ mdrReqVARCHAR_10 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqVARCHAR_10" );
+ mdrOptVARCHAR = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptVARCHAR" );
+ mdrReqCHAR_5 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqCHAR_5" );
+ mdrOptVARBINARY_16 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptVARBINARY_16" );
+ mdrOptBINARY_1048576 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptBINARY_1048576" );
+
+ mdrReqDATE = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqDATE" );
+ mdrReqTIME = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqTIME" );
+ mdrOptTIME_7 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptTIME_7" );
+ mdrOptTIMESTAMP = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptTIMESTAMP" );
+
+ mdrReqINTERVAL_Y = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Y" );
+ mdrReqINTERVAL_3Y_Mo = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3Y_Mo" );
+ mdrReqINTERVAL_Mo = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Mo" );
+ mdrReqINTERVAL_D = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_D" );
+ mdrReqINTERVAL_4D_H = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_4D_H" );
+ mdrReqINTERVAL_3D_Mi = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3D_Mi" );
+ mdrReqINTERVAL_2D_S5 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_2D_S5" );
+ mdrReqINTERVAL_H = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_H" );
+ mdrReqINTERVAL_1H_Mi = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_1H_Mi" );
+ mdrReqINTERVAL_3H_S1 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3H_S1" );
+ mdrReqINTERVAL_Mi = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Mi" );
+ mdrReqINTERVAL_5Mi_S = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_5Mi_S" );
+ mdrReqINTERVAL_S = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_S" );
+ mdrReqINTERVAL_3S = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3S" );
+ mdrReqINTERVAL_3S1 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3S1" );
/* TODO(DRILL-3253)(start): Update this once we have test plugin supporting all needed types.
mdrReqARRAY = setUpRow( "hive_test.default", "infoschematest", "listtype" );
@@ -442,7 +439,7 @@ public class DatabaseMetaDataGetColumnsTest extends JdbcTestBase {
@Test
public void test_TABLE_SCHEM_hasRightValue_mdrOptBOOLEAN() throws SQLException {
- assertThat( mdrOptBOOLEAN.getString( "TABLE_SCHEM" ), equalTo( VIEW_SCHEMA ) );
+ assertThat( mdrOptBOOLEAN.getString( "TABLE_SCHEM" ), equalTo( DFS_TMP_SCHEMA ) );
}
// Not bothering with other _local_view_ test columns for TABLE_SCHEM.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java
index a522954..e75d19c 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/DrillResultSetTest.java
@@ -31,8 +31,6 @@ import java.sql.Statement;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.jdbc.test.JdbcAssert;
-import org.apache.drill.test.DrillTest;
import org.apache.drill.categories.JdbcTest;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -40,7 +38,7 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({SlowTest.class, JdbcTest.class})
-public class DrillResultSetTest extends DrillTest {
+public class DrillResultSetTest extends JdbcTestBase {
// TODO: Move Jetty status server disabling to DrillTest.
private static final String STATUS_SERVER_PROPERTY_NAME =
@@ -66,8 +64,7 @@ public class DrillResultSetTest extends DrillTest {
public void test_next_blocksFurtherAccessAfterEnd()
throws SQLException
{
- Connection connection =
- new Driver().connect( "jdbc:drill:zk=local", JdbcAssert.getDefaultProperties() );
+ Connection connection = connect();
Statement statement = connection.createStatement();
ResultSet resultSet =
statement.executeQuery( "SELECT 1 AS x \n" +
@@ -107,8 +104,7 @@ public class DrillResultSetTest extends DrillTest {
public void test_next_blocksFurtherAccessWhenNoRows()
throws Exception
{
- Connection connection =
- new Driver().connect( "jdbc:drill:zk=local", JdbcAssert.getDefaultProperties() );
+ Connection connection = connect();
Statement statement = connection.createStatement();
ResultSet resultSet =
statement.executeQuery( "SELECT 'Hi' AS x \n" +
@@ -142,8 +138,7 @@ public class DrillResultSetTest extends DrillTest {
public void test_getRow_isOneBased()
throws Exception
{
- Connection connection =
- new Driver().connect( "jdbc:drill:zk=local", JdbcAssert.getDefaultProperties() );
+ Connection connection = connect();
Statement statement = connection.createStatement();
ResultSet resultSet =
statement.executeQuery( "VALUES (1), (2)" );
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
index 8763161..c275d00 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/JdbcTestBase.java
@@ -22,17 +22,38 @@ import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import java.util.Properties;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import org.apache.calcite.linq4j.Ord;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.logical.data.LogicalOperator;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ExecTest;
-import org.apache.drill.jdbc.test.JdbcAssert;
+import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory;
+import org.apache.drill.exec.util.StoragePluginTestUtils;
+import org.apache.drill.jdbc.test.Hook;
import org.apache.drill.categories.JdbcTest;
+import org.apache.drill.test.BaseDirTestWatcher;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
@@ -49,6 +70,9 @@ public class JdbcTestBase extends ExecTest {
@SuppressWarnings("unused")
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JdbcTestBase.class);
+ @ClassRule
+ public static final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
@Rule
public final TestRule watcher = new TestWatcher() {
@Override
@@ -67,7 +91,6 @@ public class JdbcTestBase extends ExecTest {
return DriverManager.getConnection(info.getUrl(), info.getParamsAsProperties());
}
});
- JdbcAssert.setFactory(factory);
}
/**
@@ -76,9 +99,12 @@ public class JdbcTestBase extends ExecTest {
* @throws Exception if connection fails
*/
protected static Connection connect(String url) throws SQLException {
- return connect(url, JdbcAssert.getDefaultProperties());
+ return connect(url, getDefaultProperties());
}
+ protected static Connection connect() throws SQLException {
+ return connect("jdbc:drill:zk=local", getDefaultProperties());
+ }
/**
* Creates a {@link java.sql.Connection connection} using the given parameters.
@@ -144,6 +170,307 @@ public class JdbcTestBase extends ExecTest {
}
/**
+ * Returns default bag of properties that is passed to JDBC connection.
+ * By default, includes options to:
+ * - turn off the web server
+ * - indicate DrillConnectionImpl to set up dfs.tmp schema location to an exclusive dir just for this test jvm
+ */
+ public static Properties getDefaultProperties() {
+ final Properties properties = new Properties();
+ properties.setProperty(StoragePluginTestUtils.UNIT_TEST_PROP_PREFIX, "true");
+
+ properties.setProperty(StoragePluginTestUtils.UNIT_TEST_DFS_TMP_PROP, dirTestWatcher.getDfsTestTmpDir().getAbsolutePath());
+ properties.setProperty(StoragePluginTestUtils.UNIT_TEST_DFS_ROOT_PROP, dirTestWatcher.getRootDir().getAbsolutePath());
+ properties.setProperty(StoragePluginTestUtils.UNIT_TEST_DFS_DEFAULT_PROP, dirTestWatcher.getRootDir().getAbsolutePath());
+ properties.setProperty(ExecConstants.DRILL_TMP_DIR, dirTestWatcher.getTmpDir().getAbsolutePath());
+ properties.setProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false");
+ properties.setProperty(ExecConstants.HTTP_ENABLE, "false");
+ return properties;
+ }
+
+ public static ModelAndSchema withModel(final String model, final String schema) {
+ final Properties info = getDefaultProperties();
+ info.setProperty("schema", schema);
+ info.setProperty("model", "inline:" + model);
+ return new ModelAndSchema(info, factory);
+ }
+
+ public static ModelAndSchema withFull(final String schema) {
+ final Properties info = getDefaultProperties();
+ info.setProperty("schema", schema);
+ return new ModelAndSchema(info, factory);
+ }
+
+ public static ModelAndSchema withNoDefaultSchema() {
+ return new ModelAndSchema(getDefaultProperties(), factory);
+ }
+
+ public static String toString(ResultSet resultSet, int expectedRecordCount) throws SQLException {
+ final StringBuilder buf = new StringBuilder();
+ while (resultSet.next()) {
+ final ResultSetMetaData metaData = resultSet.getMetaData();
+ final int n = metaData.getColumnCount();
+ String sep = "";
+ for (int i = 1; i <= n; i++) {
+ buf.append(sep)
+ .append(metaData.getColumnLabel(i))
+ .append("=")
+ .append(resultSet.getObject(i));
+ sep = "; ";
+ }
+ buf.append("\n");
+ }
+ return buf.toString();
+ }
+
+ public static String toString(ResultSet resultSet) throws SQLException {
+ StringBuilder buf = new StringBuilder();
+ final List<Ord<String>> columns = columnLabels(resultSet);
+ while (resultSet.next()) {
+ for (Ord<String> column : columns) {
+ buf.append(column.i == 1 ? "" : "; ").append(column.e).append("=").append(resultSet.getObject(column.i));
+ }
+ buf.append("\n");
+ }
+ return buf.toString();
+ }
+
+ static Set<String> toStringSet(ResultSet resultSet) throws SQLException {
+ ImmutableSet.Builder<String> builder = ImmutableSet.builder();
+ final List<Ord<String>> columns = columnLabels(resultSet);
+ while (resultSet.next()) {
+ StringBuilder buf = new StringBuilder();
+ for (Ord<String> column : columns) {
+ buf.append(column.i == 1 ? "" : "; ").append(column.e).append("=").append(resultSet.getObject(column.i));
+ }
+ builder.add(buf.toString());
+ buf.setLength(0);
+ }
+ return builder.build();
+ }
+
+ static List<String> toStrings(ResultSet resultSet) throws SQLException {
+ final List<String> list = new ArrayList<>();
+ StringBuilder buf = new StringBuilder();
+ final List<Ord<String>> columns = columnLabels(resultSet);
+ while (resultSet.next()) {
+ buf.setLength(0);
+ for (Ord<String> column : columns) {
+ buf.append(column.i == 1 ? "" : "; ").append(column.e).append("=").append(resultSet.getObject(column.i));
+ }
+ list.add(buf.toString());
+ }
+ return list;
+ }
+
+ private static List<Ord<String>> columnLabels(ResultSet resultSet) throws SQLException {
+ int n = resultSet.getMetaData().getColumnCount();
+ List<Ord<String>> columns = new ArrayList<>();
+ for (int i = 1; i <= n; i++) {
+ columns.add(Ord.of(i, resultSet.getMetaData().getColumnLabel(i)));
+ }
+ return columns;
+ }
+
+ public static class ModelAndSchema {
+ private final Properties info;
+ private final ConnectionFactoryAdapter adapter;
+
+ public ModelAndSchema(final Properties info, final ConnectionFactory factory) {
+ this.info = info;
+ this.adapter = new ConnectionFactoryAdapter() {
+ @Override
+ public Connection createConnection() throws SQLException {
+ return factory.getConnection(new ConnectionInfo("jdbc:drill:zk=local", ModelAndSchema.this.info));
+ }
+ };
+ }
+
+ public TestDataConnection sql(String sql) {
+ return new TestDataConnection(adapter, sql);
+ }
+
+ public <T> T withConnection(Function<Connection, T> function) throws Exception {
+ Connection connection = null;
+ try {
+ connection = adapter.createConnection();
+ return function.apply(connection);
+ } finally {
+ if (connection != null) {
+ connection.close();
+ }
+ }
+ }
+ }
+
+ public static class TestDataConnection {
+ private final ConnectionFactoryAdapter adapter;
+ private final String sql;
+
+ TestDataConnection(ConnectionFactoryAdapter adapter, String sql) {
+ this.adapter = adapter;
+ this.sql = sql;
+ }
+
+ /**
+ * Checks that the current SQL statement returns the expected result.
+ */
+ public TestDataConnection returns(String expected) throws Exception {
+ Connection connection = null;
+ Statement statement = null;
+ try {
+ connection = adapter.createConnection();
+ statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql);
+ expected = expected.trim();
+ String result = JdbcTestBase.toString(resultSet).trim();
+ resultSet.close();
+
+ if (!expected.equals(result)) {
+ Assert.fail(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected));
+ }
+ return this;
+ } finally {
+ if (statement != null) {
+ statement.close();
+ }
+ if (connection != null) {
+ connection.close();
+ }
+ }
+ }
+
+ public TestDataConnection returnsSet(Set<String> expected) throws Exception {
+ Connection connection = null;
+ Statement statement = null;
+ try {
+ connection = adapter.createConnection();
+ statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql);
+ Set<String> result = JdbcTestBase.toStringSet(resultSet);
+ resultSet.close();
+
+ if (!expected.equals(result)) {
+ Assert.fail(String.format("Generated set:\n%s\ndoes not match:\n%s", result, expected));
+ }
+ return this;
+ } finally {
+ if (statement != null) {
+ statement.close();
+ }
+ if (connection != null) {
+ connection.close();
+ }
+ }
+ }
+
+ /**
+ * Checks that the current SQL statement returns the expected result lines. Lines are compared unordered; the test
+ * succeeds if the query returns these lines in any order.
+ */
+ public TestDataConnection returnsUnordered(String... expecteds) throws Exception {
+ Connection connection = null;
+ Statement statement = null;
+ try {
+ connection = adapter.createConnection();
+ statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql);
+ Assert.assertEquals(unsortedList(Arrays.asList(expecteds)), unsortedList(JdbcTestBase.toStrings(resultSet)));
+ resultSet.close();
+ return this;
+ } finally {
+ if (statement != null) {
+ statement.close();
+ }
+ if (connection != null) {
+ connection.close();
+ }
+ }
+ }
+
+ public TestDataConnection displayResults(int recordCount) throws Exception {
+ // record count check is done in toString method
+
+ Connection connection = null;
+ Statement statement = null;
+ try {
+ connection = adapter.createConnection();
+ statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql);
+ System.out.println(JdbcTestBase.toString(resultSet, recordCount));
+ resultSet.close();
+ return this;
+ } finally {
+ if (statement != null) {
+ statement.close();
+ }
+ if (connection != null) {
+ connection.close();
+ }
+ }
+ }
+
+ private SortedSet<String> unsortedList(List<String> strings) {
+ final SortedSet<String> set = new TreeSet<>();
+ for (String string : strings) {
+ set.add(string + "\n");
+ }
+ return set;
+ }
+
+ public LogicalPlan logicalPlan() {
+ final String[] plan0 = {null};
+ Connection connection = null;
+ Statement statement = null;
+ final Hook.Closeable x = Hook.LOGICAL_PLAN.add(new Function<String, Void>() {
+ @Override
+ public Void apply(String o) {
+ plan0[0] = o;
+ return null;
+ }
+ });
+ try {
+ connection = adapter.createConnection();
+ statement = connection.prepareStatement(sql);
+ statement.close();
+ final String plan = plan0[0].trim();
+ return LogicalPlan.parse(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(DrillConfig.create()), plan);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ // ignore
+ }
+ }
+ if (connection != null) {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ // ignore
+ }
+ }
+ x.close();
+ }
+ }
+
+ public <T extends LogicalOperator> T planContains(final Class<T> operatorClazz) {
+ return (T) Iterables.find(logicalPlan().getSortedOperators(), new Predicate<LogicalOperator>() {
+ @Override
+ public boolean apply(LogicalOperator input) {
+ return input.getClass().equals(operatorClazz);
+ }
+ });
+ }
+ }
+
+ private interface ConnectionFactoryAdapter {
+ Connection createConnection() throws Exception;
+ }
+
+ /**
* Test of whether tests that get connection from JdbcTest.connect(...)
* work with resetting of connections. If enabling this (failing) test method
* causes other test methods to fail, something needs to be fixed.
@@ -155,5 +482,4 @@ public class JdbcTestBase extends ExecTest {
public void testJdbcTestConnectionResettingCompatibility() {
fail("Intentional failure--did other test methods still run?");
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/LegacyDatabaseMetaDataGetColumnsTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/LegacyDatabaseMetaDataGetColumnsTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/LegacyDatabaseMetaDataGetColumnsTest.java
index 993c3e3..068fd6b 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/LegacyDatabaseMetaDataGetColumnsTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/LegacyDatabaseMetaDataGetColumnsTest.java
@@ -24,7 +24,6 @@ import java.sql.SQLException;
import java.sql.Types;
import java.util.Properties;
-import org.apache.drill.jdbc.test.JdbcAssert;
import org.apache.drill.categories.JdbcTest;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -42,11 +41,10 @@ public class LegacyDatabaseMetaDataGetColumnsTest extends DatabaseMetaDataGetCol
// (Note: Can't use JdbcTest's connect(...) because JdbcTest closes
// Connection--and other JDBC objects--on test method failure, but this test
// class uses some objects across methods.)
- Properties defaultProperties = JdbcAssert.getDefaultProperties();
+ Properties defaultProperties = getDefaultProperties();
defaultProperties.setProperty("server.metadata.disabled", "true");
- connection = new Driver().connect( "jdbc:drill:zk=local",
- defaultProperties );
+ connection = connect("jdbc:drill:zk=local", defaultProperties);
dbMetadata = connection.getMetaData();
DatabaseMetaDataGetColumnsTest.setUpMetadataToCheck();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/ResultSetMetaDataTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/ResultSetMetaDataTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/ResultSetMetaDataTest.java
index b316b95..1ecfba6 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/ResultSetMetaDataTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/ResultSetMetaDataTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.jdbc;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
@@ -33,7 +34,6 @@ import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
-import org.apache.drill.jdbc.test.JdbcAssert;
import org.apache.drill.categories.JdbcTest;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -49,8 +49,6 @@ import org.junit.experimental.categories.Category;
*/
@Category(JdbcTest.class)
public class ResultSetMetaDataTest extends JdbcTestBase {
-
- private static final String VIEW_SCHEMA = "dfs_test.tmp";
private static final String VIEW_NAME =
ResultSetMetaDataTest.class.getSimpleName() + "_View";
@@ -122,14 +120,13 @@ public class ResultSetMetaDataTest extends JdbcTestBase {
// (Note: Can't use JdbcTest's connect(...) because JdbcTest closes
// Connection--and other JDBC objects--on test method failure, but this test
// class uses some objects across methods.)
- connection = new Driver().connect( "jdbc:drill:zk=local",
- JdbcAssert.getDefaultProperties() );
+ connection = connect("jdbc:drill:zk=local");
final Statement stmt = connection.createStatement();
ResultSet util;
// Create temporary test-columns view:
- util = stmt.executeQuery( "USE `" + VIEW_SCHEMA + "`" );
+ util = stmt.executeQuery( "USE `" + DFS_TMP_SCHEMA + "`" );
assertTrue( util.next() );
assertTrue( "Error setting schema for test: " + util.getString( 2 ),
util.getBoolean( 1 ) );
@@ -433,7 +430,7 @@ public class ResultSetMetaDataTest extends JdbcTestBase {
@Test
public void test_getSchemaName_forViewGetsName() throws SQLException {
assertThat( rowMetadata.getSchemaName( ordOptBOOLEAN ),
- anyOf( equalTo( VIEW_SCHEMA ),
+ anyOf( equalTo( DFS_TMP_SCHEMA ),
equalTo( "" ) ) );
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ConnectionCloseTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ConnectionCloseTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ConnectionCloseTest.java
index 4ecddfc..397d2a4 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ConnectionCloseTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ConnectionCloseTest.java
@@ -19,7 +19,7 @@ package org.apache.drill.jdbc.test;
import java.sql.Connection;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.categories.JdbcTest;
@@ -79,8 +79,7 @@ public class Bug1735ConnectionCloseTest extends JdbcTestQueryBase {
for ( int i = 1; i <= SMALL_ITERATION_COUNT; i++ ) {
logger.info( "iteration " + i + ":" );
System.out.println( "iteration " + i + ":" );
- Connection connection = new Driver().connect( "jdbc:drill:zk=local",
- JdbcAssert.getDefaultProperties() );
+ Connection connection = new Driver().connect("jdbc:drill:zk=local", getDefaultProperties());
connection.close();
}
}
@@ -100,8 +99,7 @@ public class Bug1735ConnectionCloseTest extends JdbcTestQueryBase {
// (Note: Can't use JdbcTest's connect(...) because it returns connection
// that doesn't really close.
- Connection connection = new Driver().connect( "jdbc:drill:zk=local",
- JdbcAssert.getDefaultProperties() );
+ Connection connection = new Driver().connect("jdbc:drill:zk=local", getDefaultProperties());
connection.close();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ResultSetCloseReleasesBuffersTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ResultSetCloseReleasesBuffersTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ResultSetCloseReleasesBuffersTest.java
index 72b8dad..9449724 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ResultSetCloseReleasesBuffersTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Bug1735ResultSetCloseReleasesBuffersTest.java
@@ -67,14 +67,13 @@ public class Bug1735ResultSetCloseReleasesBuffersTest extends JdbcTestQueryBase
@Test
public void test() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.withConnection(
new Function<Connection, Void>() {
public Void apply( Connection connection ) {
try {
Statement statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery( "USE dfs_test.tmp" );
+ ResultSet resultSet = statement.executeQuery( "USE dfs.tmp" );
// TODO: Purge nextUntilEnd(...) and calls when remaining fragment
// race conditions are fixed (not just DRILL-2245 fixes).
// resultSet.close( resultSet );
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2128GetColumnsDataTypeNotTypeCodeIntBugsTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2128GetColumnsDataTypeNotTypeCodeIntBugsTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2128GetColumnsDataTypeNotTypeCodeIntBugsTest.java
index 6d3b257..e167e4b 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2128GetColumnsDataTypeNotTypeCodeIntBugsTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2128GetColumnsDataTypeNotTypeCodeIntBugsTest.java
@@ -37,7 +37,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTestBase;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java
index 83ddac8..c920c41 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java
@@ -26,9 +26,11 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.Properties;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.categories.JdbcTest;
+import org.apache.drill.jdbc.JdbcTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -40,16 +42,14 @@ import org.junit.experimental.categories.Category;
*/
@Category(JdbcTest.class)
public class Drill2288GetColumnsMetadataWhenNoRowsTest {
-
private static Connection connection;
-
@BeforeClass
public static void setUpConnection() throws SQLException {
// (Note: Can't use JdbcTest's connect(...) because JdbcTest closes
// Connection--and other JDBC objects--on test method failure, but this test
// class uses some objects across methods.)
- connection = new Driver().connect( "jdbc:drill:zk=local", null );
+ connection = new Driver().connect( "jdbc:drill:zk=local", JdbcTestBase.getDefaultProperties());
}
@AfterClass
@@ -197,6 +197,4 @@ public class Drill2288GetColumnsMetadataWhenNoRowsTest {
assertThat( "Unexpected non-empty results. Test rot?",
false, equalTo( results.next() ) );
}
-
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2439GetBooleanFailsSayingWrongTypeBugTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2439GetBooleanFailsSayingWrongTypeBugTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2439GetBooleanFailsSayingWrongTypeBugTest.java
index e5d88d8..3fab8ce 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2439GetBooleanFailsSayingWrongTypeBugTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2439GetBooleanFailsSayingWrongTypeBugTest.java
@@ -41,7 +41,7 @@ public class Drill2439GetBooleanFailsSayingWrongTypeBugTest extends JdbcTestBase
@BeforeClass
public static void setUpConnection() throws SQLException {
- connection = new Driver().connect( "jdbc:drill:zk=local", JdbcAssert.getDefaultProperties() );
+ connection = connect();
statement = connection.createStatement();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2461IntervalsBreakInfoSchemaBugTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2461IntervalsBreakInfoSchemaBugTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2461IntervalsBreakInfoSchemaBugTest.java
index dcf5bda..7e981f4 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2461IntervalsBreakInfoSchemaBugTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2461IntervalsBreakInfoSchemaBugTest.java
@@ -56,9 +56,9 @@ public class Drill2461IntervalsBreakInfoSchemaBugTest extends JdbcTestBase {
ResultSet util;
// Create a view using an INTERVAL type:
- util = stmt.executeQuery( "USE dfs_test.tmp" );
+ util = stmt.executeQuery( "USE dfs.tmp" );
assertTrue( util.next() );
- assertTrue( "Error setting schema to dfs_test.tmp: " + util.getString( 2 ), util.getBoolean( 1 ) );
+ assertTrue( "Error setting schema to dfs.tmp: " + util.getString( 2 ), util.getBoolean( 1 ) );
util = stmt.executeQuery(
"CREATE OR REPLACE VIEW " + VIEW_NAME + " AS "
+ "\n SELECT CAST( NULL AS INTERVAL HOUR(4) TO MINUTE ) AS optINTERVAL_HM "
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2463GetNullsFailedWithAssertionsBugTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2463GetNullsFailedWithAssertionsBugTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2463GetNullsFailedWithAssertionsBugTest.java
index a4b1065..66f0446 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2463GetNullsFailedWithAssertionsBugTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2463GetNullsFailedWithAssertionsBugTest.java
@@ -45,7 +45,7 @@ public class Drill2463GetNullsFailedWithAssertionsBugTest extends JdbcTestBase {
// (Note: Can't use JdbcTest's connect(...) because JdbcTest closes
// Connection--and other JDBC objects--on test method failure, but this test
// class uses some objects across methods.)
- connection = new Driver().connect( "jdbc:drill:zk=local", JdbcAssert.getDefaultProperties() );
+ connection = connect();
statement = connection.createStatement();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java
index f26b4c6..26ab7bd 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2489CallsAfterCloseThrowExceptionsTest.java
@@ -60,7 +60,7 @@ import org.apache.drill.jdbc.AlreadyClosedSqlException;
* {@link Statement},
* {@link PreparedStatement},
* {@link ResultSet},
- * {@link ResultSetMetadata}, and
+ * {@link java.sql.ResultSetMetaData}, and
* {@link DatabaseMetaData}.
* </p>
* <p>
@@ -89,11 +89,9 @@ public class Drill2489CallsAfterCloseThrowExceptionsTest extends JdbcTestBase {
// (Note: Can't use JdbcTest's connect(...) for this test class.)
final Connection connToClose =
- new Driver().connect("jdbc:drill:zk=local",
- JdbcAssert.getDefaultProperties());
+ new Driver().connect("jdbc:drill:zk=local", getDefaultProperties());
final Connection connToKeep =
- new Driver().connect("jdbc:drill:zk=local",
- JdbcAssert.getDefaultProperties());
+ new Driver().connect("jdbc:drill:zk=local", getDefaultProperties());
final Statement plainStmtToClose = connToKeep.createStatement();
final Statement plainStmtToKeep = connToKeep.createStatement();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java
index 042c178..1a6ac40 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2769UnsupportedReportsUseSqlExceptionTest.java
@@ -34,9 +34,8 @@ import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.jdbc.AlreadyClosedSqlException;
-import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTestBase;
import org.apache.drill.categories.JdbcTest;
import org.junit.AfterClass;
@@ -84,9 +83,7 @@ public class Drill2769UnsupportedReportsUseSqlExceptionTest extends JdbcTestBase
@BeforeClass
public static void setUpObjects() throws Exception {
// (Note: Can't use JdbcTest's connect(...) for this test class.)
-
- connection = new Driver().connect("jdbc:drill:zk=local",
- JdbcAssert.getDefaultProperties());
+ connection = connect();
plainStatement = connection.createStatement();
preparedStatement =
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
deleted file mode 100644
index da66ac1..0000000
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.jdbc.test;
-
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.calcite.linq4j.Ord;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.logical.data.LogicalOperator;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory;
-import org.apache.drill.jdbc.ConnectionFactory;
-import org.apache.drill.jdbc.ConnectionInfo;
-import org.junit.Assert;
-
-import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSet.Builder;
-import com.google.common.collect.Iterables;
-
-/**
- * Fluent interface for writing JDBC and query-planning tests.
- */
-public class JdbcAssert {
-
- private static ConnectionFactory factory = null;
-
- public static void setFactory(ConnectionFactory factory) {
- JdbcAssert.factory = factory;
- }
-
- /**
- * Returns default bag of properties that is passed to JDBC connection.
- * By default, includes options to:
- * - turn off the web server
- * - indicate DrillConnectionImpl to set up dfs_test.tmp schema location to an exclusive dir just for this test jvm
- */
- public static Properties getDefaultProperties() {
- final Properties properties = new Properties();
- properties.setProperty("drillJDBCUnitTests", "true");
-
- // Must set this to false to ensure that the tests ignore any existing
- // plugin configurations stored in /tmp/drill.
-
- properties.setProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false");
- properties.setProperty(ExecConstants.HTTP_ENABLE, "false");
- return properties;
- }
-
- public static ModelAndSchema withModel(final String model, final String schema) {
- final Properties info = getDefaultProperties();
- info.setProperty("schema", schema);
- info.setProperty("model", "inline:" + model);
- return new ModelAndSchema(info, factory);
- }
-
- public static ModelAndSchema withFull(final String schema) {
- final Properties info = getDefaultProperties();
- info.setProperty("schema", schema);
- return new ModelAndSchema(info, factory);
- }
-
- public static ModelAndSchema withNoDefaultSchema() {
- return new ModelAndSchema(getDefaultProperties(), factory);
- }
-
- static String toString(ResultSet resultSet, int expectedRecordCount) throws SQLException {
- final StringBuilder buf = new StringBuilder();
- while (resultSet.next()) {
- final ResultSetMetaData metaData = resultSet.getMetaData();
- final int n = metaData.getColumnCount();
- String sep = "";
- for (int i = 1; i <= n; i++) {
- buf.append(sep)
- .append(metaData.getColumnLabel(i))
- .append("=")
- .append(resultSet.getObject(i));
- sep = "; ";
- }
- buf.append("\n");
- }
- return buf.toString();
- }
-
- static String toString(ResultSet resultSet) throws SQLException {
- StringBuilder buf = new StringBuilder();
- final List<Ord<String>> columns = columnLabels(resultSet);
- while (resultSet.next()) {
- for (Ord<String> column : columns) {
- buf.append(column.i == 1 ? "" : "; ").append(column.e).append("=").append(resultSet.getObject(column.i));
- }
- buf.append("\n");
- }
- return buf.toString();
- }
-
- static Set<String> toStringSet(ResultSet resultSet) throws SQLException {
- Builder<String> builder = ImmutableSet.builder();
- final List<Ord<String>> columns = columnLabels(resultSet);
- while (resultSet.next()) {
- StringBuilder buf = new StringBuilder();
- for (Ord<String> column : columns) {
- buf.append(column.i == 1 ? "" : "; ").append(column.e).append("=").append(resultSet.getObject(column.i));
- }
- builder.add(buf.toString());
- buf.setLength(0);
- }
- return builder.build();
- }
-
- static List<String> toStrings(ResultSet resultSet) throws SQLException {
- final List<String> list = new ArrayList<>();
- StringBuilder buf = new StringBuilder();
- final List<Ord<String>> columns = columnLabels(resultSet);
- while (resultSet.next()) {
- buf.setLength(0);
- for (Ord<String> column : columns) {
- buf.append(column.i == 1 ? "" : "; ").append(column.e).append("=").append(resultSet.getObject(column.i));
- }
- list.add(buf.toString());
- }
- return list;
- }
-
- private static List<Ord<String>> columnLabels(ResultSet resultSet) throws SQLException {
- int n = resultSet.getMetaData().getColumnCount();
- List<Ord<String>> columns = new ArrayList<>();
- for (int i = 1; i <= n; i++) {
- columns.add(Ord.of(i, resultSet.getMetaData().getColumnLabel(i)));
- }
- return columns;
- }
-
- public static class ModelAndSchema {
- private final Properties info;
- private final ConnectionFactoryAdapter adapter;
-
- public ModelAndSchema(final Properties info, final ConnectionFactory factory) {
- this.info = info;
- this.adapter = new ConnectionFactoryAdapter() {
- @Override
- public Connection createConnection() throws SQLException {
- return factory.getConnection(new ConnectionInfo("jdbc:drill:zk=local", ModelAndSchema.this.info));
- }
- };
- }
-
- public TestDataConnection sql(String sql) {
- return new TestDataConnection(adapter, sql);
- }
-
- public <T> T withConnection(Function<Connection, T> function) throws Exception {
- Connection connection = null;
- try {
- connection = adapter.createConnection();
- return function.apply(connection);
- } finally {
- if (connection != null) {
- connection.close();
- }
- }
- }
- }
-
- public static class TestDataConnection {
- private final ConnectionFactoryAdapter adapter;
- private final String sql;
-
- TestDataConnection(ConnectionFactoryAdapter adapter, String sql) {
- this.adapter = adapter;
- this.sql = sql;
- }
-
- /**
- * Checks that the current SQL statement returns the expected result.
- */
- public TestDataConnection returns(String expected) throws Exception {
- Connection connection = null;
- Statement statement = null;
- try {
- connection = adapter.createConnection();
- statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery(sql);
- expected = expected.trim();
- String result = JdbcAssert.toString(resultSet).trim();
- resultSet.close();
-
- if (!expected.equals(result)) {
- Assert.fail(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected));
- }
- return this;
- } finally {
- if (statement != null) {
- statement.close();
- }
- if (connection != null) {
- connection.close();
- }
- }
- }
-
- public TestDataConnection returnsSet(Set<String> expected) throws Exception {
- Connection connection = null;
- Statement statement = null;
- try {
- connection = adapter.createConnection();
- statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery(sql);
- Set<String> result = JdbcAssert.toStringSet(resultSet);
- resultSet.close();
-
- if (!expected.equals(result)) {
- Assert.fail(String.format("Generated set:\n%s\ndoes not match:\n%s", result, expected));
- }
- return this;
- } finally {
- if (statement != null) {
- statement.close();
- }
- if (connection != null) {
- connection.close();
- }
- }
- }
-
- /**
- * Checks that the current SQL statement returns the expected result lines. Lines are compared unordered; the test
- * succeeds if the query returns these lines in any order.
- */
- public TestDataConnection returnsUnordered(String... expecteds) throws Exception {
- Connection connection = null;
- Statement statement = null;
- try {
- connection = adapter.createConnection();
- statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery(sql);
- Assert.assertEquals(unsortedList(Arrays.asList(expecteds)), unsortedList(JdbcAssert.toStrings(resultSet)));
- resultSet.close();
- return this;
- } finally {
- if (statement != null) {
- statement.close();
- }
- if (connection != null) {
- connection.close();
- }
- }
- }
-
- public TestDataConnection displayResults(int recordCount) throws Exception {
- // record count check is done in toString method
-
- Connection connection = null;
- Statement statement = null;
- try {
- connection = adapter.createConnection();
- statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery(sql);
- System.out.println(JdbcAssert.toString(resultSet, recordCount));
- resultSet.close();
- return this;
- } finally {
- if (statement != null) {
- statement.close();
- }
- if (connection != null) {
- connection.close();
- }
- }
- }
-
- private SortedSet<String> unsortedList(List<String> strings) {
- final SortedSet<String> set = new TreeSet<>();
- for (String string : strings) {
- set.add(string + "\n");
- }
- return set;
- }
-
- public LogicalPlan logicalPlan() {
- final String[] plan0 = {null};
- Connection connection = null;
- Statement statement = null;
- final Hook.Closeable x = Hook.LOGICAL_PLAN.add(new Function<String, Void>() {
- @Override
- public Void apply(String o) {
- plan0[0] = o;
- return null;
- }
- });
- try {
- connection = adapter.createConnection();
- statement = connection.prepareStatement(sql);
- statement.close();
- final String plan = plan0[0].trim();
- return LogicalPlan.parse(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(DrillConfig.create()), plan);
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- if (statement != null) {
- try {
- statement.close();
- } catch (SQLException e) {
- // ignore
- }
- }
- if (connection != null) {
- try {
- connection.close();
- } catch (SQLException e) {
- // ignore
- }
- }
- x.close();
- }
- }
-
- public <T extends LogicalOperator> T planContains(final Class<T> operatorClazz) {
- return (T) Iterables.find(logicalPlan().getSortedOperators(), new Predicate<LogicalOperator>() {
- @Override
- public boolean apply(LogicalOperator input) {
- return input.getClass().equals(operatorClazz);
- }
- });
- }
- }
-
- private static interface ConnectionFactoryAdapter {
- Connection createConnection() throws Exception;
- }
-}
-
-// End JdbcAssert.java
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcConnectTriesTestEmbeddedBits.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcConnectTriesTestEmbeddedBits.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcConnectTriesTestEmbeddedBits.java
index 44b2e22..d7edd1e 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcConnectTriesTestEmbeddedBits.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcConnectTriesTestEmbeddedBits.java
@@ -53,7 +53,7 @@ public class JdbcConnectTriesTestEmbeddedBits extends JdbcTestBase {
Connection connection = null;
try {
connection = testDrillDriver.connect("jdbc:drill:drillbit=127.0.0.1:5000,127.0.0.1:5001;" + "tries=2",
- JdbcAssert.getDefaultProperties());
+ getDefaultProperties());
fail();
} catch (SQLException ex) {
assertNull(connection);
@@ -67,7 +67,7 @@ public class JdbcConnectTriesTestEmbeddedBits extends JdbcTestBase {
Connection connection = null;
try {
connection = testDrillDriver.connect("jdbc:drill:drillbit=127.0.0.1:5000,127.0.0.1:5001;tries=5",
- JdbcAssert.getDefaultProperties());
+ getDefaultProperties());
fail();
} catch (SQLException ex) {
assertNull(connection);
@@ -81,7 +81,7 @@ public class JdbcConnectTriesTestEmbeddedBits extends JdbcTestBase {
Connection connection = null;
try {
connection = testDrillDriver.connect("jdbc:drill:drillbit=127.0.0.1:5000,127.0.0.1:5001;tries=1",
- JdbcAssert.getDefaultProperties());
+ getDefaultProperties());
fail();
} catch (SQLException ex) {
assertNull(connection);
@@ -95,7 +95,7 @@ public class JdbcConnectTriesTestEmbeddedBits extends JdbcTestBase {
Connection connection = null;
try {
connection = testDrillDriver.connect("jdbc:drill:drillbit=127.0.0.1:5000,127.0.0.1:5001;tries=abc",
- JdbcAssert.getDefaultProperties());
+ getDefaultProperties());
fail();
} catch (SQLException ex) {
assertNull(connection);
@@ -108,7 +108,7 @@ public class JdbcConnectTriesTestEmbeddedBits extends JdbcTestBase {
Connection connection = null;
try {
connection = testDrillDriver.connect("jdbc:drill:drillbit=127.0.0.1:5000,127.0.0.1:5001;tries=0",
- JdbcAssert.getDefaultProperties());
+ getDefaultProperties());
fail();
} catch (SQLException ex) {
assertNull(connection);
@@ -122,7 +122,7 @@ public class JdbcConnectTriesTestEmbeddedBits extends JdbcTestBase {
Connection connection = null;
try {
connection = testDrillDriver.connect("jdbc:drill:drillbit=127.0.0.1:5000,127.0.0.1:5001;tries=-5",
- JdbcAssert.getDefaultProperties());
+ getDefaultProperties());
fail();
} catch (SQLException ex) {
assertNull(connection);
@@ -133,21 +133,21 @@ public class JdbcConnectTriesTestEmbeddedBits extends JdbcTestBase {
@Test
public void testZKSuccessfulConnectionZeroConnectTries() throws SQLException {
- Connection connection = testDrillDriver.connect("jdbc:drill:zk=local;tries=0", JdbcAssert.getDefaultProperties());
+ Connection connection = testDrillDriver.connect("jdbc:drill:zk=local;tries=0", getDefaultProperties());
assertNotNull(connection);
connection.close();
}
@Test
public void testZKSuccessfulConnectionNegativeConnectTries() throws SQLException {
- Connection connection = testDrillDriver.connect("jdbc:drill:zk=local;tries=-1", JdbcAssert.getDefaultProperties());
+ Connection connection = testDrillDriver.connect("jdbc:drill:zk=local;tries=-1", getDefaultProperties());
assertNotNull(connection);
connection.close();
}
@Test
public void testZKSuccessfulConnectionGreaterThanConnectTries() throws SQLException {
- Connection connection = testDrillDriver.connect("jdbc:drill:zk=local;tries=7", JdbcAssert.getDefaultProperties());
+ Connection connection = testDrillDriver.connect("jdbc:drill:zk=local;tries=7", getDefaultProperties());
assertNotNull(connection);
connection.close();
}
@@ -156,7 +156,7 @@ public class JdbcConnectTriesTestEmbeddedBits extends JdbcTestBase {
public void testZKConnectionInvalidConnectTries() throws SQLException {
Connection connection = null;
try {
- connection = testDrillDriver.connect("jdbc:drill:zk=local;tries=abc", JdbcAssert.getDefaultProperties());
+ connection = testDrillDriver.connect("jdbc:drill:zk=local;tries=abc", getDefaultProperties());
fail();
} catch (SQLException ex) {
assertNull(connection);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcDataTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcDataTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcDataTest.java
index ad532cf..e4d97af 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcDataTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcDataTest.java
@@ -40,7 +40,6 @@ import org.apache.drill.common.logical.data.Scan;
import org.apache.drill.common.logical.data.Store;
import org.apache.drill.common.logical.data.Union;
import org.apache.drill.jdbc.JdbcTestBase;
-import org.apache.drill.jdbc.test.JdbcAssert.TestDataConnection;
import org.apache.drill.categories.JdbcTest;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -112,7 +111,7 @@ public class JdbcDataTest extends JdbcTestBase {
/** Load driver, make a connection, prepare a statement. */
@Test
public void testPrepare() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").withConnection(new Function<Connection, Void>() {
+ withModel(MODEL, "DONUTS").withConnection(new Function<Connection, Void>() {
@Override
public Void apply(Connection connection) {
try {
@@ -129,14 +128,13 @@ public class JdbcDataTest extends JdbcTestBase {
/** Simple query against JSON. */
@Test
public void testSelectJson() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").sql("select * from donuts").returns(EXPECTED);
+ withModel(MODEL, "DONUTS").sql("select * from donuts").returns(EXPECTED);
}
/** Simple query against EMP table in HR database. */
@Test
public void testSelectEmployees() throws Exception {
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select * from employees")
.returns(
"_MAP={deptId=31, lastName=Rafferty}\n" + "_MAP={deptId=33, lastName=Jones}\n"
@@ -147,8 +145,7 @@ public class JdbcDataTest extends JdbcTestBase {
/** Simple query against EMP table in HR database. */
@Test
public void testSelectEmpView() throws Exception {
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select * from emp")
.returns(
"DEPTID=31; LASTNAME=Rafferty\n" + "DEPTID=33; LASTNAME=Jones\n" + "DEPTID=33; LASTNAME=Steinberg\n"
@@ -158,8 +155,7 @@ public class JdbcDataTest extends JdbcTestBase {
/** Simple query against EMP table in HR database. */
@Test
public void testSelectDept() throws Exception {
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select * from departments")
.returns(
"_MAP={deptId=31, name=Sales}\n" + "_MAP={deptId=33, name=Engineering}\n"
@@ -169,29 +165,28 @@ public class JdbcDataTest extends JdbcTestBase {
/** Query with project list. No field references yet. */
@Test
public void testProjectConstant() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").sql("select 1 + 3 as c from donuts")
+ withModel(MODEL, "DONUTS").sql("select 1 + 3 as c from donuts")
.returns("C=4\n" + "C=4\n" + "C=4\n" + "C=4\n" + "C=4\n");
}
/** Query that projects an element from the map. */
@Test
public void testProject() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").sql("select _MAP['ppu'] as ppu from donuts")
+ withModel(MODEL, "DONUTS").sql("select _MAP['ppu'] as ppu from donuts")
.returns("PPU=0.55\n" + "PPU=0.69\n" + "PPU=0.55\n" + "PPU=0.69\n" + "PPU=1.0\n");
}
/** Same logic as {@link #testProject()}, but using a subquery. */
@Test
public void testProjectOnSubquery() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").sql("select d['ppu'] as ppu from (\n" + " select _MAP as d from donuts)")
+ withModel(MODEL, "DONUTS").sql("select d['ppu'] as ppu from (\n" + " select _MAP as d from donuts)")
.returns("PPU=0.55\n" + "PPU=0.69\n" + "PPU=0.55\n" + "PPU=0.69\n" + "PPU=1.0\n");
}
/** Checks the logical plan. */
@Test
public void testProjectPlan() throws Exception {
- LogicalPlan plan = JdbcAssert
- .withModel(MODEL, "DONUTS")
+ LogicalPlan plan = withModel(MODEL, "DONUTS")
.sql("select _MAP['ppu'] as ppu from donuts")
.logicalPlan();
@@ -221,8 +216,7 @@ public class JdbcDataTest extends JdbcTestBase {
*/
@Test
public void testProjectFilterSubquery() throws Exception {
- JdbcAssert
- .withModel(MODEL, "DONUTS")
+ withModel(MODEL, "DONUTS")
.sql(
"select d['name'] as name, d['xx'] as xx from (\n" + " select _MAP as d from donuts)\n"
+ "where cast(d['ppu'] as double) > 0.6")
@@ -244,8 +238,7 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testProjectFilterSubqueryPlan() throws Exception {
- LogicalPlan plan = JdbcAssert
- .withModel(MODEL, "DONUTS")
+ LogicalPlan plan = withModel(MODEL, "DONUTS")
.sql(
"select d['name'] as name, d['xx'] as xx from (\n" + " select _MAP['donuts'] as d from donuts)\n"
+ "where cast(d['ppu'] as double) > 0.6")
@@ -279,74 +272,34 @@ public class JdbcDataTest extends JdbcTestBase {
/** Query that projects one field. (Disabled; uses sugared syntax.) */
@Test @Ignore
public void testProjectNestedFieldSugared() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").sql("select donuts.ppu from donuts")
+ withModel(MODEL, "DONUTS").sql("select donuts.ppu from donuts")
.returns("C=4\n" + "C=4\n" + "C=4\n" + "C=4\n" + "C=4\n");
}
/** Query with filter. No field references yet. */
@Test
public void testFilterConstantFalse() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").sql("select * from donuts where 3 > 4").returns("");
+ withModel(MODEL, "DONUTS").sql("select * from donuts where 3 > 4").returns("");
}
@Test
public void testFilterConstant() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").sql("select * from donuts where 3 < 4").returns(EXPECTED);
+ withModel(MODEL, "DONUTS").sql("select * from donuts where 3 < 4").returns(EXPECTED);
}
@Ignore
@Test
public void testValues() throws Exception {
- JdbcAssert.withModel(MODEL, "DONUTS").sql("values (1)").returns("EXPR$0=1\n");
+ withModel(MODEL, "DONUTS").sql("values (1)").returns("EXPR$0=1\n");
// Enable when https://issues.apache.org/jira/browse/DRILL-57 fixed
// .planContains("store");
}
-// @Test
-// public void testDistinct() throws Exception {
-// JdbcAssert.withModel(MODEL, "HR").sql("select distinct deptId from emp")
-// .returnsUnordered("DEPTID=null", "DEPTID=31", "DEPTID=34", "DEPTID=33")
-// .planContains(CollapsingAggregate.class);
-// }
-//
-// @Test
-// public void testCountNoGroupBy() throws Exception {
-// // 5 out of 6 employees have a not-null deptId
-// JdbcAssert.withModel(MODEL, "HR").sql("select count(deptId) as cd, count(*) as c from emp").returns("CD=5; C=6\n")
-// .planContains(CollapsingAggregate.class);
-// }
-//
-// @Test
-// public void testDistinctCountNoGroupBy() throws Exception {
-// JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp").returns("C=3\n")
-// .planContains(CollapsingAggregate.class);
-// }
-//
-// @Test
-// public void testDistinctCountGroupByEmpty() throws Exception {
-// JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
-// .planContains(CollapsingAggregate.class);
-// }
-//
-// @Test
-// public void testCountNull() throws Exception {
-// JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
-// .planContains(CollapsingAggregate.class);
-// }
-//
-// @Test
-// public void testCount() throws Exception {
-// JdbcAssert.withModel(MODEL, "HR").sql("select deptId, count(*) as c from emp group by deptId")
-// .returnsUnordered("DEPTID=31; C=1", "DEPTID=33; C=2", "DEPTID=34; C=2", "DEPTID=null; C=1")
-// .planContains(CollapsingAggregate.class); // make sure using drill
-// }
-
@Test
public void testJoin() throws Exception {
- Join join = JdbcAssert
- .withModel(MODEL, "HR")
+ Join join = withModel(MODEL, "HR")
.sql("select * from emp join dept on emp.deptId = dept.deptId")
.returnsUnordered("DEPTID=31; LASTNAME=Rafferty; DEPTID0=31; NAME=Sales",
"DEPTID=33; LASTNAME=Jones; DEPTID0=33; NAME=Engineering",
@@ -358,8 +311,7 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testLeftJoin() throws Exception {
- Join join = JdbcAssert
- .withModel(MODEL, "HR")
+ Join join = withModel(MODEL, "HR")
.sql("select * from emp left join dept on emp.deptId = dept.deptId")
.returnsUnordered("DEPTID=31; LASTNAME=Rafferty; DEPTID0=31; NAME=Sales",
"DEPTID=33; LASTNAME=Jones; DEPTID0=33; NAME=Engineering",
@@ -375,15 +327,14 @@ public class JdbcDataTest extends JdbcTestBase {
*/
@Test @Ignore
public void testRightJoin() throws Exception {
- Join join = JdbcAssert.withModel(MODEL, "HR").sql("select * from emp right join dept on emp.deptId = dept.deptId")
+ Join join = withModel(MODEL, "HR").sql("select * from emp right join dept on emp.deptId = dept.deptId")
.returnsUnordered("xx").planContains(Join.class);
Assert.assertEquals(JoinRelType.LEFT, join.getJoinType());
}
@Test
public void testFullJoin() throws Exception {
- Join join = JdbcAssert
- .withModel(MODEL, "HR")
+ Join join = withModel(MODEL, "HR")
.sql("select * from emp full join dept on emp.deptId = dept.deptId")
.returnsUnordered("DEPTID=31; LASTNAME=Rafferty; DEPTID0=31; NAME=Sales",
"DEPTID=33; LASTNAME=Jones; DEPTID0=33; NAME=Engineering",
@@ -401,8 +352,7 @@ public class JdbcDataTest extends JdbcTestBase {
*/
@Test
public void testJoinOnSubquery() throws Exception {
- Join join = JdbcAssert
- .withModel(MODEL, "HR")
+ Join join = withModel(MODEL, "HR")
.sql(
"select * from (\n" + "select deptId, lastname, 'x' as name from emp) as e\n"
+ " join dept on e.deptId = dept.deptId")
@@ -417,8 +367,7 @@ public class JdbcDataTest extends JdbcTestBase {
/** Tests that one of the FoodMart tables is present. */
@Test @Ignore
public void testFoodMart() throws Exception {
- JdbcAssert
- .withModel(MODEL, "FOODMART")
+ withModel(MODEL, "FOODMART")
.sql("select * from product_class where cast(_map['product_class_id'] as integer) < 3")
.returnsUnordered(
"_MAP={product_category=Seafood, product_class_id=2, product_department=Seafood, product_family=Food, product_subcategory=Shellfish}",
@@ -427,25 +376,26 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testUnionAll() throws Exception {
- Union union = JdbcAssert.withModel(MODEL, "HR").sql("select deptId from dept\n" + "union all\n" + "select deptId from emp")
- .returnsUnordered("DEPTID=31", "DEPTID=33", "DEPTID=34", "DEPTID=35", "DEPTID=null")
- .planContains(Union.class);
+ Union union = withModel(MODEL, "HR")
+ .sql("select deptId from dept\n" + "union all\n" + "select deptId from emp")
+ .returnsUnordered("DEPTID=31", "DEPTID=33", "DEPTID=34", "DEPTID=35", "DEPTID=null")
+ .planContains(Union.class);
Assert.assertFalse(union.isDistinct());
}
@Test
public void testUnion() throws Exception {
- Union union = JdbcAssert.withModel(MODEL, "HR").sql("select deptId from dept\n" + "union\n" + "select deptId from emp")
- .returnsUnordered("DEPTID=31", "DEPTID=33", "DEPTID=34", "DEPTID=35", "DEPTID=null")
- .planContains(Union.class);
+ Union union = withModel(MODEL, "HR")
+ .sql("select deptId from dept\n" + "union\n" + "select deptId from emp")
+ .returnsUnordered("DEPTID=31", "DEPTID=33", "DEPTID=34", "DEPTID=35", "DEPTID=null")
+ .planContains(Union.class);
Assert.assertTrue(union.isDistinct());
}
@Test
public void testOrderByDescNullsFirst() throws Exception {
// desc nulls last
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select * from emp order by deptId desc nulls first")
.returns(
"DEPTID=null; LASTNAME=John\n" + "DEPTID=34; LASTNAME=Robinson\n" + "DEPTID=34; LASTNAME=Smith\n"
@@ -456,8 +406,7 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testOrderByDescNullsLast() throws Exception {
// desc nulls first
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select * from emp order by deptId desc nulls last")
.returns(
"DEPTID=34; LASTNAME=Robinson\n" + "DEPTID=34; LASTNAME=Smith\n" + "DEPTID=33; LASTNAME=Jones\n"
@@ -469,8 +418,7 @@ public class JdbcDataTest extends JdbcTestBase {
public void testOrderByDesc() throws Exception {
// desc is implicitly "nulls first" (i.e. null sorted as +inf)
// Current behavior is to sort nulls last. This is wrong.
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select * from emp order by deptId desc")
.returns(
"DEPTID=null; LASTNAME=John\n" + "DEPTID=34; LASTNAME=Robinson\n" + "DEPTID=34; LASTNAME=Smith\n"
@@ -481,8 +429,7 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testOrderBy() throws Exception {
// no sort order specified is implicitly "asc", and asc is "nulls last"
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select * from emp order by deptId")
.returns(
"DEPTID=31; LASTNAME=Rafferty\n"
@@ -496,8 +443,7 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testLimit() throws Exception {
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select LASTNAME from emp limit 2")
.returns("LASTNAME=Rafferty\n" +
"LASTNAME=Jones")
@@ -507,8 +453,7 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testLimitOrderBy() throws Exception {
- TestDataConnection tdc = JdbcAssert
- .withModel(MODEL, "HR")
+ TestDataConnection tdc = withModel(MODEL, "HR")
.sql("select LASTNAME from emp order by LASTNAME limit 2")
.returns("LASTNAME=John\n" +
"LASTNAME=Jones");
@@ -519,8 +464,7 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testOrderByWithOffset() throws Exception {
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select LASTNAME from emp order by LASTNAME asc offset 3")
.returns("LASTNAME=Robinson\n" +
"LASTNAME=Smith\n" +
@@ -531,13 +475,10 @@ public class JdbcDataTest extends JdbcTestBase {
@Test
public void testOrderByWithOffsetAndFetch() throws Exception {
- JdbcAssert
- .withModel(MODEL, "HR")
+ withModel(MODEL, "HR")
.sql("select LASTNAME from emp order by LASTNAME asc offset 3 fetch next 2 rows only")
.returns("LASTNAME=Robinson\n" +
"LASTNAME=Smith")
.planContains(Limit.class);
}
}
-
-// End JdbcTest.java
[04/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java
new file mode 100644
index 0000000..7987647
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java
@@ -0,0 +1,695 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.drill.test.DrillTestWrapper.TestServices;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.parser.ExprLexer;
+import org.apache.drill.common.expression.parser.ExprParser;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryType;
+import org.apache.drill.exec.proto.UserProtos.PreparedStatementHandle;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.util.JsonStringHashMap;
+import org.apache.drill.exec.util.Text;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import org.joda.time.DateTimeZone;
+
+public class TestBuilder {
+
+ /**
+ * Test query to rung. Type of object depends on the {@link #queryType}
+ */
+ private Object query;
+ // the type of query for the test
+ private UserBitShared.QueryType queryType;
+ // should the validation enforce ordering
+ private Boolean ordered;
+ private boolean approximateEquality;
+ private TestServices services;
+ // Used to pass the type information associated with particular column names rather than relying on the
+ // ordering of the columns in the CSV file, or the default type inferences when reading JSON, this is used for the
+ // case where results of the test query are adding type casts to the baseline queries, this saves a little bit of
+ // setup in cases where strict type enforcement is not necessary for a given test
+ protected Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap;
+ // queries to run before the baseline or test queries, can be used to set options
+ private String baselineOptionSettingQueries;
+ private String testOptionSettingQueries;
+ // two different methods are available for comparing ordered results, the default reads all of the records
+ // into giant lists of objects, like one giant on-heap batch of 'vectors'
+ // this flag enables the other approach which iterates through a hyper batch for the test query results and baseline
+ // while this does work faster and use less memory, it can be harder to debug as all of the elements are not in a
+ // single list
+ private boolean highPerformanceComparison;
+ // column names for use with the baseline values
+ protected String[] baselineColumns;
+ // In cases where we need to verify larger datasets without the risk of running the baseline data through
+ // the drill engine, results can be provided in a list of maps. While this model does make a lot of sense, there is a
+ // lot of work to make the type handling/casting work correctly, and making robust complex type handling work completely outside
+ // of the drill engine for generating baselines would likely be more work than it would be worth. For now we will be
+ // going with an approach of using this facility to validate the parts of the drill engine that could break in ways
+ // that would affect the reading of baseline files (i.e. we need robust test for storage engines, project and casting that
+ // use this interface) and then rely on the engine for the rest of the tests that will use the baseline queries.
+ private List<Map<String, Object>> baselineRecords;
+
+ private int expectedNumBatches = DrillTestWrapper.EXPECTED_BATCH_COUNT_NOT_SET;
+
+ public TestBuilder(TestServices services) {
+ this.services = services;
+ reset();
+ }
+
+ public TestBuilder(TestServices services, Object query, UserBitShared.QueryType queryType, Boolean ordered,
+ boolean approximateEquality, Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap,
+ String baselineOptionSettingQueries, String testOptionSettingQueries, boolean highPerformanceComparison,
+ int expectedNumBatches) {
+ this(services);
+ if (ordered == null) {
+ throw new RuntimeException("Ordering not set, when using a baseline file or query you must explicitly call the ordered() or unOrdered() method on the " + this.getClass().getSimpleName());
+ }
+ this.query = query;
+ this.queryType = queryType;
+ this.ordered = ordered;
+ this.approximateEquality = approximateEquality;
+ this.baselineTypeMap = baselineTypeMap;
+ this.baselineOptionSettingQueries = baselineOptionSettingQueries;
+ this.testOptionSettingQueries = testOptionSettingQueries;
+ this.highPerformanceComparison = highPerformanceComparison;
+ this.expectedNumBatches = expectedNumBatches;
+ }
+
+ protected TestBuilder reset() {
+ query = "";
+ ordered = null;
+ approximateEquality = false;
+ highPerformanceComparison = false;
+ testOptionSettingQueries = "";
+ baselineOptionSettingQueries = "";
+ baselineRecords = null;
+ return this;
+ }
+
+ public DrillTestWrapper build() throws Exception {
+ if ( ! ordered && highPerformanceComparison ) {
+ throw new Exception("High performance comparison only available for ordered checks, to enforce this restriction, ordered() must be called first.");
+ }
+ return new DrillTestWrapper(this, services, query, queryType, baselineOptionSettingQueries, testOptionSettingQueries,
+ getValidationQueryType(), ordered, highPerformanceComparison, baselineColumns, baselineRecords, expectedNumBatches);
+ }
+
+ public List<Pair<SchemaPath, TypeProtos.MajorType>> getExpectedSchema() {
+ return null;
+ }
+
+ public void go() throws Exception {
+ build().run();
+ }
+
+ public TestBuilder sqlQuery(String query) {
+ this.query = QueryTestUtil.normalizeQuery(query);
+ this.queryType = UserBitShared.QueryType.SQL;
+ return this;
+ }
+
+ public TestBuilder sqlQuery(String query, Object... replacements) {
+ return sqlQuery(String.format(query, replacements));
+ }
+
+ public TestBuilder preparedStatement(PreparedStatementHandle preparedStatementHandle) {
+ queryType = QueryType.PREPARED_STATEMENT;
+ query = preparedStatementHandle;
+ return this;
+ }
+
+ public TestBuilder sqlQueryFromFile(String queryFile) throws IOException {
+ String query = BaseTestQuery.getFile(queryFile);
+ this.query = query;
+ queryType = UserBitShared.QueryType.SQL;
+ return this;
+ }
+
+ public TestBuilder physicalPlanFromFile(String queryFile) throws IOException {
+ String query = BaseTestQuery.getFile(queryFile);
+ this.query = query;
+ queryType = UserBitShared.QueryType.PHYSICAL;
+ return this;
+ }
+
+ public TestBuilder ordered() {
+ ordered = true;
+ return this;
+ }
+
+ public TestBuilder unOrdered() {
+ ordered = false;
+ return this;
+ }
+
+ // this can only be used with ordered verifications, it does run faster and use less memory but may be
+ // a little harder to debug as it iterates over a hyper batch rather than reading all of the values into
+ // large on-heap lists
+ public TestBuilder highPerformanceComparison() throws Exception {
+ highPerformanceComparison = true;
+ return this;
+ }
+
+ // list of queries to run before the baseline query, can be used to set several options
+ // list takes the form of a semi-colon separated list
+ public TestBuilder optionSettingQueriesForBaseline(String queries) {
+ baselineOptionSettingQueries = queries;
+ return this;
+ }
+
+ public TestBuilder optionSettingQueriesForBaseline(String queries, Object... args) {
+ baselineOptionSettingQueries = String.format(queries, args);
+ return this;
+ }
+
+ /**
+ * list of queries to run before the test query, can be used to set several options
+ * list takes the form of a semi-colon separated list.
+ * @param queries queries that set session and system options
+ * @return this test builder
+ */
+
+ public TestBuilder optionSettingQueriesForTestQuery(String queries) {
+ testOptionSettingQueries = queries;
+ return this;
+ }
+
+ public TestBuilder optionSettingQueriesForTestQuery(String query, Object... args) throws Exception {
+ testOptionSettingQueries = String.format(query, args);
+ return this;
+ }
+
+ public TestBuilder approximateEquality() {
+ approximateEquality = true;
+ return this;
+ }
+
+ // modified code from SchemaPath.De class. This should be used sparingly and only in tests if absolutely needed.
+ public static SchemaPath parsePath(String path) {
+ try {
+ ExprLexer lexer = new ExprLexer(new ANTLRStringStream(path));
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+ ExprParser parser = new ExprParser(tokens);
+
+ ExprParser.parse_return ret = parser.parse();
+
+ if (ret.e instanceof SchemaPath) {
+ return (SchemaPath) ret.e;
+ } else {
+ throw new IllegalStateException("Schema path is not a valid format.");
+ }
+ } catch (RecognitionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ Object getValidationQuery() throws Exception {
+ throw new RuntimeException("Must provide some kind of baseline, either a baseline file or another query");
+ }
+
+ protected UserBitShared.QueryType getValidationQueryType() throws Exception {
+ if (singleExplicitBaselineRecord()) {
+ return null;
+ }
+
+ if (ordered) {
+ // If there are no base line records or no baseline query then we will check to make sure that the records are in ascending order
+ return null;
+ }
+
+ throw new RuntimeException("Must provide some kind of baseline, either a baseline file or another query");
+ }
+
+ public JSONTestBuilder jsonBaselineFile(String filePath) {
+ return new JSONTestBuilder(filePath, services, query, queryType, ordered, approximateEquality,
+ baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison,
+ expectedNumBatches);
+ }
+
+ public CSVTestBuilder csvBaselineFile(String filePath) {
+ return new CSVTestBuilder(filePath, services, query, queryType, ordered, approximateEquality,
+ baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison,
+ expectedNumBatches);
+ }
+
+ public SchemaTestBuilder schemaBaseLine(BatchSchema batchSchema) {
+ List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = new ArrayList<>();
+ for (final MaterializedField field : batchSchema) {
+ expectedSchema.add(Pair.of(SchemaPath.getSimplePath(field.getName()), field.getType()));
+ }
+ return schemaBaseLine(expectedSchema);
+ }
+
+ public SchemaTestBuilder schemaBaseLine(List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema) {
+ assert expectedSchema != null : "The expected schema can be provided once";
+ assert baselineColumns == null : "The column information should be captured in expected schema, not baselineColumns";
+
+ return new SchemaTestBuilder(
+ services,
+ query,
+ queryType,
+ baselineOptionSettingQueries,
+ testOptionSettingQueries,
+ expectedSchema);
+ }
+
+ public TestBuilder baselineTypes(Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap) {
+ this.baselineTypeMap = baselineTypeMap;
+ return this;
+ }
+
+ boolean typeInfoSet() {
+ if (baselineTypeMap != null) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Indicate that the tests query should be checked for an empty result set.
+ * @return the test builder
+ */
+ public TestBuilder expectsEmptyResultSet() {
+ unOrdered();
+ baselineRecords = new ArrayList<>();
+ return this;
+ }
+
+ /**
+ * Sets the number of expected batch count for this query. The test will fail if the query returns a different number
+ * of batches
+ *
+ * @param expectedNumBatches expected batch count
+ * @return this test builder
+ */
+ public TestBuilder expectsNumBatches(int expectedNumBatches) {
+ this.expectedNumBatches = expectedNumBatches;
+ return this;
+ }
+
+ /**
+ * This method is used to pass in a simple list of values for a single record verification without
+ * the need to create a CSV or JSON file to store the baseline.
+ *
+ * This can be called repeatedly to pass a list of records to verify. It works for both ordered and unordered
+ * checks.
+ *
+ * @param baselineValues - the baseline values to validate
+ * @return the test builder
+ */
+ public TestBuilder baselineValues(Object ... baselineValues) {
+ assert getExpectedSchema() == null : "The expected schema is not needed when baselineValues are provided ";
+ if (ordered == null) {
+ throw new RuntimeException("Ordering not set, before specifying baseline data you must explicitly call the ordered() or unOrdered() method on the " + this.getClass().getSimpleName());
+ }
+ if (baselineRecords == null) {
+ baselineRecords = new ArrayList<>();
+ }
+ Map<String, Object> ret = new HashMap<>();
+ int i = 0;
+ assertEquals("Must supply the same number of baseline values as columns.", baselineValues.length, baselineColumns.length);
+ for (String s : baselineColumns) {
+ ret.put(s, baselineValues[i]);
+ i++;
+ }
+ this.baselineRecords.add(ret);
+ return this;
+ }
+
+ /**
+ * This can be used in cases where we want to avoid issues with the assumptions made by the test framework.
+ * Most of the methods for verification in the framework run drill queries to generate the read baseline files or
+ * execute alternative baseline queries. This model relies on basic functionality of reading files with storage
+ * plugins and applying casts/projects to be stable.
+ *
+ * This method can be used to verify the engine for these cases and any other future execution paths that would
+ * be used by both the test query and baseline. Without tests like this it is possible that some tests
+ * could falsely report as passing, as both the test query and baseline query could run into the same problem
+ * with an assumed stable code path and produce the same erroneous result.
+ *
+ * @param materializedRecords - a list of maps representing materialized results
+ * @return the test builder
+ */
+ public TestBuilder baselineRecords(List<Map<String, Object>> materializedRecords) {
+ this.baselineRecords = materializedRecords;
+ return this;
+ }
+
+ /**
+ * This setting has a slightly different impact on the test depending on some of the other
+ * configuration options are set.
+ *
+ * If a JSON baseline file is given, this list will act as a project list to verify the
+ * test query against a subset of the columns in the file.
+ *
+ * For a CSV baseline file, these will act as aliases for columns [0 .. n] in the repeated
+ * varchar column that is read out of CSV.
+ *
+ * For a baseline sql query, this currently has no effect.
+ *
+ * For explicit baseline values given in java code with the baselineValues() method, these will
+ * be used to create a map for the one record verification.
+ */
+ public TestBuilder baselineColumns(String... columns) {
+ assert getExpectedSchema() == null : "The expected schema is not needed when baselineColumns are provided ";
+ for (int i = 0; i < columns.length; i++) {
+ columns[i] = parsePath(columns[i]).toExpr();
+ }
+ this.baselineColumns = columns;
+ return this;
+ }
+
+ private boolean singleExplicitBaselineRecord() {
+ return baselineRecords != null;
+ }
+
+ /**
+ * Provide a SQL query to validate against.
+ * @param baselineQuery
+ * @return the test builder
+ */
+ public BaselineQueryTestBuilder sqlBaselineQuery(Object baselineQuery) {
+ return new BaselineQueryTestBuilder(baselineQuery, UserBitShared.QueryType.SQL, services, query, queryType, ordered, approximateEquality,
+ baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches);
+ }
+
+ public BaselineQueryTestBuilder sqlBaselineQuery(String query, String ...replacements) {
+ return sqlBaselineQuery(String.format(query, (Object[]) replacements));
+ }
+
+ // provide a path to a file containing a SQL query to use as a baseline
+ public BaselineQueryTestBuilder sqlBaselineQueryFromFile(String baselineQueryFilename) throws IOException {
+ String baselineQuery = BaseTestQuery.getFile(baselineQueryFilename);
+ return new BaselineQueryTestBuilder(baselineQuery, UserBitShared.QueryType.SQL, services, query, queryType, ordered, approximateEquality,
+ baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches);
+ }
+
+ // as physical plans are verbose, this is the only option provided for specifying them, we should enforce
+ // that physical plans, or any large JSON strings do not live in the Java source as literals
+ public BaselineQueryTestBuilder physicalPlanBaselineQueryFromFile(String baselinePhysicalPlanPath) throws IOException {
+ String baselineQuery = BaseTestQuery.getFile(baselinePhysicalPlanPath);
+ return new BaselineQueryTestBuilder(baselineQuery, UserBitShared.QueryType.PHYSICAL, services, query, queryType, ordered, approximateEquality,
+ baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches);
+ }
+
+ private String getDecimalPrecisionScaleInfo(TypeProtos.MajorType type) {
+ String precision = "";
+ switch(type.getMinorType()) {
+ case DECIMAL18:
+ case DECIMAL28SPARSE:
+ case DECIMAL38SPARSE:
+ case DECIMAL38DENSE:
+ case DECIMAL28DENSE:
+ case DECIMAL9:
+ precision = String.format("(%d,%d)", type.getPrecision(), type.getScale());
+ break;
+ default:
+ ; // do nothing empty string set above
+ }
+ return precision;
+ }
+
+ public class CSVTestBuilder extends TestBuilder {
+
+ // path to the baseline file that will be inserted into the validation query
+ private String baselineFilePath;
+ // use to cast the baseline file columns, if not set the types
+ // that come out of the test query drive interpretation of baseline
+ private TypeProtos.MajorType[] baselineTypes;
+
+ CSVTestBuilder(String baselineFile, TestServices services, Object query, UserBitShared.QueryType queryType, Boolean ordered,
+ boolean approximateEquality, Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap,
+ String baselineOptionSettingQueries, String testOptionSettingQueries, boolean highPerformanceComparison,
+ int expectedNumBatches) {
+ super(services, query, queryType, ordered, approximateEquality, baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries,
+ highPerformanceComparison, expectedNumBatches);
+ this.baselineFilePath = baselineFile;
+ }
+
+ public CSVTestBuilder baselineTypes(TypeProtos.MajorType... baselineTypes) {
+ this.baselineTypes = baselineTypes;
+ this.baselineTypeMap = null;
+ return this;
+ }
+
+ // convenience method to convert minor types to major types if no decimals with precisions are needed
+ public CSVTestBuilder baselineTypes(TypeProtos.MinorType ... baselineTypes) {
+ TypeProtos.MajorType[] majorTypes = new TypeProtos.MajorType[baselineTypes.length];
+ int i = 0;
+ for(TypeProtos.MinorType minorType : baselineTypes) {
+ majorTypes[i] = Types.required(minorType);
+ i++;
+ }
+ this.baselineTypes = majorTypes;
+ this.baselineTypeMap = null;
+ return this;
+ }
+
+ @Override
+ protected TestBuilder reset() {
+ super.reset();
+ baselineTypeMap = null;
+ baselineTypes = null;
+ baselineFilePath = null;
+ return this;
+ }
+
+ @Override
+ boolean typeInfoSet() {
+ if (super.typeInfoSet() || baselineTypes != null) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ String getValidationQuery() throws Exception {
+ if (baselineColumns.length == 0) {
+ throw new Exception("Baseline CSV files require passing column names, please call the baselineColumns() method on the test builder.");
+ }
+
+ if (baselineTypes != null) {
+ assertEquals("Must pass the same number of types as column names if types are provided.", baselineTypes.length, baselineColumns.length);
+ }
+
+ String[] aliasedExpectedColumns = new String[baselineColumns.length];
+ for (int i = 0; i < baselineColumns.length; i++) {
+ aliasedExpectedColumns[i] = "columns[" + i + "] ";
+ TypeProtos.MajorType majorType;
+ if (baselineTypes != null) {
+ majorType = baselineTypes[i];
+ } else if (baselineTypeMap != null) {
+ majorType = baselineTypeMap.get(parsePath(baselineColumns[i]));
+ } else {
+ throw new Exception("Type information not set for interpreting csv baseline file.");
+ }
+ String precision = getDecimalPrecisionScaleInfo(majorType);
+ // TODO - determine if there is a better behavior here, if we do not specify a length the default behavior is
+ // to cast to varchar with length 1
+ // set default cast size for varchar, the cast function will take the lesser of this passed value and the
+ // length of the incoming data when choosing the length for the outgoing data
+ if (majorType.getMinorType() == TypeProtos.MinorType.VARCHAR ||
+ majorType.getMinorType() == TypeProtos.MinorType.VARBINARY) {
+ precision = "(65000)";
+ }
+ aliasedExpectedColumns[i] = "cast(" + aliasedExpectedColumns[i] + " as " +
+ Types.getNameOfMinorType(majorType.getMinorType()) + precision + " ) " + baselineColumns[i];
+ }
+ String query = "select " + Joiner.on(", ").join(aliasedExpectedColumns) + " from cp.`" + baselineFilePath + "`";
+ return query;
+ }
+
+ @Override
+ protected UserBitShared.QueryType getValidationQueryType() throws Exception {
+ return UserBitShared.QueryType.SQL;
+ }
+ }
+
+ public class SchemaTestBuilder extends TestBuilder {
+ private List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema;
+ SchemaTestBuilder(TestServices services, Object query, UserBitShared.QueryType queryType,
+ String baselineOptionSettingQueries, String testOptionSettingQueries, List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema) {
+ super(services, query, queryType, false, false, null, baselineOptionSettingQueries, testOptionSettingQueries, false, -1);
+ expectsEmptyResultSet();
+ this.expectedSchema = expectedSchema;
+ }
+
+ @Override
+ public TestBuilder baselineColumns(String... columns) {
+ assert false : "The column information should be captured in expected scheme, not baselineColumns";
+ return this;
+ }
+
+ @Override
+ public TestBuilder baselineRecords(List<Map<String, Object>> materializedRecords) {
+ assert false : "Since only schema will be compared in this test, no record is expected";
+ return this;
+ }
+
+ @Override
+ public TestBuilder baselineValues(Object... objects) {
+ assert false : "Since only schema will be compared in this test, no record is expected";
+ return this;
+ }
+
+ @Override
+ protected UserBitShared.QueryType getValidationQueryType() throws Exception {
+ return null;
+ }
+
+ @Override
+ public List<Pair<SchemaPath, TypeProtos.MajorType>> getExpectedSchema() {
+ return expectedSchema;
+ }
+ }
+
+ public class JSONTestBuilder extends TestBuilder {
+
+ // path to the baseline file that will be inserted into the validation query
+ private String baselineFilePath;
+
+ JSONTestBuilder(String baselineFile, TestServices services, Object query, UserBitShared.QueryType queryType, Boolean ordered,
+ boolean approximateEquality, Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap,
+ String baselineOptionSettingQueries, String testOptionSettingQueries, boolean highPerformanceComparison,
+ int expectedNumBatches) {
+ super(services, query, queryType, ordered, approximateEquality, baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries,
+ highPerformanceComparison, expectedNumBatches);
+ this.baselineFilePath = baselineFile;
+ this.baselineColumns = new String[] {"*"};
+ }
+
+ @Override
+ String getValidationQuery() {
+ return "select " + Joiner.on(", ").join(baselineColumns) + " from cp.`" + baselineFilePath + "`";
+ }
+
+ @Override
+ protected UserBitShared.QueryType getValidationQueryType() throws Exception {
+ return UserBitShared.QueryType.SQL;
+ }
+
+ }
+
+ public class BaselineQueryTestBuilder extends TestBuilder {
+
+ /**
+ * Baseline query. Type of object depends on {@link #baselineQueryType}
+ */
+ private Object baselineQuery;
+ private UserBitShared.QueryType baselineQueryType;
+
+ BaselineQueryTestBuilder(Object baselineQuery, UserBitShared.QueryType baselineQueryType, TestServices services,
+ Object query, UserBitShared.QueryType queryType, Boolean ordered,
+ boolean approximateEquality, Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap,
+ String baselineOptionSettingQueries, String testOptionSettingQueries, boolean highPerformanceComparison,
+ int expectedNumBatches) {
+ super(services, query, queryType, ordered, approximateEquality, baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries,
+ highPerformanceComparison, expectedNumBatches);
+ this.baselineQuery = baselineQuery;
+ this.baselineQueryType = baselineQueryType;
+ }
+
+ @Override
+ Object getValidationQuery() {
+ return baselineQuery;
+ }
+
+ @Override
+ protected UserBitShared.QueryType getValidationQueryType() throws Exception {
+ return baselineQueryType;
+ }
+
+ // This currently assumes that all explicit baseline queries will have fully qualified type information
+ // if this changes, the baseline query can be run in a sub query with the implicit or explicit type passing
+ // added on top of it, as is currently when done when reading a baseline file
+ @Override
+ boolean typeInfoSet() {
+ return true;
+ }
+
+ }
+
+ /**
+ * Convenience method to create a {@link JsonStringArrayList list} from the given values.
+ */
+ public static JsonStringArrayList<Object> listOf(Object... values) {
+ final JsonStringArrayList<Object> list = new JsonStringArrayList<>();
+ for (Object value:values) {
+ if (value instanceof CharSequence) {
+ list.add(new Text(value.toString()));
+ } else {
+ list.add(value);
+ }
+ }
+ return list;
+ }
+
+ /**
+ * Convenience method to create a {@link JsonStringHashMap<String, Object> map} instance with the given key value sequence.
+ *
+ * Key value sequence consists of key - value pairs such that a key precedes its value. For instance:
+ *
+ * mapOf("name", "Adam", "age", 41) corresponds to {"name": "Adam", "age": 41} in JSON.
+ */
+ public static JsonStringHashMap<String, Object> mapOf(Object... keyValueSequence) {
+ Preconditions.checkArgument(keyValueSequence.length%2==0, "Length of key value sequence must be even");
+ final JsonStringHashMap<String, Object> map = new JsonStringHashMap<>();
+ for (int i=0; i<keyValueSequence.length; i+=2) {
+ Object value = keyValueSequence[i+1];
+ if (value instanceof CharSequence) {
+ value = new Text(value.toString());
+ }
+ map.put(String.class.cast(keyValueSequence[i]), value);
+ }
+ return map;
+ }
+
+ /**
+ * Helper method for the timestamp values that depend on the local timezone
+ * @param value expected timestamp value in UTC
+ * @return timestamp value for the local timezone
+ */
+ public static Timestamp convertToLocalTimestamp(String value) {
+ long UTCTimestamp = Timestamp.valueOf(value).getTime();
+ return new Timestamp(DateTimeZone.getDefault().convertUTCToLocal(UTCTimestamp));
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
deleted file mode 100644
index 83c60b3..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.test;
-
-import org.apache.drill.categories.OptionsTest;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.server.options.OptionDefinition;
-import org.apache.drill.exec.server.options.OptionMetaData;
-import org.apache.drill.exec.server.options.OptionValidator;
-import org.apache.drill.exec.server.options.OptionValue;
-import org.apache.drill.exec.server.options.TypeValidators;
-import org.apache.drill.exec.store.sys.SystemTable;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import static org.junit.Assert.assertEquals;
-/*
- * Tests to test if the linkage between the two config option systems
- * i.e., the linkage between boot-config system and system/session options.
- * Tests to assert if the config options are read in the order of session , system, boot-config.
- * Max width per node is slightly different from other options since it is set to zero by default
- * in the config and the option value is computed dynamically everytime if the value is zero
- * i.e., if the value is not set in system/session.
- * */
-
-@Category(OptionsTest.class)
-public class TestConfigLinkage {
- public static final String MOCK_PROPERTY = "mock.prop";
-
- public static OptionDefinition createMockPropOptionDefinition() {
- return new OptionDefinition(new TypeValidators.StringValidator(MOCK_PROPERTY), new OptionMetaData(OptionValue.AccessibleScopes.ALL, false, true));
- }
-
- @Test
- public void testDefaultInternalValue() throws Exception {
- OptionDefinition optionDefinition = createMockPropOptionDefinition();
-
- ClusterFixtureBuilder builder = ClusterFixture.builder().
- configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
- putDefinition(optionDefinition);
-
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String mockProp = client.queryBuilder().
- sql("SELECT string_val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS.getTableName(), MOCK_PROPERTY).singletonString();
- String mockProp2 = client.queryBuilder().
- sql("SELECT val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS_VAL.getTableName(), MOCK_PROPERTY).singletonString();
-
- assertEquals("a", mockProp);
- assertEquals("a", mockProp2);
- }
- }
-
- @Test
- public void testDefaultValidatorInternalValue() throws Exception {
- OptionDefinition optionDefinition = createMockPropOptionDefinition();
-
- ClusterFixtureBuilder builder = ClusterFixture.builder().
- putDefinition(optionDefinition).
- configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a");
-
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String mockProp = client.queryBuilder().
- sql("SELECT string_val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS.getTableName(), MOCK_PROPERTY).singletonString();
- String mockProp2 = client.queryBuilder().
- sql("SELECT val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS_VAL.getTableName(), MOCK_PROPERTY).singletonString();
-
- assertEquals("a", mockProp);
- assertEquals("a", mockProp2);
- }
- }
-
- /* Test if session option takes precedence */
- @Test
- public void testSessionOption() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder().sessionOption(ExecConstants.SLICE_TARGET, 10);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SESSION'", SystemTable.OPTION_VAL
- .getTableName())
- .singletonString();
- assertEquals(slice_target, "10");
- }
- }
-
- /* Test if system option takes precedence over the boot option */
- @Test
- public void testSystemOption() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 10000);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- assertEquals(slice_target, "10000");
- }
- }
-
- @Test
- public void testInternalSystemOption() throws Exception {
- OptionDefinition optionDefinition = createMockPropOptionDefinition();
-
- ClusterFixtureBuilder builder = ClusterFixture.builder().
- putDefinition(optionDefinition).
- configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
- systemOption(MOCK_PROPERTY, "blah");
-
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String mockProp = client.queryBuilder().
- sql("SELECT string_val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS.getTableName(), MOCK_PROPERTY)
- .singletonString();
- String mockProp2 = client.queryBuilder().
- sql("SELECT val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS_VAL.getTableName(), MOCK_PROPERTY)
- .singletonString();
-
- assertEquals("blah", mockProp);
- assertEquals("blah", mockProp2);
- }
- }
-
- /* Test if config option takes precedence if config option is not set */
- @Test
- public void testConfigOption() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
- .setOptionDefault(ExecConstants.SLICE_TARGET, 1000);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'BOOT'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- assertEquals(slice_target, "1000");
- }
- }
-
- /* Test if altering system option takes precedence over config option */
- @Test
- public void testAlterSystem() throws Exception {
- try (ClusterFixture cluster = ClusterFixture.standardCluster();
- ClientFixture client = cluster.clientFixture()) {
- client.queryBuilder().sql("ALTER SYSTEM SET `planner.slice_target` = 10000").run();
- String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- assertEquals(slice_target, "10000");
- }
- }
-
- /* Test if altering session option takes precedence over system option */
- @Test
- public void testSessionPrecedence() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 100000);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- client.queryBuilder().sql("ALTER SESSION SET `planner.slice_target` = 10000").run();
- String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SESSION'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- assertEquals(slice_target, "10000");
- }
- }
-
- /* Test if setting maxwidth option through config takes effect */
- @Test
- public void testMaxWidthPerNodeConfig() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'BOOT'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- assertEquals("2", maxWidth);
- }
- }
-
- /* Test if setting maxwidth at system level takes precedence */
- @Test
- public void testMaxWidthPerNodeSystem() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- assertEquals("3", maxWidth);
- }
- }
-
- /* Test if setting maxwidth at session level takes precedence */
- @Test
- public void testMaxWidthPerNodeSession() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'SESSION'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- assertEquals("2", maxWidth);
- }
- }
-
- /* Test if max width is computed correctly using the cpu load average
- when the option is not set at either system or session level
- */
- @Test
- public void testMaxWidthPerNodeDefault() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- long maxWidth = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(0.70, 0);
- int availProc = Runtime.getRuntime().availableProcessors();
- long maxWidthPerNode = Math.max(1, Math.min(availProc, Math.round(availProc * 0.70)));
- assertEquals(maxWidthPerNode, maxWidth);
- }
- }
-
- /* Test if the scope is set during BOOT time and scope is actually BOOT */
- @Test
- public void testScope() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.SLICE_TARGET, 100000);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String scope = client.queryBuilder()
- .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- Assert.assertEquals("BOOT",scope);
- }
- }
-
- /* Test if the option is set at SYSTEM scope and the scope is actually SYSTEM */
- @Test
- public void testScopeSystem() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.SLICE_TARGET, 10000);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String scope = client.queryBuilder()
- .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- Assert.assertEquals("SYSTEM",scope);
- }
- }
-
- /* Test if the option is set at SESSION scope and the scope is actually SESSION */
- @Test
- public void testScopeSession() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.SLICE_TARGET, 100000);
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String scope = client.queryBuilder()
- .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- Assert.assertEquals("SESSION",scope);
- }
- }
-
- /* Test if the option is altered at SYSTEM scope and the scope is actually SYSTEM */
- @Test
- public void testScopeAlterSystem() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder();
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- client.queryBuilder().sql("ALTER SYSTEM set `planner.slice_target`= 10000").run();
- String scope = client.queryBuilder()
- .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- Assert.assertEquals("SYSTEM",scope);
- }
- }
-
- /* Test if the option is altered at SESSION scope and the scope is actually SESSION */
- @Test
- public void testScopeAlterSession() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.bareBuilder();
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- client.queryBuilder().sql("ALTER SESSION set `planner.slice_target`= 10000").run();
- String scope = client.queryBuilder()
- .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
- .singletonString();
- Assert.assertEquals("SESSION",scope);
- }
- }
-
- @Test
- public void testAlterInternalSystemOption() throws Exception {
- OptionDefinition optionDefinition = createMockPropOptionDefinition();
-
- ClusterFixtureBuilder builder = ClusterFixture.builder().
- configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
- putDefinition(optionDefinition);
-
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- client.queryBuilder().sql("ALTER SYSTEM SET `%s` = 'bleh'", MOCK_PROPERTY).run();
-
- client.queryBuilder().sql("SELECT * FROM sys.%s", SystemTable.INTERNAL_OPTIONS.getTableName()).printCsv();
- client.queryBuilder().sql("SELECT * FROM sys.%s", SystemTable.INTERNAL_OPTIONS_VAL.getTableName()).printCsv();
-
- String mockProp = client.queryBuilder().
- sql("SELECT string_val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS, MOCK_PROPERTY).singletonString();
- String mockProp2 = client.queryBuilder().
- sql("SELECT val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS_VAL, MOCK_PROPERTY).singletonString();
-
- assertEquals("bleh", mockProp);
- assertEquals("bleh", mockProp2);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java b/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java
index dac71ec..e1817dd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java
@@ -16,55 +16,103 @@
* limitations under the License.
*/
/**
- * Provides a variety of test framework tools to simplify Drill unit
- * tests and ad-hoc tests created while developing features. Key components
- * include:
+ * <p>
+ * Provides a variety of test framework tools to simplify Drill unit
+ * tests and ad-hoc tests created while developing features. Key components
+ * include:
+ * </p>
* <ul>
- * <li>{@link ClusterFixtureBuilder}: Builder pattern to create an embedded Drillbit,
- * or cluster of Drillbits, using a specified set of configuration, session
- * and system options.</li>
- * <li>{@link ClusterFixture}: The cluster created by the builder.</li>
- * <li>{@link ClientFixture}: A facade to the Drill client that provides
- * convenience methods for setting session options, running queries and
- * so on. A client is associated with a cluster. If tests desire, multiple
- * clients can be created for a single cluster, though most need just one
- * client. A builder exists for clients, but most tests get the client
- * directly from the cluster.</li>
- * <li>{@link QueryBuilder}: a builder pattern for constructing and
- * running any form of query (SQL, logical or physical) and running the
- * query in a wide variety of ways (just count the rows, return the
- * results as a list, run using a listener, etc.)</li>
- * <li>{@link QueryBuilder.QuerySummary QuerySummary}: a summary of a
- * query returned from running the query. Contains the query ID, the
- * row count, the batch count and elapsed run time.</li>
- * <li>{@link ProfileParser}: A simple tool to load a query profile and
- * provide access to the profile structure. Also prints the key parts of
- * the profile for diagnostic purposes.</li>
- * <li>{@link LogFixture}: Allows per-test changes to log settings to,
- * say, send a particular logger to the console for easier debugging, or
- * to suppress logging of a deliberately created failure.</li>
+ * <li>
+ * {@link org.apache.drill.test.ClusterFixtureBuilder}: Builder pattern to create an embedded Drillbit,
+ * or cluster of Drillbits, using a specified set of configuration, session
+ * and system options.
+ * </li>
+ * <li>
+ * {@link org.apache.drill.test.ClusterFixture}: The cluster created by the builder.
+ * </li>
+ * <li>
+ * {@link org.apache.drill.test.ClientFixture}: A facade to the Drill client that provides
+ * convenience methods for setting session options, running queries and
+ * so on. A client is associated with a cluster. If tests desire, multiple
+ * clients can be created for a single cluster, though most need just one
+ * client. A builder exists for clients, but most tests get the client
+ * directly from the cluster.
+ * </li>
+ * <li>
+ * {@link org.apache.drill.test.QueryBuilder}: a builder pattern for constructing and
+ * running any form of query (SQL, logical or physical) and running the
+ * query in a wide variety of ways (just count the rows, return the
+ * results as a list, run using a listener, etc.)
+ * </li>
+ * <li>
+ * {@link org.apache.drill.test.QueryBuilder.QuerySummary QuerySummary}: a summary of a
+ * query returned from running the query. Contains the query ID, the
+ * row count, the batch count and elapsed run time.
+ * </li>
+ * <li>
+ * {@link org.apache.drill.test.ProfileParser}: A simple tool to load a query profile and
+ * provide access to the profile structure. Also prints the key parts of
+ * the profile for diagnostic purposes.
+ * </li>
+ * <li>
+ * {@link org.apache.drill.test.LogFixture}: Allows per-test changes to log settings to,
+ * say, send a particular logger to the console for easier debugging, or
+ * to suppress logging of a deliberately created failure.
+ * </li>
+ * <li>
+ * {@link org.apache.drill.test.BaseDirTestWatcher}: Creates temporary directories which are used for the following
+ * aspects of drill in unit tests:
+ * <ul>
+ * <li>
+ * The <b>drill.tmp-dir</b> property: A temp directory ({@link org.apache.drill.test.BaseDirTestWatcher#getTmpDir()}) is created and
+ * configured as the tmp directory for drill bits.
+ * </li>
+ * <li>
+ * The <b>drill.exec.sys.store.provider.local.path</b> property: A temp directory ({@link org.apache.drill.test.BaseDirTestWatcher#getStoreDir()}) is created and
+ * configured as the store directory for drillbits.
+ * </li>
+ * <li>
+ * The <b>dfs.default</b> workspace: A temp directory ({@link org.apache.drill.test.BaseDirTestWatcher#getRootDir()} is created and configured as the directory
+ * for this workspace.
+ * </li>
+ * <li>
+ * The <b>dfs.root</b> workspace: <b>dfs.root</b> is configured to use the same temp directory ad <b>dfs.default</b>.
+ * </li>
+ * <li>
+ * The <b>dfs.tmp</b> workspace: A temp directory ({@link org.apache.drill.test.BaseDirTestWatcher#getDfsTestTmpDir()}) is created and configured as the
+ * directory for this workspace.
+ * </li>
+ * </ul>
+ * By default a {@link org.apache.drill.test.BaseDirTestWatcher} is used in tests that extend {@link org.apache.drill.test.BaseTestQuery} and tests that use
+ * the {@link org.apache.drill.test.ClusterFixture}.
+ * </li>
* </ul>
* <h3>Usage</h3>
- * A typical test using this framework looks like this:
+ * <p>
+ * A typical test using this framework looks like this:
+ * </p>
* <code><pre>
+ {@literal @}org.junit.Rule
+ public final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
{@literal @}Test
public void exampleTest() throws Exception {
+ createEmployeeCsv(dirTestWatcher.getRootDir());
// Configure the cluster. One Drillbit by default.
- FixtureBuilder builder = ClusterFixture.builder()
+ FixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
// Set up per-test specialized config and session options.
.configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true)
.configProperty(ExecConstants.REMOVER_ENABLE_GENERIC_COPIER, true)
.sessionOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3L * 1024 * 1024 * 1024)
- .maxParallelization(1)
- ;
+ .maxParallelization(1);
// Launch the cluster and client.
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
// Run a query (using the mock data source) and print a summary.
- String sql = "SELECT id_i FROM `mock`.employee_1M ORDER BY id_i";
+ String sql = "SELECT id_i FROM dfs.`employee.csv` ORDER BY id_i";
QuerySummary summary = client.queryBuilder().sql(sql).run();
assertEquals(1_000_000, summary.recordCount());
System.out.println(String.format("Sorted %,d records in %d batches.", summary.recordCount(), summary.batchCount()));
@@ -73,8 +121,8 @@
}
}
* </pre></code>
- * <p>
- * Typical usage for the logging fixture: <pre><code>
+ * Typical usage for the logging fixture:
+ * <pre><code>
* {@literal @}Test
* public void myTest() {
* LogFixtureBuilder logBuilder = LogFixture.builder()
@@ -84,7 +132,8 @@
* try (LogFixture logs = logBuilder.build()) {
* // Test code here
* }
- * }</code></pre>
+ * }
+ * </code></pre>
*
*/
package org.apache.drill.test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java
index 6401f75..474508c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java
@@ -85,8 +85,7 @@ public interface RowSet {
* new row set with the updated columns, then merge the new
* and old row sets to create a new immutable row set.
*/
-
- public interface RowSetWriter extends TupleWriter {
+ interface RowSetWriter extends TupleWriter {
void setRow(Object...values);
boolean valid();
int index();
@@ -97,8 +96,7 @@ public interface RowSet {
/**
* Reader for all types of row sets.
*/
-
- public interface RowSetReader extends TupleReader {
+ interface RowSetReader extends TupleReader {
/**
* Total number of rows in the row set.
@@ -159,7 +157,6 @@ public interface RowSet {
*
* @return memory size in bytes
*/
-
long size();
RowSet merge(RowSet other);
@@ -169,8 +166,7 @@ public interface RowSet {
/**
* Row set that manages a single batch of rows.
*/
-
- public interface SingleRowSet extends RowSet {
+ interface SingleRowSet extends RowSet {
ValueVector[] vectors();
SingleRowSet toIndirect();
SelectionVector2 getSv2();
@@ -181,8 +177,7 @@ public interface RowSet {
* Once writing is complete, the row set becomes an
* immutable direct row set.
*/
-
- public interface ExtendableRowSet extends SingleRowSet {
+ interface ExtendableRowSet extends SingleRowSet {
void allocate(int recordCount);
void setRowCount(int rowCount);
RowSetWriter writer(int initialRowCount);
@@ -192,8 +187,7 @@ public interface RowSet {
* Row set comprised of multiple single row sets, along with
* an indirection vector (SV4).
*/
-
- public interface HyperRowSet extends RowSet {
+ interface HyperRowSet extends RowSet {
SelectionVector4 getSv4();
HyperVectorWrapper<ValueVector> getHyperVector(int i);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java
index 80e8ae4..6f9a8d9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java
@@ -24,7 +24,7 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
/**
* Fluent builder to quickly build up an row set (record batch)
- * programmatically. Starting with an {@link OperatorFixture}:
+ * programmatically. Starting with an {@link org.apache.drill.test.OperatorFixture}:
* <pre></code>
* OperatorFixture fixture = ...
* RowSet rowSet = fixture.rowSetBuilder(batchSchema)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java
index ea50074..6e72923 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java
@@ -25,6 +25,8 @@ import org.apache.drill.exec.vector.accessor.ColumnReader;
import org.apache.drill.test.rowSet.RowSet.RowSetReader;
import org.bouncycastle.util.Arrays;
+import java.util.Comparator;
+
/**
* For testing, compare the contents of two row sets (record batches)
* to verify that they are identical. Supports masks to exclude certain
@@ -255,4 +257,39 @@ public class RowSetComparison {
}
}
}
+
+ // TODO make a native RowSetComparison comparator
+ public static class ObjectComparator implements Comparator<Object> {
+ public static final ObjectComparator INSTANCE = new ObjectComparator();
+
+ private ObjectComparator() {
+ }
+
+ @Override
+ public int compare(Object a, Object b) {
+ if (a instanceof Integer) {
+ int aInt = (Integer) a;
+ int bInt = (Integer) b;
+ return aInt - bInt;
+ } else if (a instanceof Long) {
+ Long aLong = (Long) a;
+ Long bLong = (Long) b;
+ return aLong.compareTo(bLong);
+ } else if (a instanceof Float) {
+ Float aFloat = (Float) a;
+ Float bFloat = (Float) b;
+ return aFloat.compareTo(bFloat);
+ } else if (a instanceof Double) {
+ Double aDouble = (Double) a;
+ Double bDouble = (Double) b;
+ return aDouble.compareTo(bDouble);
+ } else if (a instanceof String) {
+ String aString = (String) a;
+ String bString = (String) b;
+ return aString.compareTo(bString);
+ } else {
+ throw new UnsupportedOperationException(String.format("Unsupported type %s", a.getClass().getCanonicalName()));
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
new file mode 100644
index 0000000..ff93bf0
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.test.rowSet.file;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.accessor.ColumnAccessor;
+import org.apache.drill.exec.vector.accessor.ColumnReader;
+import org.apache.drill.test.rowSet.RowSet;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public class JsonFileBuilder
+{
+ public static final String DEFAULT_DOUBLE_FORMATTER = "%f";
+ public static final String DEFAULT_INTEGER_FORMATTER = "%d";
+ public static final String DEFAULT_LONG_FORMATTER = "%d";
+ public static final String DEFAULT_STRING_FORMATTER = "\"%s\"";
+ public static final String DEFAULT_DECIMAL_FORMATTER = "%s";
+ public static final String DEFAULT_PERIOD_FORMATTER = "%s";
+
+ public static final Map<String, String> DEFAULT_FORMATTERS = new ImmutableMap.Builder()
+ .put(ColumnAccessor.ValueType.DOUBLE, DEFAULT_DOUBLE_FORMATTER)
+ .put(ColumnAccessor.ValueType.INTEGER, DEFAULT_INTEGER_FORMATTER)
+ .put(ColumnAccessor.ValueType.LONG, DEFAULT_LONG_FORMATTER)
+ .put(ColumnAccessor.ValueType.STRING, DEFAULT_STRING_FORMATTER)
+ .put(ColumnAccessor.ValueType.DECIMAL, DEFAULT_DECIMAL_FORMATTER)
+ .put(ColumnAccessor.ValueType.PERIOD, DEFAULT_PERIOD_FORMATTER)
+ .build();
+
+ private final RowSet rowSet;
+ private final Map<String, String> customFormatters = Maps.newHashMap();
+
+ public JsonFileBuilder(RowSet rowSet) {
+ this.rowSet = Preconditions.checkNotNull(rowSet);
+ Preconditions.checkArgument(rowSet.rowCount() > 0, "The given rowset is empty.");
+ }
+
+ public JsonFileBuilder setCustomFormatter(final String columnName, final String columnFormatter) {
+ Preconditions.checkNotNull(columnName);
+ Preconditions.checkNotNull(columnFormatter);
+
+ Iterator<MaterializedField> fields = rowSet
+ .schema()
+ .batch()
+ .iterator();
+
+ boolean hasColumn = false;
+
+ while (!hasColumn && fields.hasNext()) {
+ hasColumn = fields.next()
+ .getName()
+ .equals(columnName);
+ }
+
+ final String message = String.format("(%s) is not a valid column", columnName);
+ Preconditions.checkArgument(hasColumn, message);
+
+ customFormatters.put(columnName, columnFormatter);
+
+ return this;
+ }
+
+ public void build(File tableFile) throws IOException {
+ tableFile.getParentFile().mkdirs();
+
+ try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(tableFile))) {
+ final RowSet.RowSetReader reader = rowSet.reader();
+ final int numCols = rowSet
+ .schema()
+ .batch()
+ .getFieldCount();
+ final Iterator<MaterializedField> fieldIterator = rowSet
+ .schema()
+ .batch()
+ .iterator();
+ final List<String> columnNames = Lists.newArrayList();
+ final List<String> columnFormatters = Lists.newArrayList();
+
+ // Build formatters from first row.
+ while (fieldIterator.hasNext()) {
+ final String columnName = fieldIterator.next().getName();
+ final ColumnReader columnReader = reader.column(columnName);
+ final ColumnAccessor.ValueType valueType = columnReader.valueType();
+ final String columnFormatter;
+
+ if (customFormatters.containsKey(columnName)) {
+ columnFormatter = customFormatters.get(columnName);
+ } else if (DEFAULT_FORMATTERS.containsKey(valueType)) {
+ columnFormatter = DEFAULT_FORMATTERS.get(valueType);
+ } else {
+ final String message = String.format("Unsupported column type %s", valueType);
+ throw new UnsupportedOperationException(message);
+ }
+
+ columnNames.add(columnName);
+ columnFormatters.add(columnFormatter);
+ }
+
+ final StringBuilder sb = new StringBuilder();
+ String lineSeparator = "";
+
+ for (int index = 0; index < rowSet.rowCount(); index++) {
+ reader.next();
+ sb.append(lineSeparator);
+ sb.append('{');
+ String separator = "";
+
+ for (int columnIndex = 0; columnIndex < numCols; columnIndex++) {
+ sb.append(separator);
+
+ final String columnName = columnNames.get(columnIndex);
+ final ColumnReader columnReader = reader.column(columnIndex);
+ final String columnFormatter = columnFormatters.get(columnIndex);
+ final Object columnObject = columnReader.getObject();
+ final String columnString = String.format(columnFormatter, columnObject);
+
+ sb.append('"')
+ .append(columnName)
+ .append('"')
+ .append(':')
+ .append(columnString);
+
+ separator = ",";
+ }
+
+ sb.append('}');
+ lineSeparator = "\n";
+ os.write(sb.toString().getBytes());
+ sb.delete(0, sb.length());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
deleted file mode 100644
index 35ca26b..0000000
--- a/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
+++ /dev/null
@@ -1,75 +0,0 @@
-{
- "storage":{
- dfs_test: {
- type: "file",
- connection: "file:///",
- workspaces: {
- "home" : {
- location: "/",
- writable: false
- },
- "tmp" : {
- location: "/tmp",
- writable: true
- }
- },
- formats: {
- "psv" : {
- type: "text",
- extensions: [ "tbl" ],
- delimiter: "|"
- },
- "csv" : {
- type: "text",
- extensions: [ "csv", "bcp" ],
- delimiter: ","
- },
- "tsv" : {
- type: "text",
- extensions: [ "tsv" ],
- delimiter: "\t"
- },
- "ssv" : {
- type: "text",
- extensions: [ "ssv" ],
- delimiter: " "
- },
- "parquet" : {
- type: "parquet"
- },
- "json" : {
- type: "json"
- },
- "httpd" : {
- type: "httpd",
- logFormat: "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\""
- },
- "txt" : {
- type : "text",
- extensions: [ "txt" ],
- delimiter: "\u0000"
- },
- "avro" : {
- type: "avro"
- },
- "sequencefile": {
- type: "sequencefile",
- extensions: [ "seq" ]
- },
- "csvh" : {
- type: "text",
- extensions: [ "csvh" ],
- delimiter: ",",
- extractHeader: true
- },
- "csvh-test" : {
- type: "text",
- extensions: [ "csvh-test" ],
- delimiter: ",",
- extractHeader: true,
- skipFirstLine: true
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_nullable.json b/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
index 56963a8..3e09f83 100644
--- a/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
+++ b/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
@@ -8,10 +8,10 @@
}
},
storage:{
- "dfs_test" :
+ "dfs" :
{
"type":"named",
- "name":"dfs_test"
+ "name":"dfs"
}
},
query:[
@@ -19,7 +19,7 @@
@id:"1",
op:"scan",
memo:"initial_scan",
- storageengine:"dfs_test",
+ storageengine:"dfs",
selection: {
format: {type: "parquet"},
files: [
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json b/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
index ea81b6c..9547f00 100644
--- a/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
+++ b/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
@@ -8,10 +8,10 @@
}
},
storage:{
- "dfs_test" :
+ "dfs" :
{
"type":"named",
- "name":"dfs_test"
+ "name":"dfs"
}
},
query:[
@@ -19,7 +19,7 @@
@id:"1",
op:"scan",
memo:"initial_scan",
- storageengine:"dfs_test",
+ storageengine:"dfs",
selection: {
format: {type: "parquet"},
files: [
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json
index ebb0b24..50281f4 100644
--- a/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json
+++ b/exec/java-exec/src/test/resources/parquet/parquet_scan_screen.json
@@ -8,10 +8,10 @@
}
},
storage:{
- "dfs_test" :
+ "dfs" :
{
"type":"named",
- "name":"dfs_test"
+ "name":"dfs"
}
},
query:[
@@ -19,7 +19,7 @@
@id:"1",
op:"scan",
memo:"initial_scan",
- storageengine:"dfs_test",
+ storageengine:"dfs",
selection: {
format: {type: "parquet"},
files: [
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/queries/tpch/15.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/queries/tpch/15.sql b/exec/java-exec/src/test/resources/queries/tpch/15.sql
index 49927a4..3613d68 100644
--- a/exec/java-exec/src/test/resources/queries/tpch/15.sql
+++ b/exec/java-exec/src/test/resources/queries/tpch/15.sql
@@ -1,5 +1,5 @@
-- tpch15 using 1395599672 as a seed to the RNG
-use dfs_test.tmp; -- views can only be created in dfs schema
+use dfs.tmp; -- views can only be created in dfs schema
create view revenue0 (supplier_no, total_revenue) as
select
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/store/text/test.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/text/test.json b/exec/java-exec/src/test/resources/store/text/test.json
index 4a312d6..78ac516 100644
--- a/exec/java-exec/src/test/resources/store/text/test.json
+++ b/exec/java-exec/src/test/resources/store/text/test.json
@@ -15,7 +15,7 @@
],
storage : {
type : "named",
- name: "dfs_test"
+ name: "dfs"
},
format: {
type: "named",
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/topN/one_key_sort.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/topN/one_key_sort.json b/exec/java-exec/src/test/resources/topN/one_key_sort.json
index 45b3fc3..39b9355 100644
--- a/exec/java-exec/src/test/resources/topN/one_key_sort.json
+++ b/exec/java-exec/src/test/resources/topN/one_key_sort.json
@@ -12,11 +12,11 @@
pop:"mock-scan",
url: "http://apache.org",
entries:[
- {records: 10000000, types: [
+ {records: 100000, types: [
{name: "blue", type: "INT", mode: "REQUIRED"},
{name: "green", type: "INT", mode: "REQUIRED"}
]},
- {records: 10000000, types: [
+ {records: 100000, types: [
{name: "blue", type: "INT", mode: "REQUIRED"},
{name: "green", type: "INT", mode: "REQUIRED"}
]}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/tpcds-sf1/q73.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/tpcds-sf1/q73.sql b/exec/java-exec/src/test/resources/tpcds-sf1/q73.sql
index 094ca2b..4db8a4f 100644
--- a/exec/java-exec/src/test/resources/tpcds-sf1/q73.sql
+++ b/exec/java-exec/src/test/resources/tpcds-sf1/q73.sql
@@ -6,10 +6,10 @@ select c.c_last_name,
dj.cnt
from (
select ss.ss_ticket_number as sstn, ss.ss_customer_sk as sscsk, count(*) cnt
- from dfs_test.tpcds.store_sales as ss,
- dfs_test.tpcds.date_dim as d,
- dfs_test.tpcds.store as s,
- dfs_test.tpcds.household_demographics as hd
+ from dfs.tpcds.store_sales as ss,
+ dfs.tpcds.date_dim as d,
+ dfs.tpcds.store as s,
+ dfs.tpcds.household_demographics as hd
where ss.ss_sold_date_sk = d.d_date_sk
and ss.ss_store_sk = s.s_store_sk
and ss.ss_hdemo_sk = hd.hd_demo_sk
@@ -20,7 +20,7 @@ from (
and ss.ss_sold_date_sk between 2451180 and 2451269
group by ss.ss_ticket_number, ss.ss_customer_sk
) dj,
- dfs_test.tpcds.customer as c
+ dfs.tpcds.customer as c
where dj.sscsk = c.c_customer_sk
and dj.cnt between 1 and 5
order by dj.cnt desc
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/3604.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/3604.sql b/exec/java-exec/src/test/resources/window/3604.sql
index f88beaf..7219b00 100644
--- a/exec/java-exec/src/test/resources/window/3604.sql
+++ b/exec/java-exec/src/test/resources/window/3604.sql
@@ -1,4 +1,4 @@
select
lead(col3) over(partition by col2 order by col0) lead_col0
from
- dfs_test.`%s/window/fewRowsAllData.parquet`
\ No newline at end of file
+ dfs.`window/fewRowsAllData.parquet`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/3605.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/3605.sql b/exec/java-exec/src/test/resources/window/3605.sql
index 1e5d54a..96e3b26 100644
--- a/exec/java-exec/src/test/resources/window/3605.sql
+++ b/exec/java-exec/src/test/resources/window/3605.sql
@@ -2,4 +2,4 @@ select
col2,
lead(col2) over(partition by col2 order by col0) as lead_col2
from
- dfs_test.`%s/window/fewRowsAllData.parquet`
\ No newline at end of file
+ dfs.`window/fewRowsAllData.parquet`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/3606.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/3606.sql b/exec/java-exec/src/test/resources/window/3606.sql
index d96469e..9e39bc2 100644
--- a/exec/java-exec/src/test/resources/window/3606.sql
+++ b/exec/java-exec/src/test/resources/window/3606.sql
@@ -2,4 +2,4 @@ select
col2,
lead(col2) over(order by col0) as lead_col2
from
- dfs_test.`%s/window/fewRowsAllData.parquet`
\ No newline at end of file
+ dfs.`window/fewRowsAllData.parquet`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/3648.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/3648.sql b/exec/java-exec/src/test/resources/window/3648.sql
index c63ba17..baef342 100644
--- a/exec/java-exec/src/test/resources/window/3648.sql
+++ b/exec/java-exec/src/test/resources/window/3648.sql
@@ -2,4 +2,4 @@ select
ntile(5)
over(partition by col7 order by col0) as `ntile`
from
- dfs_test.`%s/window/3648.parquet`
\ No newline at end of file
+ dfs.`window/3648.parquet`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/3668.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/3668.sql b/exec/java-exec/src/test/resources/window/3668.sql
index e7ea7ab..0aec03f 100644
--- a/exec/java-exec/src/test/resources/window/3668.sql
+++ b/exec/java-exec/src/test/resources/window/3668.sql
@@ -1,6 +1,6 @@
select count(fv) as cnt
from (
select first_value(c2) over(partition by c2 order by c1) as fv
- from dfs_test.`%s/window/3668.parquet`
+ from dfs.`window/3668.parquet`
)
where fv = 'e'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/fval.alltypes.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/fval.alltypes.sql b/exec/java-exec/src/test/resources/window/fval.alltypes.sql
index b11db6f..6ba0508 100644
--- a/exec/java-exec/src/test/resources/window/fval.alltypes.sql
+++ b/exec/java-exec/src/test/resources/window/fval.alltypes.sql
@@ -9,6 +9,6 @@ select
first_value(col7) over w as col7,
first_value(col8) over w as col8
from
- dfs_test.`%s/window/fewRowsAllData.parquet`
+ dfs.`window/fewRowsAllData.parquet`
window w as ()
limit 1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/fval.pby.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/fval.pby.sql b/exec/java-exec/src/test/resources/window/fval.pby.sql
index 29d7f54..8afc84e 100644
--- a/exec/java-exec/src/test/resources/window/fval.pby.sql
+++ b/exec/java-exec/src/test/resources/window/fval.pby.sql
@@ -1,4 +1,4 @@
select
first_value(employee_id) over(partition by position_id order by line_no) as `first_value`
from
- dfs_test.`%s/window/b4.p4`
\ No newline at end of file
+ dfs.`window/b4.p4`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/lag.oby.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/lag.oby.sql b/exec/java-exec/src/test/resources/window/lag.oby.sql
index 91dadb5..dd776a9 100644
--- a/exec/java-exec/src/test/resources/window/lag.oby.sql
+++ b/exec/java-exec/src/test/resources/window/lag.oby.sql
@@ -1,3 +1,3 @@
select
lag(line_no) over(order by sub, employee_id) as `lag`
-from dfs_test.`%s/window/b4.p4`
\ No newline at end of file
+from dfs.`window/b4.p4`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/lag.pby.oby.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/lag.pby.oby.sql b/exec/java-exec/src/test/resources/window/lag.pby.oby.sql
index 64a890e..1d954c2 100644
--- a/exec/java-exec/src/test/resources/window/lag.pby.oby.sql
+++ b/exec/java-exec/src/test/resources/window/lag.pby.oby.sql
@@ -1,3 +1,3 @@
select
lag(line_no) over(partition by position_id order by sub, employee_id) as `lag`
-from dfs_test.`%s/window/b4.p4`
\ No newline at end of file
+from dfs.`window/b4.p4`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/lead.oby.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/lead.oby.sql b/exec/java-exec/src/test/resources/window/lead.oby.sql
index 211a142..e48f02d 100644
--- a/exec/java-exec/src/test/resources/window/lead.oby.sql
+++ b/exec/java-exec/src/test/resources/window/lead.oby.sql
@@ -1,3 +1,3 @@
select
lead(line_no) over(order by sub, employee_id) as `lead`
-from dfs_test.`%s/window/b4.p4`
\ No newline at end of file
+from dfs.`window/b4.p4`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/lead.pby.oby.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/lead.pby.oby.sql b/exec/java-exec/src/test/resources/window/lead.pby.oby.sql
index 7d9904a..ac53133 100644
--- a/exec/java-exec/src/test/resources/window/lead.pby.oby.sql
+++ b/exec/java-exec/src/test/resources/window/lead.pby.oby.sql
@@ -1,3 +1,3 @@
select
lead(line_no) over(partition by position_id order by sub, employee_id) as `lead`
-from dfs_test.`%s/window/b4.p4`
\ No newline at end of file
+from dfs.`window/b4.p4`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/lval.alltypes.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/lval.alltypes.sql b/exec/java-exec/src/test/resources/window/lval.alltypes.sql
index 9f2e7fd..e70799d 100644
--- a/exec/java-exec/src/test/resources/window/lval.alltypes.sql
+++ b/exec/java-exec/src/test/resources/window/lval.alltypes.sql
@@ -9,6 +9,6 @@ select
last_value(col7) over w as col7,
last_value(col8) over w as col8
from
- dfs_test.`%s/window/fewRowsAllData.parquet`
+ dfs.`window/fewRowsAllData.parquet`
window w as ()
limit 1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/lval.pby.oby.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/lval.pby.oby.sql b/exec/java-exec/src/test/resources/window/lval.pby.oby.sql
index 6e77a54..d348f02 100644
--- a/exec/java-exec/src/test/resources/window/lval.pby.oby.sql
+++ b/exec/java-exec/src/test/resources/window/lval.pby.oby.sql
@@ -1,4 +1,4 @@
select
last_value(employee_id) over(partition by position_id order by line_no) as `last_value`
from
- dfs_test.`%s/window/b4.p4`
\ No newline at end of file
+ dfs.`window/b4.p4`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/ntile.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/ntile.sql b/exec/java-exec/src/test/resources/window/ntile.sql
index 44184a8..54497b1 100644
--- a/exec/java-exec/src/test/resources/window/ntile.sql
+++ b/exec/java-exec/src/test/resources/window/ntile.sql
@@ -1,4 +1,4 @@
select
ntile(3) over(partition by position_id order by 1) as `ntile`
from
- dfs_test.`%s/window/b2.p4`
\ No newline at end of file
+ dfs.`window/b2.p4`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/q1.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/q1.sql b/exec/java-exec/src/test/resources/window/q1.sql
index 401ee97..7b193cb 100644
--- a/exec/java-exec/src/test/resources/window/q1.sql
+++ b/exec/java-exec/src/test/resources/window/q1.sql
@@ -2,5 +2,5 @@ select
count(*) over pos_win `count`,
sum(salary) over pos_win `sum`
from
- dfs_test.`%s/window/%s`
+ dfs.`window/%s`
window pos_win as %s
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/q2.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/q2.sql b/exec/java-exec/src/test/resources/window/q2.sql
index beae8e0..8e25901 100644
--- a/exec/java-exec/src/test/resources/window/q2.sql
+++ b/exec/java-exec/src/test/resources/window/q2.sql
@@ -6,5 +6,5 @@ select
dense_rank() over pos_win `dense_rank`,
cume_dist() over pos_win `cume_dist`,
percent_rank() over pos_win `percent_rank`
-from dfs_test.`%s/window/%s`
+from dfs.`window/%s`
window pos_win as %s
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/q3.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/q3.sql b/exec/java-exec/src/test/resources/window/q3.sql
index 0efb137..d6875fa 100644
--- a/exec/java-exec/src/test/resources/window/q3.sql
+++ b/exec/java-exec/src/test/resources/window/q3.sql
@@ -6,4 +6,4 @@ SELECT
ORDER by employee_id
RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `last_value`
FROM
- dfs_test.`%s/window/b4.p4`
+ dfs.`window/b4.p4`
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/q3218.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/q3218.sql b/exec/java-exec/src/test/resources/window/q3218.sql
index 45bfd46..6e12c2e 100644
--- a/exec/java-exec/src/test/resources/window/q3218.sql
+++ b/exec/java-exec/src/test/resources/window/q3218.sql
@@ -2,4 +2,4 @@ select
max(cast(columns[2] as char(2)))
over(partition by cast(columns[2] as char(2))
order by cast(columns[0] as int))
-from dfs_test.`%s/window/allData.csv`
\ No newline at end of file
+from dfs.`window/allData.csv`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/q3220.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/q3220.sql b/exec/java-exec/src/test/resources/window/q3220.sql
index 3fdb047..0e33ad6 100644
--- a/exec/java-exec/src/test/resources/window/q3220.sql
+++ b/exec/java-exec/src/test/resources/window/q3220.sql
@@ -1,3 +1,3 @@
select
count(1) over(partition by position_id order by sub)
-from dfs_test.`%s/window/b1.p1`
\ No newline at end of file
+from dfs.`window/b1.p1`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/window/q4.sql
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/window/q4.sql b/exec/java-exec/src/test/resources/window/q4.sql
index 5e1fb22..34ff9e2 100644
--- a/exec/java-exec/src/test/resources/window/q4.sql
+++ b/exec/java-exec/src/test/resources/window/q4.sql
@@ -4,6 +4,6 @@ SELECT
MAX(employee_id) OVER(PARTITION BY position_id) AS `last_value`
FROM (
SELECT *
- FROM dfs_test.`%s/window/b4.p4`
+ FROM dfs.`window/b4.p4`
ORDER BY position_id, employee_id
)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/resources/writer/simple_csv_writer.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/writer/simple_csv_writer.json b/exec/java-exec/src/test/resources/writer/simple_csv_writer.json
index 7980eb5..b3c215a 100644
--- a/exec/java-exec/src/test/resources/writer/simple_csv_writer.json
+++ b/exec/java-exec/src/test/resources/writer/simple_csv_writer.json
@@ -27,17 +27,17 @@
@id: 2,
child: 1,
pop: "fs-writer",
- "location" : "/tmp/csvtest",
+ "location" : "%TEST_DIR%",
"storage" : {
"type" : "file",
"connection" : "file:///",
"workspaces" : {
"root" : {
- "location" : "/",
+ "location" : "%ROOT_DIR%",
"writable" : false
},
"tmp" : {
- "location" : "/tmp",
+ "location" : "%TMP_DIR%",
"writable" : true
}
},
[07/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
index 1be8352..463021f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
@@ -17,19 +17,15 @@
*/
package org.apache.drill.exec.store.parquet;
-import com.google.common.io.Resources;
import mockit.Mock;
import mockit.MockUp;
-import mockit.integration.junit4.JMockit;
import com.google.common.collect.Lists;
+import mockit.integration.junit4.JMockit;
import org.apache.drill.PlanTestBase;
import org.apache.drill.categories.UnlikelyTest;
-import org.apache.drill.common.util.TestTools;
import org.apache.commons.io.FileUtils;
import org.apache.drill.exec.store.dfs.MetadataContext;
import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -37,7 +33,10 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.io.File;
+import java.io.IOException;
import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.List;
import static org.junit.Assert.assertEquals;
@@ -46,42 +45,23 @@ import static org.junit.Assert.assertTrue;
@RunWith(JMockit.class)
public class TestParquetMetadataCache extends PlanTestBase {
- private static final String WORKING_PATH = TestTools.getWorkingPath();
- private static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
- private static final String tableName1 = "parquetTable1";
- private static final String tableName2 = "parquetTable2";
- private static File dataDir1;
- private static File dataDir2;
-
+ private static final String TABLE_NAME_1 = "parquetTable1";
+ private static final String TABLE_NAME_2 = "parquetTable2";
@BeforeClass
public static void copyData() throws Exception {
- // copy the data into the temporary location
- String tmpLocation = getDfsTestTmpSchemaLocation();
- dataDir1 = new File(tmpLocation, tableName1);
- dataDir1.mkdir();
- FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet", TEST_RES_PATH))),
- dataDir1);
-
- dataDir2 = new File(tmpLocation, tableName2);
- dataDir2.mkdir();
- FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet2", TEST_RES_PATH))),
- dataDir2);
- }
-
- @AfterClass
- public static void cleanupTestData() throws Exception {
- FileUtils.deleteQuietly(dataDir1);
- FileUtils.deleteQuietly(dataDir2);
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel/parquet"), Paths.get(TABLE_NAME_1));
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel/parquet2"), Paths.get(TABLE_NAME_2));
+ dirTestWatcher.copyResourceToRoot(Paths.get("parquet"));
}
@Test
public void testPartitionPruningWithMetadataCache_1() throws Exception {
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName1));
- checkForMetadataFile(tableName1);
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir0=1994 and dir1 in ('Q1', 'Q2')",
- getDfsTestTmpSchemaLocation(), tableName1);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_1);
+ checkForMetadataFile(TABLE_NAME_1);
+ String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir0=1994 and dir1 in ('Q1', 'Q2')", TABLE_NAME_1);
int expectedRowCount = 20;
int expectedNumFiles = 2;
@@ -89,7 +69,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1994", getDfsTestTmpSchemaLocation(), tableName1);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1994", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_1);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {});
}
@@ -97,11 +77,10 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test // DRILL-3917, positive test case for DRILL-4530
@Category(UnlikelyTest.class)
public void testPartitionPruningWithMetadataCache_2() throws Exception {
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName1));
- checkForMetadataFile(tableName1);
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir0=1994",
- getDfsTestTmpSchemaLocation(), tableName1);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_1);
+ checkForMetadataFile(TABLE_NAME_1);
+ String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir0=1994", TABLE_NAME_1);
int expectedRowCount = 40;
int expectedNumFiles = 4;
@@ -109,7 +88,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1994", getDfsTestTmpSchemaLocation(), tableName1);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1994", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_1);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {"Filter"});
}
@@ -118,11 +97,10 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testPartitionPruningWithMetadataCache_3() throws Exception {
String tableName = "orders_ctas_varchar";
- test("use dfs_test.tmp");
-
- test(String.format("create table %s (o_orderdate, o_orderpriority) partition by (o_orderpriority) "
- + "as select o_orderdate, o_orderpriority from dfs_test.`%s/multilevel/parquet/1994/Q1`", tableName, TEST_RES_PATH));
- test(String.format("refresh table metadata %s", tableName));
+ test("use dfs");
+ test("create table %s (o_orderdate, o_orderpriority) partition by (o_orderpriority) "
+ + "as select o_orderdate, o_orderpriority from dfs.`multilevel/parquet/1994/Q1`", tableName);
+ test("refresh table metadata %s", tableName);
checkForMetadataFile(tableName);
String query = String.format("select * from %s where o_orderpriority = '1-URGENT'", tableName);
int expectedRowCount = 3;
@@ -140,12 +118,12 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testPartitionPruningWithMetadataCache_4() throws Exception {
String tableName = "orders_ctas_binary";
- test("use dfs_test.tmp");
+ test("use dfs");
- test(String.format("create table %s (o_orderdate, o_orderpriority) partition by (o_orderpriority) "
+ test("create table %s (o_orderdate, o_orderpriority) partition by (o_orderpriority) "
+ "as select o_orderdate, convert_to(o_orderpriority, 'UTF8') as o_orderpriority "
- + "from dfs_test.`%s/multilevel/parquet/1994/Q1`", tableName, TEST_RES_PATH));
- test(String.format("refresh table metadata %s", tableName));
+ + "from dfs.`multilevel/parquet/1994/Q1`", tableName);
+ test("refresh table metadata %s", tableName);
checkForMetadataFile(tableName);
String query = String.format("select * from %s where o_orderpriority = '1-URGENT'", tableName);
int expectedRowCount = 3;
@@ -162,10 +140,10 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test
public void testCache() throws Exception {
String tableName = "nation_ctas";
- test("use dfs_test.tmp");
- test(String.format("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName));
- test(String.format("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName));
- test(String.format("refresh table metadata %s", tableName));
+ test("use dfs");
+ test("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName);
+ test("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName);
+ test("refresh table metadata %s", tableName);
checkForMetadataFile(tableName);
String query = String.format("select * from %s", tableName);
int rowCount = testSql(query);
@@ -176,12 +154,12 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test
public void testUpdate() throws Exception {
String tableName = "nation_ctas_update";
- test("use dfs_test.tmp");
- test(String.format("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName));
- test(String.format("refresh table metadata %s", tableName));
+ test("use dfs");
+ test("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName);
+ test("refresh table metadata %s", tableName);
checkForMetadataFile(tableName);
Thread.sleep(1000);
- test(String.format("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName));
+ test("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName);
int rowCount = testSql(String.format("select * from %s", tableName));
Assert.assertEquals(50, rowCount);
}
@@ -189,23 +167,23 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test
public void testCacheWithSubschema() throws Exception {
String tableName = "nation_ctas_subschema";
- test(String.format("create table dfs_test.tmp.`%s/t1` as select * from cp.`tpch/nation.parquet`", tableName));
- test(String.format("refresh table metadata dfs_test.tmp.%s", tableName));
+ test("create table dfs.`%s/t1` as select * from cp.`tpch/nation.parquet`", tableName);
+ test("refresh table metadata dfs.%s", tableName);
checkForMetadataFile(tableName);
- int rowCount = testSql(String.format("select * from dfs_test.tmp.%s", tableName));
+ int rowCount = testSql(String.format("select * from dfs.%s", tableName));
Assert.assertEquals(25, rowCount);
}
@Test
public void testFix4449() throws Exception {
- runSQL("CREATE TABLE dfs_test.tmp.`4449` PARTITION BY(l_discount) AS SELECT l_orderkey, l_discount FROM cp.`tpch/lineitem.parquet`");
- runSQL("REFRESH TABLE METADATA dfs_test.tmp.`4449`");
+ runSQL("CREATE TABLE dfs.tmp.`4449` PARTITION BY(l_discount) AS SELECT l_orderkey, l_discount FROM cp.`tpch/lineitem.parquet`");
+ runSQL("REFRESH TABLE METADATA dfs.tmp.`4449`");
testBuilder()
.sqlQuery("SELECT COUNT(*) cnt FROM (" +
- "SELECT l_orderkey FROM dfs_test.tmp.`4449` WHERE l_discount < 0.05" +
+ "SELECT l_orderkey FROM dfs.tmp.`4449` WHERE l_discount < 0.05" +
" UNION ALL" +
- " SELECT l_orderkey FROM dfs_test.tmp.`4449` WHERE l_discount > 0.02)")
+ " SELECT l_orderkey FROM dfs.tmp.`4449` WHERE l_discount > 0.02)")
.unOrdered()
.baselineColumns("cnt")
.baselineValues(71159L)
@@ -215,10 +193,10 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test
public void testAbsentPluginOrWorkspaceError() throws Exception {
testBuilder()
- .sqlQuery("refresh table metadata dfs_test.incorrect.table_name")
+ .sqlQuery("refresh table metadata dfs.incorrect.table_name")
.unOrdered()
.baselineColumns("ok", "summary")
- .baselineValues(false, "Storage plugin or workspace does not exist [dfs_test.incorrect]")
+ .baselineValues(false, "Storage plugin or workspace does not exist [dfs.incorrect]")
.go();
testBuilder()
@@ -232,15 +210,17 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test //DRILL-4511
@Category(UnlikelyTest.class)
public void testTableDoesNotExistWithEmptyDirectory() throws Exception {
- File path = new File(getTempDir("empty_directory"));
- String pathString = path.toURI().getPath();
+ String dirName = "empty_directory";
+ File path = new File(dirTestWatcher.getRootDir(), dirName);
+ path.mkdirs();
+
try {
path.mkdir();
testBuilder()
- .sqlQuery("refresh table metadata dfs.`%s`", pathString)
+ .sqlQuery("refresh table metadata dfs.`%s`", dirName)
.unOrdered()
.baselineColumns("ok", "summary")
- .baselineValues(false, String.format("Table %s does not exist.", pathString))
+ .baselineValues(false, String.format("Table %s does not exist.", dirName))
.go();
} finally {
FileUtils.deleteQuietly(path);
@@ -274,13 +254,12 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testDrill4530_1() throws Exception {
// create metadata cache
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
- checkForMetadataFile(tableName2);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_2);
+ checkForMetadataFile(TABLE_NAME_2);
// run query and check correctness
- String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir0=1995 and dir1='Q3'",
- getDfsTestTmpSchemaLocation(), tableName2);
+ String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir0=1995 and dir1='Q3'", TABLE_NAME_2);
int expectedRowCount = 20;
int expectedNumFiles = 2;
@@ -288,7 +267,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995/Q3", getDfsTestTmpSchemaLocation(), tableName2);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995/Q3", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2);
PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {"Filter"});
}
@@ -297,13 +276,12 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testDrill4530_2() throws Exception {
// create metadata cache
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
- checkForMetadataFile(tableName2);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_2);
+ checkForMetadataFile(TABLE_NAME_2);
// run query and check correctness
- String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir0=1995",
- getDfsTestTmpSchemaLocation(), tableName2);
+ String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir0=1995", TABLE_NAME_2);
int expectedRowCount = 80;
int expectedNumFiles = 8;
@@ -311,7 +289,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995", getDfsTestTmpSchemaLocation(), tableName2);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2);
PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {"Filter"});
}
@@ -320,13 +298,12 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testDrill4530_3() throws Exception {
// create metadata cache
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
- checkForMetadataFile(tableName2);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_2);
+ checkForMetadataFile(TABLE_NAME_2);
// run query and check correctness
- String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir1='Q3'",
- getDfsTestTmpSchemaLocation(), tableName2);
+ String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir1='Q3'", TABLE_NAME_2);
int expectedRowCount = 40;
int expectedNumFiles = 4;
@@ -334,7 +311,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName2);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2);
PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {});
}
@@ -343,13 +320,12 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testDrill4530_4() throws Exception {
// create metadata cache
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
- checkForMetadataFile(tableName2);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_2);
+ checkForMetadataFile(TABLE_NAME_2);
// run query and check correctness
- String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir0=1995 and dir1='Q6'",
- getDfsTestTmpSchemaLocation(), tableName2);
+ String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir0=1995 and dir1='Q6'", TABLE_NAME_2);
int expectedRowCount = 0;
int expectedNumFiles = 1;
@@ -357,7 +333,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/*/*", getDfsTestTmpSchemaLocation(), tableName2);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/*/*", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2);
PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {});
}
@@ -365,11 +341,10 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test // DRILL-4794
@Category(UnlikelyTest.class)
public void testDrill4794() throws Exception {
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName1));
- checkForMetadataFile(tableName1);
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir0=1994 or dir1='Q3'",
- getDfsTestTmpSchemaLocation(), tableName1);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_1);
+ checkForMetadataFile(TABLE_NAME_1);
+ String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir0=1994 or dir1='Q3'", TABLE_NAME_1);
int expectedRowCount = 60;
int expectedNumFiles = 6;
@@ -378,7 +353,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName1);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_1);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {});
}
@@ -387,13 +362,12 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testDrill4786_1() throws Exception {
// create metadata cache
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
- checkForMetadataFile(tableName2);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_2);
+ checkForMetadataFile(TABLE_NAME_2);
// run query and check correctness
- String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir0=1995 and dir1 in ('Q1', 'Q2')",
- getDfsTestTmpSchemaLocation(), tableName2);
+ String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir0=1995 and dir1 in ('Q1', 'Q2')", TABLE_NAME_2);
int expectedRowCount = 40;
int expectedNumFiles = 4;
@@ -402,7 +376,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995", getDfsTestTmpSchemaLocation(), tableName2);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s/1995", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2);
PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {});
@@ -412,13 +386,12 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testDrill4786_2() throws Exception {
// create metadata cache
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
- checkForMetadataFile(tableName2);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_2);
+ checkForMetadataFile(TABLE_NAME_2);
// run query and check correctness
- String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/%s` " +
- " where dir0 in (1994, 1995) and dir1 = 'Q3'",
- getDfsTestTmpSchemaLocation(), tableName2);
+ String query1 = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs.`%s` " +
+ " where dir0 in (1994, 1995) and dir1 = 'Q3'", TABLE_NAME_2);
int expectedRowCount = 40;
int expectedNumFiles = 4;
@@ -427,22 +400,20 @@ public class TestParquetMetadataCache extends PlanTestBase {
assertEquals(expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName2);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2);
PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {});
-
}
@Test // DRILL-4877
@Category(UnlikelyTest.class)
public void testDrill4877() throws Exception {
// create metadata cache
- test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
- checkForMetadataFile(tableName2);
+ test("refresh table metadata dfs.`%s`", TABLE_NAME_2);
+ checkForMetadataFile(TABLE_NAME_2);
// run query and check correctness
- String query1 = String.format("select max(dir0) as max0, max(dir1) as max1 from dfs_test.`%s/%s` ",
- getDfsTestTmpSchemaLocation(), tableName2);
+ String query1 = String.format("select max(dir0) as max0, max(dir1) as max1 from dfs.`%s` ", TABLE_NAME_2);
testBuilder()
.sqlQuery(query1)
@@ -455,7 +426,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName2);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), TABLE_NAME_2);
PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {});
@@ -466,13 +437,13 @@ public class TestParquetMetadataCache extends PlanTestBase {
final String tableName = "nation_move";
final String newTableName = "nation_moved";
try {
- test("use dfs_test.tmp");
+ test("use dfs");
test("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName);
test("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName);
test("refresh table metadata %s", tableName);
checkForMetadataFile(tableName);
- File srcFile = new File(getDfsTestTmpSchemaLocation(), tableName);
- File dstFile = new File(getDfsTestTmpSchemaLocation(), newTableName);
+ File srcFile = new File(dirTestWatcher.getRootDir(), tableName);
+ File dstFile = new File(dirTestWatcher.getRootDir(), newTableName);
FileUtils.moveDirectory(srcFile, dstFile);
assertFalse("Cache file was not moved successfully", srcFile.exists());
int rowCount = testSql(String.format("select * from %s", newTableName));
@@ -484,28 +455,49 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test
public void testOldMetadataVersions() throws Exception {
- final String tablePath = "absolute_paths_metadata";
- String rootMetadataPath = new Path("parquet", "metadata_files_with_old_versions").toUri().getPath();
+ final Path tablePath = Paths.get("absolute_paths_metadata");
+ final Path rootMetadataPath = Paths.get("parquet", "metadata_files_with_old_versions");
// gets folders with different metadata cache versions
- String[] metadataPaths = new File(Resources.getResource(rootMetadataPath).getFile()).list();
+ String[] metadataPaths = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(rootMetadataPath)
+ .toFile()
+ .list();
+
for (String metadataPath : metadataPaths) {
try {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
// creating two inner directories to leverage METADATA_DIRECTORIES_FILENAME metadata file as well
- final String absolutePathsMetadataT1 = new Path(tablePath, "t1").toUri().getPath();
- final String absolutePathsMetadataT2 = new Path(tablePath, "t2").toUri().getPath();
+ final Path absolutePathsMetadataT1 = tablePath.resolve("t1");
+ final Path absolutePathsMetadataT2 = tablePath.resolve("t2");
+
String createQuery = "create table `%s` as select * from cp.`tpch/nation.parquet`";
+
test(createQuery, absolutePathsMetadataT1);
test(createQuery, absolutePathsMetadataT2);
- Path relativePath = new Path(rootMetadataPath, metadataPath);
- copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_directories.requires_replace.txt"),
- tablePath, Metadata.METADATA_DIRECTORIES_FILENAME);
- copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table.requires_replace.txt"),
- tablePath, Metadata.METADATA_FILENAME);
- copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table_t1.requires_replace.txt"),
- absolutePathsMetadataT1, Metadata.METADATA_FILENAME);
- copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table_t2.requires_replace.txt"),
- absolutePathsMetadataT2, Metadata.METADATA_FILENAME);
+
+ Path relativePath = rootMetadataPath.resolve(metadataPath);
+
+ File metaFile = dirTestWatcher.copyResourceToTestTmp(
+ relativePath.resolve("metadata_directories.requires_replace.txt"),
+ tablePath.resolve(Metadata.METADATA_DIRECTORIES_FILENAME));
+ dirTestWatcher.replaceMetaDataContents(metaFile, dirTestWatcher.getDfsTestTmpDir(),
+ null);
+ metaFile = dirTestWatcher.copyResourceToTestTmp(
+ relativePath.resolve("metadata_table.requires_replace.txt"),
+ tablePath.resolve(Metadata.METADATA_FILENAME));
+ dirTestWatcher.replaceMetaDataContents(metaFile, dirTestWatcher.getDfsTestTmpDir(),
+ null);
+ metaFile = dirTestWatcher.copyResourceToTestTmp(
+ relativePath.resolve("metadata_table_t1.requires_replace.txt"),
+ absolutePathsMetadataT1.resolve(Metadata.METADATA_FILENAME));
+ dirTestWatcher.replaceMetaDataContents(metaFile, dirTestWatcher.getDfsTestTmpDir(),
+ null);
+ metaFile = dirTestWatcher.copyResourceToTestTmp(
+ relativePath.resolve("metadata_table_t2.requires_replace.txt"),
+ absolutePathsMetadataT2.resolve(Metadata.METADATA_FILENAME));
+ dirTestWatcher.replaceMetaDataContents(metaFile, dirTestWatcher.getDfsTestTmpDir(),
+ null);
String query = String.format("select * from %s", tablePath);
int expectedRowCount = 50;
int expectedNumFiles = 1; // point to selectionRoot since no pruning is done in this query
@@ -514,7 +506,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tablePath);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getDfsTestTmpDir().getCanonicalPath(), tablePath);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[]{"Filter"});
} finally {
@@ -527,7 +519,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
public void testSpacesInMetadataCachePath() throws Exception {
final String pathWithSpaces = "path with spaces";
try {
- test("use dfs_test.tmp");
+ test("use dfs");
// creating multilevel table to store path with spaces in both metadata files (METADATA and METADATA_DIRECTORIES)
test("create table `%s` as select * from cp.`tpch/nation.parquet`", pathWithSpaces);
test("create table `%1$s/%1$s` as select * from cp.`tpch/nation.parquet`", pathWithSpaces);
@@ -541,7 +533,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=true";
- String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), pathWithSpaces);
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", dirTestWatcher.getRootDir().getCanonicalPath(), pathWithSpaces);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
new String[] {"Filter"});
} finally {
@@ -553,13 +545,15 @@ public class TestParquetMetadataCache extends PlanTestBase {
public void testFutureUnsupportedMetadataVersion() throws Exception {
final String unsupportedMetadataVersion = "unsupported_metadata_version";
try {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create table `%s` as select * from cp.`tpch/nation.parquet`", unsupportedMetadataVersion);
MetadataVersion lastVersion = MetadataVersion.Constants.SUPPORTED_VERSIONS.last();
// Get the future version, which is absent in MetadataVersions.SUPPORTED_VERSIONS set
String futureVersion = new MetadataVersion(lastVersion.getMajor() + 1, 0).toString();
- copyMetaDataCacheToTempWithReplacements("parquet/unsupported_metadata/unsupported_metadata_version.requires_replace.txt",
- unsupportedMetadataVersion, Metadata.METADATA_FILENAME, futureVersion);
+ File metaDataFile = dirTestWatcher.copyResourceToTestTmp(
+ Paths.get("parquet", "unsupported_metadata", "unsupported_metadata_version.requires_replace.txt"),
+ Paths.get(unsupportedMetadataVersion, Metadata.METADATA_FILENAME));
+ dirTestWatcher.replaceMetaDataContents(metaDataFile, dirTestWatcher.getDfsTestTmpDir(), futureVersion);
String query = String.format("select * from %s", unsupportedMetadataVersion);
int expectedRowCount = 25;
int expectedNumFiles = 1;
@@ -579,10 +573,11 @@ public class TestParquetMetadataCache extends PlanTestBase {
public void testCorruptedMetadataFile() throws Exception {
final String corruptedMetadata = "corrupted_metadata";
try {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create table `%s` as select * from cp.`tpch/nation.parquet`", corruptedMetadata);
- copyMetaDataCacheToTempReplacingInternalPaths("parquet/unsupported_metadata/" +
- "corrupted_metadata.requires_replace.txt", corruptedMetadata, Metadata.METADATA_FILENAME);
+ dirTestWatcher.copyResourceToTestTmp(
+ Paths.get("parquet", "unsupported_metadata", "corrupted_metadata.requires_replace.txt"),
+ Paths.get(corruptedMetadata, Metadata.METADATA_FILENAME));
String query = String.format("select * from %s", corruptedMetadata);
int expectedRowCount = 25;
int expectedNumFiles = 1;
@@ -602,10 +597,11 @@ public class TestParquetMetadataCache extends PlanTestBase {
public void testEmptyMetadataFile() throws Exception {
final String emptyMetadataFile = "empty_metadata_file";
try {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create table `%s` as select * from cp.`tpch/nation.parquet`", emptyMetadataFile);
- copyMetaDataCacheToTempReplacingInternalPaths("parquet/unsupported_metadata/" +
- "empty_metadata_file.requires_replace.txt", emptyMetadataFile, Metadata.METADATA_FILENAME);
+ dirTestWatcher.copyResourceToTestTmp(
+ Paths.get("parquet", "unsupported_metadata", "empty_metadata_file.requires_replace.txt"),
+ Paths.get(emptyMetadataFile, Metadata.METADATA_FILENAME));
String query = String.format("select * from %s", emptyMetadataFile);
int expectedRowCount = 25;
int expectedNumFiles = 1;
@@ -623,40 +619,35 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test
public void testRootMetadataFileIsAbsent() throws Exception {
- final String tmpDir = getDfsTestTmpSchemaLocation();
final String rootMetaCorruptedTable = "root_meta_corrupted_table";
- File dataDir = new File(tmpDir, rootMetaCorruptedTable);
- try {
- // copy the data into the temporary location, delete root metadata file
- dataDir.mkdir();
- FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet", TEST_RES_PATH))), dataDir);
+ File dataDir = dirTestWatcher.copyResourceToRoot(
+ Paths.get("multilevel", "parquet"),
+ Paths.get(rootMetaCorruptedTable));
- test("use dfs_test.tmp");
+ try {
+ test("use dfs");
test("refresh table metadata `%s`", rootMetaCorruptedTable);
checkForMetadataFile(rootMetaCorruptedTable);
- File rootMetadataFile = FileUtils.getFile(tmpDir, rootMetaCorruptedTable, Metadata.METADATA_FILENAME);
- assertTrue(String.format("Metadata cache file '%s' isn't deleted", rootMetadataFile.getPath()),
- rootMetadataFile.delete());
+ File rootMetadataFile = FileUtils.getFile(dataDir, Metadata.METADATA_FILENAME);
+ assertTrue(String.format("Metadata cache file '%s' isn't deleted", rootMetadataFile.getPath()), rootMetadataFile.delete());
// mock Metadata tableModified method to avoid occasional metadata files updating
new MockUp<Metadata>() {
@Mock
- boolean tableModified(List<String> directories, Path metaFilePath, Path parentDir, MetadataContext metaContext) {
+ boolean tableModified(List<String> directories, org.apache.hadoop.fs.Path metaFilePath,
+ org.apache.hadoop.fs.Path parentDir, MetadataContext metaContext) {
return false;
}
};
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from `%s` " +
- " where dir0=1994 or dir1='Q3'", rootMetaCorruptedTable);
+ String query = String.format("select dir0, dir1, o_custkey, o_orderdate from `%s` " + " where dir0=1994 or dir1='Q3'", rootMetaCorruptedTable);
int expectedRowCount = 60;
int expectedNumFiles = 6;
int actualRowCount = testSql(query);
- assertEquals("An incorrect result was obtained while querying a table with metadata cache files",
- expectedRowCount, actualRowCount);
+ assertEquals("An incorrect result was obtained while querying a table with metadata cache files", expectedRowCount, actualRowCount);
String numFilesPattern = "numFiles=" + expectedNumFiles;
String usedMetaPattern = "usedMetadataFile=false";
- PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern},
- new String[] {"cacheFileRoot", "Filter"});
+ PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern}, new String[]{"cacheFileRoot", "Filter"});
} finally {
FileUtils.deleteQuietly(dataDir);
}
@@ -664,19 +655,17 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test
public void testInnerMetadataFilesAreAbsent() throws Exception {
- final String tmpDir = getDfsTestTmpSchemaLocation();
final String innerMetaCorruptedTable = "inner_meta_corrupted_table";
- File dataDir = new File(tmpDir, innerMetaCorruptedTable);
- try {
- // copy the data into the temporary location, delete a few inner metadata files
- dataDir.mkdir();
- FileUtils.copyDirectory(new File(String.format(String.format("%s/multilevel/parquet", TEST_RES_PATH))), dataDir);
+ File dataDir = dirTestWatcher.copyResourceToRoot(
+ Paths.get("multilevel", "parquet"),
+ Paths.get(innerMetaCorruptedTable));
- test("use dfs_test.tmp");
+ try {
+ test("use dfs");
test("refresh table metadata `%s`", innerMetaCorruptedTable);
checkForMetadataFile(innerMetaCorruptedTable);
- File firstInnerMetadataFile = FileUtils.getFile(tmpDir, innerMetaCorruptedTable, "1994", Metadata.METADATA_FILENAME);
- File secondInnerMetadataFile = FileUtils.getFile(tmpDir, innerMetaCorruptedTable, "1994", "Q3", Metadata.METADATA_FILENAME);
+ File firstInnerMetadataFile = FileUtils.getFile(dataDir, "1994", Metadata.METADATA_FILENAME);
+ File secondInnerMetadataFile = FileUtils.getFile(dataDir, "1994", "Q3", Metadata.METADATA_FILENAME);
assertTrue(String.format("Metadata cache file '%s' isn't deleted", firstInnerMetadataFile.getPath()),
firstInnerMetadataFile.delete());
assertTrue(String.format("Metadata cache file '%s' isn't deleted", secondInnerMetadataFile.getPath()),
@@ -685,7 +674,8 @@ public class TestParquetMetadataCache extends PlanTestBase {
// mock Metadata tableModified method to avoid occasional metadata files updating
new MockUp<Metadata>() {
@Mock
- boolean tableModified(List<String> directories, Path metaFilePath, Path parentDir, MetadataContext metaContext) {
+ boolean tableModified(List<String> directories, org.apache.hadoop.fs.Path metaFilePath,
+ org.apache.hadoop.fs.Path parentDir, MetadataContext metaContext) {
return false;
}
};
@@ -709,7 +699,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test // DRILL-4264
@Category(UnlikelyTest.class)
public void testMetadataCacheFieldWithDots() throws Exception {
- final String tableWithDots = "dfs_test.tmp.`complex_table`";
+ final String tableWithDots = "dfs.tmp.`complex_table`";
try {
test("create table %s as\n" +
"select cast(1 as int) as `column.with.dots`, t.`column`.`with.dots`\n" +
@@ -734,7 +724,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test // DRILL-4139
public void testBooleanPartitionPruning() throws Exception {
- final String boolPartitionTable = "dfs_test.tmp.`interval_bool_partition`";
+ final String boolPartitionTable = "dfs.tmp.`interval_bool_partition`";
try {
test("create table %s partition by (col_bln) as " +
"select * from cp.`parquet/alltypes_required.parquet`", boolPartitionTable);
@@ -758,7 +748,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test // DRILL-4139
public void testIntervalDayPartitionPruning() throws Exception {
- final String intervalDayPartitionTable = "dfs_test.tmp.`interval_day_partition`";
+ final String intervalDayPartitionTable = "dfs.tmp.`interval_day_partition`";
try {
test("create table %s partition by (col_intrvl_day) as " +
"select * from cp.`parquet/alltypes_optional.parquet`", intervalDayPartitionTable);
@@ -783,7 +773,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test // DRILL-4139
public void testIntervalYearPartitionPruning() throws Exception {
- final String intervalYearPartitionTable = "dfs_test.tmp.`interval_yr_partition`";
+ final String intervalYearPartitionTable = "dfs.tmp.`interval_yr_partition`";
try {
test("create table %s partition by (col_intrvl_yr) as " +
"select * from cp.`parquet/alltypes_optional.parquet`", intervalYearPartitionTable);
@@ -808,7 +798,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test // DRILL-4139
public void testVarCharWithNullsPartitionPruning() throws Exception {
- final String intervalYearPartitionTable = "dfs_test.tmp.`varchar_optional_partition`";
+ final String intervalYearPartitionTable = "dfs.tmp.`varchar_optional_partition`";
try {
test("create table %s partition by (col_vrchr) as " +
"select * from cp.`parquet/alltypes_optional.parquet`", intervalYearPartitionTable);
@@ -845,7 +835,7 @@ public class TestParquetMetadataCache extends PlanTestBase {
ctasQueries.add("create table %s partition by (manager_id) as " +
"select cast(manager_id as decimal(18, 6)) as manager_id, EMPLOYEE_ID, FIRST_NAME, LAST_NAME " +
"from cp.`parquet/fixedlenDecimal.parquet`");
- final String decimalPartitionTable = "dfs_test.tmp.`decimal_optional_partition`";
+ final String decimalPartitionTable = "dfs.tmp.`decimal_optional_partition`";
for (String ctasQuery : ctasQueries) {
try {
test("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY);
@@ -873,76 +863,76 @@ public class TestParquetMetadataCache extends PlanTestBase {
@Test // DRILL-4139
public void testIntWithNullsPartitionPruning() throws Exception {
try {
- test("create table dfs_test.tmp.`t5/a` as\n" +
+ test("create table dfs.tmp.`t5/a` as\n" +
"select 100 as mykey from cp.`tpch/nation.parquet`\n" +
"union all\n" +
"select col_notexist from cp.`tpch/region.parquet`");
- test("create table dfs_test.tmp.`t5/b` as\n" +
+ test("create table dfs.tmp.`t5/b` as\n" +
"select 200 as mykey from cp.`tpch/nation.parquet`\n" +
"union all\n" +
"select col_notexist from cp.`tpch/region.parquet`");
- String query = "select mykey from dfs_test.tmp.`t5` where mykey = 100";
+ String query = "select mykey from dfs.tmp.`t5` where mykey = 100";
int actualRowCount = testSql(query);
assertEquals("Row count does not match the expected value", 25, actualRowCount);
- test("refresh table metadata dfs_test.tmp.`t5`");
+ test("refresh table metadata dfs.tmp.`t5`");
actualRowCount = testSql(query);
assertEquals("Row count does not match the expected value", 25, actualRowCount);
} finally {
- test("drop table if exists dfs_test.tmp.`t5`");
+ test("drop table if exists dfs.tmp.`t5`");
}
}
@Test // DRILL-4139
public void testPartitionPruningWithIsNull() throws Exception {
try {
- test("create table dfs_test.tmp.`t6/a` as\n" +
+ test("create table dfs.tmp.`t6/a` as\n" +
"select col_notexist as mykey from cp.`tpch/region.parquet`");
- test("create table dfs_test.tmp.`t6/b` as\n" +
+ test("create table dfs.tmp.`t6/b` as\n" +
"select 100 as mykey from cp.`tpch/region.parquet`");
- String query = "select mykey from dfs_test.tmp.t6 where mykey is null";
+ String query = "select mykey from dfs.tmp.t6 where mykey is null";
int actualRowCount = testSql(query);
assertEquals("Row count does not match the expected value", 5, actualRowCount);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=false"}, new String[]{"Filter"});
- test("refresh table metadata dfs_test.tmp.`t6`");
+ test("refresh table metadata dfs.tmp.`t6`");
actualRowCount = testSql(query);
assertEquals("Row count does not match the expected value", 5, actualRowCount);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=true"}, new String[]{"Filter"});
} finally {
- test("drop table if exists dfs_test.tmp.`t6`");
+ test("drop table if exists dfs.tmp.`t6`");
}
}
@Test // DRILL-4139
public void testPartitionPruningWithIsNotNull() throws Exception {
try {
- test("create table dfs_test.tmp.`t7/a` as\n" +
+ test("create table dfs.tmp.`t7/a` as\n" +
"select col_notexist as mykey from cp.`tpch/region.parquet`");
- test("create table dfs_test.tmp.`t7/b` as\n" +
+ test("create table dfs.tmp.`t7/b` as\n" +
"select 100 as mykey from cp.`tpch/region.parquet`");
- String query = "select mykey from dfs_test.tmp.t7 where mykey is null";
+ String query = "select mykey from dfs.tmp.t7 where mykey is null";
int actualRowCount = testSql(query);
assertEquals("Row count does not match the expected value", 5, actualRowCount);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=false"}, new String[]{"Filter"});
- test("refresh table metadata dfs_test.tmp.`t7`");
+ test("refresh table metadata dfs.tmp.`t7`");
actualRowCount = testSql(query);
assertEquals("Row count does not match the expected value", 5, actualRowCount);
PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=true"}, new String[]{"Filter"});
} finally {
- test("drop table if exists dfs_test.tmp.`t7`");
+ test("drop table if exists dfs.tmp.`t7`");
}
}
@@ -952,7 +942,14 @@ public class TestParquetMetadataCache extends PlanTestBase {
* @param table table name or table path
*/
private void checkForMetadataFile(String table) {
- String tmpDir = getDfsTestTmpSchemaLocation();
+ final String tmpDir;
+
+ try {
+ tmpDir = dirTestWatcher.getRootDir().getCanonicalPath();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
File metaFile = table.startsWith(tmpDir) ? FileUtils.getFile(table, Metadata.METADATA_FILENAME)
: FileUtils.getFile(tmpDir, table, Metadata.METADATA_FILENAME);
assertTrue(String.format("There is no metadata cache file for the %s table", table),
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java
index 39df4e0..c95b201 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetScan.java
@@ -18,41 +18,29 @@
package org.apache.drill.exec.store.parquet;
import com.google.common.io.Resources;
-import org.apache.drill.BaseTestQuery;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.BeforeClass;
+import org.apache.commons.io.FileUtils;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
-public class TestParquetScan extends BaseTestQuery {
-
- static FileSystem fs;
-
- @BeforeClass
- public static void initFs() throws Exception {
- fs = getLocalFileSystem();
- }
+import java.io.File;
+import java.nio.file.Path;
+public class TestParquetScan extends BaseTestQuery {
@Test
public void testSuccessFile() throws Exception {
- Path p = new Path("/tmp/nation_test_parquet_scan");
- if (fs.exists(p)) {
- fs.delete(p, true);
- }
-
- fs.mkdirs(p);
+ final byte[] bytes = Resources.toByteArray(Resources.getResource("tpch/nation.parquet"));
- byte[] bytes = Resources.toByteArray(Resources.getResource("tpch/nation.parquet"));
+ final Path rootPath = dirTestWatcher.getRootDir().toPath();
+ final File scanFile = rootPath.resolve("nation_test_parquet_scan").toFile();
+ final File successFile = rootPath.resolve("_SUCCESS").toFile();
+ final File logsFile = rootPath.resolve("_logs").toFile();
- FSDataOutputStream os = fs.create(new Path(p, "nation.parquet"));
- os.write(bytes);
- os.close();
- fs.create(new Path(p, "_SUCCESS")).close();
- fs.create(new Path(p, "_logs")).close();
+ FileUtils.writeByteArrayToFile(scanFile, bytes);
+ successFile.createNewFile();
+ logsFile.createNewFile();
testBuilder()
- .sqlQuery("select count(*) c from dfs.tmp.nation_test_parquet_scan where 1 = 1")
+ .sqlQuery("select count(*) c from dfs.nation_test_parquet_scan where 1 = 1")
.unOrdered()
.baselineColumns("c")
.baselineValues(25L)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
index bfd894d..cf11d5f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.store.parquet.columnreaders;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.junit.AfterClass;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java
index e95842c..b641707 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestDateReader.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.store.parquet.columnreaders;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
public class TestDateReader extends BaseTestQuery {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
index 477b825..ec2651d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.store.parquet2;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -75,12 +75,12 @@ public class TestDrillParquetReader extends BaseTestQuery {
@Test
public void test4349() throws Exception {
// start by creating a parquet file from the input csv file
- runSQL("CREATE TABLE dfs_test.tmp.`4349` AS SELECT columns[0] id, CAST(NULLIF(columns[1], '') AS DOUBLE) val FROM cp.`parquet2/4349.csv.gz`");
+ runSQL("CREATE TABLE dfs.tmp.`4349` AS SELECT columns[0] id, CAST(NULLIF(columns[1], '') AS DOUBLE) val FROM cp.`parquet2/4349.csv.gz`");
// querying the parquet file should return the same results found in the csv file
testBuilder()
.unOrdered()
- .sqlQuery("SELECT * FROM dfs_test.tmp.`4349` WHERE id = 'b'")
+ .sqlQuery("SELECT * FROM dfs.tmp.`4349` WHERE id = 'b'")
.sqlBaselineQuery("SELECT columns[0] id, CAST(NULLIF(columns[1], '') AS DOUBLE) val FROM cp.`parquet2/4349.csv.gz` WHERE columns[0] = 'b'")
.go();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java
index 56c2dbe..6e58ccf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoder.java
@@ -17,10 +17,9 @@
package org.apache.drill.exec.store.pcap;
import com.google.common.io.Resources;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.store.pcap.decoder.Packet;
import org.apache.drill.exec.store.pcap.decoder.PacketDecoder;
-import org.junit.BeforeClass;
import org.junit.Test;
import java.io.BufferedInputStream;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java
index ce2a4f8..706694d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java
@@ -16,33 +16,39 @@
*/
package org.apache.drill.exec.store.pcap;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
+import java.nio.file.Paths;
import java.util.List;
public class TestPcapRecordReader extends BaseTestQuery {
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("store", "pcap"));
+ }
@Test
public void testStarQuery() throws Exception {
- runSQLVerifyCount("select * from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 16);
- runSQLVerifyCount("select distinct DST_IP from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1);
- runSQLVerifyCount("select distinct DsT_IP from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1);
- runSQLVerifyCount("select distinct dst_ip from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1);
+ runSQLVerifyCount("select * from dfs.`store/pcap/tcp-1.pcap`", 16);
+ runSQLVerifyCount("select distinct DST_IP from dfs.`store/pcap/tcp-1.pcap`", 1);
+ runSQLVerifyCount("select distinct DsT_IP from dfs.`store/pcap/tcp-1.pcap`", 1);
+ runSQLVerifyCount("select distinct dst_ip from dfs.`store/pcap/tcp-1.pcap`", 1);
}
@Test
public void testCountQuery() throws Exception {
- runSQLVerifyCount("select count(*) from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1);
- runSQLVerifyCount("select count(*) from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-2.pcap`", 1);
+ runSQLVerifyCount("select count(*) from dfs.`store/pcap/tcp-1.pcap`", 1);
+ runSQLVerifyCount("select count(*) from dfs.`store/pcap/tcp-2.pcap`", 1);
}
@Test
public void testDistinctQuery() throws Exception {
- runSQLVerifyCount("select distinct * from dfs.`${WORKING_PATH}/src/test/resources/store/pcap/tcp-1.pcap`", 1);
+ runSQLVerifyCount("select distinct * from dfs.`store/pcap/tcp-1.pcap`", 1);
}
private void runSQLVerifyCount(String sql, int expectedRowCount) throws Exception {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java
index f48eb2c..6d3ce64 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sequencefile/TestSequenceFileReader.java
@@ -21,8 +21,7 @@ import java.io.DataOutputStream;
import java.io.ByteArrayOutputStream;
import org.junit.Test;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.hadoop.io.BytesWritable;
public class TestSequenceFileReader extends BaseTestQuery {
@@ -37,11 +36,9 @@ public class TestSequenceFileReader extends BaseTestQuery {
@Test
public void testSequenceFileReader() throws Exception {
- String root = FileUtils.getResourceAsFile("/sequencefiles/simple.seq").toURI().toString();
- final String query = String.format("select convert_from(t.binary_key, 'UTF8') as k, convert_from(t.binary_value, 'UTF8') as v " +
- "from dfs_test.`%s` t", root);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select convert_from(t.binary_key, 'UTF8') as k, convert_from(t.binary_value, 'UTF8') as v " +
+ "from cp.`sequencefiles/simple.seq` t")
.ordered()
.baselineColumns("k", "v")
.baselineValues(byteWritableString("key0"), byteWritableString("value0"))
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java
index bcd2f5e..d3116c7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java
@@ -41,7 +41,10 @@ public class PStoreTestUtil {
expectedMap.put(keys[i], values[i]);
store.put(keys[i], values[i]);
}
- // allow one second for puts to propagate back to cache
+
+ // Wait for store caches to update, this is necessary because ZookeeperClient caches can update asynchronously in some cases.
+ waitForNumProps(store, keys.length);
+
{
Iterator<Map.Entry<String, String>> iter = store.getAll();
for(int i =0; i < keys.length; i++){
@@ -61,8 +64,29 @@ public class PStoreTestUtil {
}
}
- // allow one second for deletes to propagate back to cache
-
+ // Wait for store caches to update, this is necessary because ZookeeperClient caches can update asynchronously in some cases.
+ waitForNumProps(store, 0);
assertFalse(store.getAll().hasNext());
}
+
+ private static void waitForNumProps(PersistentStore store, int expected) throws InterruptedException {
+ for (int numProps = getNumProps(store);
+ numProps < expected;
+ numProps = getNumProps(store)) {
+ Thread.sleep(100L);
+ }
+ }
+
+ private static int getNumProps(PersistentStore store) {
+ Iterator<Map.Entry<String, String>> iter = store.getAll();
+
+ int numProps = 0;
+
+ while (iter.hasNext()) {
+ iter.next();
+ numProps++;
+ }
+
+ return numProps;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
index e7b150c..7325fd7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
@@ -22,8 +22,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryNTimes;
+import org.apache.drill.categories.SlowTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.TestWithZookeeper;
import org.apache.drill.exec.coord.zk.PathUtils;
@@ -32,22 +33,22 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.server.options.PersistedOptionValue;
import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
import org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider;
+import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterFixtureBuilder;
-import org.apache.drill.testutils.DirTestWatcher;
import org.apache.zookeeper.CreateMode;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import java.io.File;
+@Category(SlowTest.class)
public class TestPStoreProviders extends TestWithZookeeper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestPStoreProviders.class);
-
@Rule
- public DirTestWatcher dirTestWatcher = new DirTestWatcher();
+ public BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
@Test
public void verifyLocalStore() throws Exception {
@@ -79,18 +80,14 @@ public class TestPStoreProviders extends TestWithZookeeper {
private void localLoadTestHelper(String propertiesDir) throws Exception {
File localOptionsResources = new File(propertiesDir);
- String optionsDirPath = String.format("%s/sys.options", dirTestWatcher.getDirPath());
- File optionsDir = new File(optionsDirPath);
- optionsDir.mkdirs();
-
- org.apache.commons.io.FileUtils.copyDirectory(localOptionsResources, optionsDir);
-
- ClusterFixtureBuilder builder = ClusterFixture.builder().
- configProperty(ExecConstants.HTTP_ENABLE, false).
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).
configProperty(ExecConstants.SYS_STORE_PROVIDER_CLASS, LocalPersistentStoreProvider.class.getCanonicalName()).
- configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, String.format("file://%s", dirTestWatcher.getDirPath())).
configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true);
+ File optionsDir = new File(dirTestWatcher.getStoreDir(), "sys.options");
+ optionsDir.mkdirs();
+ org.apache.commons.io.FileUtils.copyDirectory(localOptionsResources, optionsDir);
+
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String parquetPushdown = client.queryBuilder().
@@ -129,7 +126,7 @@ public class TestPStoreProviders extends TestWithZookeeper {
try (ZookeeperClient zkClient = new ZookeeperClient(curator,
PathUtils.join("/", storeConfig.getName()), CreateMode.PERSISTENT)) {
zkClient.start();
- String oldOptionJson = FileUtils.getResourceAsString("/options/old_booleanopt.json");
+ String oldOptionJson = DrillFileUtils.getResourceAsString("/options/old_booleanopt.json");
zkClient.put(oldName, oldOptionJson.getBytes(), null);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
index dff1504..c6c5a05 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestSystemTable.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.store.sys;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.exec.ExecConstants;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java
index cf54bb0..2685496 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestCsvHeader.java
@@ -18,13 +18,13 @@
package org.apache.drill.exec.store.text;
import com.google.common.collect.Lists;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.TestBuilder;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestBuilder;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
+import java.nio.file.Paths;
import java.util.List;
import org.junit.Before;
@@ -32,20 +32,20 @@ import org.junit.Test;
public class TestCsvHeader extends BaseTestQuery{
- private String root;
+ private static final String ROOT = "store/text/data/cars.csvh";
@Before
public void initialize() throws Exception {
- root = FileUtils.getResourceAsFile("/store/text/data/cars.csvh").toURI().toString();
+ dirTestWatcher.copyResourceToRoot(Paths.get("store", "text", "data"));
+
test("alter session set `exec.errors.verbose` = true ");
}
@Test //DRILL-951
public void testCsvWithHeader() throws Exception {
- //Pre DRILL-951: Qry: select * from dfs_test.`%s` LIMIT 2
- String query = String.format("select * from dfs_test.`%s` LIMIT 2", root);
+ //Pre DRILL-951: Qry: select * from cp.`%s` LIMIT 2
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from cp.`%s` LIMIT 2", ROOT)
.unOrdered()
.baselineColumns("Year", "Make", "Model", "Description", "Price")
.baselineValues("1997", "Ford", "E350", "ac, abs, moon", "3000.00")
@@ -55,11 +55,9 @@ public class TestCsvHeader extends BaseTestQuery{
@Test //DRILL-951
public void testCsvWhereWithHeader() throws Exception {
- //Pre DRILL-951: Qry: select * from dfs_test.`%s` where columns[1] = 'Chevy'
- String query = String.format("select * from dfs_test.`%s` where Make = 'Chevy'", root);
-
+ //Pre DRILL-951: Qry: select * from cp.`%s` where columns[1] = 'Chevy'
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from cp.`%s` where Make = 'Chevy'", ROOT)
.unOrdered()
.baselineColumns("Year", "Make", "Model", "Description", "Price")
.baselineValues("1999", "Chevy", "Venture \"Extended Edition\"", "", "4900.00")
@@ -69,11 +67,9 @@ public class TestCsvHeader extends BaseTestQuery{
@Test //DRILL-951
public void testCsvStarPlusWithHeader() throws Exception {
- //Pre DRILL-951: Qry: select *,columns[1] from dfs_test.`%s` where columns[1] = 'Chevy'
- String query = String.format("select *, Make from dfs_test.`%s` where Make = 'Chevy'", root);
-
+ //Pre DRILL-951: Qry: select *,columns[1] from cp.`%s` where columns[1] = 'Chevy'
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select *, Make from cp.`%s` where Make = 'Chevy'", ROOT)
.unOrdered()
.baselineColumns("Year", "Make", "Model", "Description", "Price", "Make0")
.baselineValues("1999", "Chevy", "Venture \"Extended Edition\"", "", "4900.00", "Chevy")
@@ -83,10 +79,9 @@ public class TestCsvHeader extends BaseTestQuery{
@Test //DRILL-951
public void testCsvWhereColumnsWithHeader() throws Exception {
- //Pre DRILL-951: Qry: select columns[1] from dfs_test.`%s` where columns[1] = 'Chevy'
- String query = String.format("select Make from dfs_test.`%s` where Make = 'Chevy'", root);
+ //Pre DRILL-951: Qry: select columns[1] from cp.`%s` where columns[1] = 'Chevy'
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select Make from cp.`%s` where Make = 'Chevy'", ROOT)
.unOrdered()
.baselineColumns("Make")
.baselineValues("Chevy")
@@ -96,11 +91,9 @@ public class TestCsvHeader extends BaseTestQuery{
@Test //DRILL-951
public void testCsvColumnsWithHeader() throws Exception {
- //Pre DRILL-951: Qry: select columns[0],columns[2],columns[4] from dfs_test.`%s` where columns[1] = 'Chevy'
- String query = String.format("select `Year`, Model, Price from dfs_test.`%s` where Make = 'Chevy'", root);
-
+ //Pre DRILL-951: Qry: select columns[0],columns[2],columns[4] from cp.`%s` where columns[1] = 'Chevy'
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select `Year`, Model, Price from cp.`%s` where Make = 'Chevy'", ROOT)
.unOrdered()
.baselineColumns("Year", "Model", "Price")
.baselineValues("1999", "Venture \"Extended Edition\"", "4900.00")
@@ -110,10 +103,8 @@ public class TestCsvHeader extends BaseTestQuery{
@Test //DRILL-951
public void testCsvHeaderShortCircuitReads() throws Exception {
- String query = String.format("select `Year`, Model from dfs_test.`%s` where Make = 'Chevy'", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select `Year`, Model from cp.`%s` where Make = 'Chevy'", ROOT)
.unOrdered()
.baselineColumns("Year", "Model")
.baselineValues("1999", "Venture \"Extended Edition\"")
@@ -123,10 +114,8 @@ public class TestCsvHeader extends BaseTestQuery{
@Test //DRILL-4108
public void testCsvHeaderNonExistingColumn() throws Exception {
- String query = String.format("select `Year`, Model, Category from dfs_test.`%s` where Make = 'Chevy'", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select `Year`, Model, Category from cp.`%s` where Make = 'Chevy'", ROOT)
.unOrdered()
.baselineColumns("Year", "Model", "Category")
.baselineValues("1999", "Venture \"Extended Edition\"", "")
@@ -136,10 +125,8 @@ public class TestCsvHeader extends BaseTestQuery{
@Test //DRILL-4108
public void testCsvHeaderMismatch() throws Exception {
- String ddir = FileUtils.getResourceAsFile("/store/text/data/d2").toURI().toString();
- String query = String.format("select `Year`, Model, Category from dfs_test.`%s` where Make = 'Chevy'", ddir);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select `Year`, Model, Category from dfs.`store/text/data/d2` where Make = 'Chevy'")
.unOrdered()
.baselineColumns("Year", "Model", "Category")
.baselineValues("1999", "", "Venture \"Extended Edition\"")
@@ -151,12 +138,8 @@ public class TestCsvHeader extends BaseTestQuery{
@Test //DRILL-4108
public void testCsvHeaderSkipFirstLine() throws Exception {
- // test that header is not skipped when skipFirstLine is true
- // testing by defining new format plugin with skipFirstLine set to true and diff file extension
- String dfile = FileUtils.getResourceAsFile("/store/text/data/cars.csvh-test").toURI().toString();
- String query = String.format("select `Year`, Model from dfs_test.`%s` where Make = 'Chevy'", dfile);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select `Year`, Model from cp.`store/text/data/cars.csvh-test` where Make = 'Chevy'")
.unOrdered()
.baselineColumns("Year", "Model")
.baselineValues("1999", "Venture \"Extended Edition\"")
@@ -166,9 +149,7 @@ public class TestCsvHeader extends BaseTestQuery{
@Test
public void testEmptyFinalColumn() throws Exception {
- String dfs_temp = getDfsTestTmpSchemaLocation();
- File table_dir = new File(dfs_temp, "emptyFinalColumn");
- table_dir.mkdir();
+ File table_dir = dirTestWatcher.makeTestTmpSubDir(Paths.get("emptyFinalColumn"));
BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.csvh")));
os.write("field1,field2\n".getBytes());
for (int i = 0; i < 10000; i++) {
@@ -176,7 +157,7 @@ public class TestCsvHeader extends BaseTestQuery{
}
os.flush();
os.close();
- String query = "select * from dfs_test.tmp.emptyFinalColumn";
+ String query = "select * from dfs.tmp.emptyFinalColumn";
TestBuilder builder = testBuilder()
.sqlQuery(query)
.ordered()
@@ -189,12 +170,12 @@ public class TestCsvHeader extends BaseTestQuery{
@Test
public void testCountOnCsvWithHeader() throws Exception {
- final String query = "select count(%s) as cnt from %s.`%s`";
+ final String query = "select count(%s) as cnt from cp.`%s`";
final List<Object> options = Lists.<Object>newArrayList("*", 1, "'A'");
for (Object option : options) {
testBuilder()
- .sqlQuery(query, option, TEMP_SCHEMA, root)
+ .sqlQuery(query, option, ROOT)
.unOrdered()
.baselineColumns("cnt")
.baselineValues(4L)
@@ -202,5 +183,4 @@ public class TestCsvHeader extends BaseTestQuery{
.run();
}
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java
index bfc3cc2..8aa939e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestNewTextReader.java
@@ -21,10 +21,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserRemoteException;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType;
import org.junit.Ignore;
import org.junit.Test;
@@ -69,12 +68,9 @@ public class TestNewTextReader extends BaseTestQuery {
@Test // see DRILL-3718
@Category(UnlikelyTest.class)
public void testTabSeparatedWithQuote() throws Exception {
- final String root = FileUtils.getResourceAsFile("/store/text/WithQuote.tsv").toURI().toString();
- final String query = String.format("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" +
- "from dfs_test.`%s` ", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" +
+ "from cp.`store/text/WithQuote.tsv`")
.unOrdered()
.baselineColumns("c0", "c1", "c2")
.baselineValues("a", "a", "a")
@@ -87,12 +83,9 @@ public class TestNewTextReader extends BaseTestQuery {
@Test // see DRILL-3718
@Category(UnlikelyTest.class)
public void testSpaceSeparatedWithQuote() throws Exception {
- final String root = FileUtils.getResourceAsFile("/store/text/WithQuote.ssv").toURI().toString();
- final String query = String.format("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" +
- "from dfs_test.`%s` ", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" +
+ "from cp.`store/text/WithQuote.ssv`")
.unOrdered()
.baselineColumns("c0", "c1", "c2")
.baselineValues("a", "a", "a")
@@ -105,12 +98,9 @@ public class TestNewTextReader extends BaseTestQuery {
@Test // see DRILL-3718
@Category(UnlikelyTest.class)
public void testPipSeparatedWithQuote() throws Exception {
- final String root = FileUtils.getResourceAsFile("/store/text/WithQuote.tbl").toURI().toString();
- final String query = String.format("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" +
- "from dfs_test.`%s` ", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" +
+ "from cp.`store/text/WithQuote.tbl`")
.unOrdered()
.baselineColumns("c0", "c1", "c2")
.baselineValues("a", "a", "a")
@@ -123,12 +113,9 @@ public class TestNewTextReader extends BaseTestQuery {
@Test // see DRILL-3718
@Category(UnlikelyTest.class)
public void testCrLfSeparatedWithQuote() throws Exception {
- final String root = FileUtils.getResourceAsFile("/store/text/WithQuotedCrLf.tbl").toURI().toString();
- final String query = String.format("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" +
- "from dfs_test.`%s` ", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select columns[0] as c0, columns[1] as c1, columns[2] as c2 \n" +
+ "from cp.`store/text/WithQuotedCrLf.tbl`")
.unOrdered()
.baselineColumns("c0", "c1", "c2")
.baselineValues("a\n1", "a", "a")
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java
index 1ff6818..f106cca 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java
@@ -14,24 +14,21 @@ package org.apache.drill.exec.store.text;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.junit.Test;
import static org.junit.Assert.*;
public class TestTextColumn extends BaseTestQuery {
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTextColumn.class);
-
@Test
public void testCsvColumnSelection() throws Exception {
- test("select columns[0] as region_id, columns[1] as country from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/regions.csv`");
+ test("select columns[0] as region_id, columns[1] as country from cp.`store/text/data/regions.csv`");
}
@Test
public void testDefaultDelimiterColumnSelection() throws Exception {
List<QueryDataBatch> actualResults = testSqlWithResults("SELECT columns[0] as entire_row "
- + "from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/letters.txt`");
+ + "from cp.`store/text/data/letters.txt`");
final TestResultSet expectedResultSet = new TestResultSet();
expectedResultSet.addRow("a, b,\",\"c\",\"d,, \\n e");
@@ -45,7 +42,7 @@ public class TestTextColumn extends BaseTestQuery {
@Test
public void testCsvColumnSelectionCommasInsideQuotes() throws Exception {
List<QueryDataBatch> actualResults = testSqlWithResults("SELECT columns[0] as col1, columns[1] as col2, columns[2] as col3,"
- + "columns[3] as col4 from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/letters.csv`");
+ + "columns[3] as col4 from cp.`store/text/data/letters.csv`");
final TestResultSet expectedResultSet = new TestResultSet();
expectedResultSet.addRow("a, b,", "c", "d,, \\n e", "f\\\"g");
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java
index 5e781d2..d4098f9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TextRecordReaderTest.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertEquals;
import java.util.List;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -47,8 +47,8 @@ public class TextRecordReaderTest extends PopUnitTestBase {
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
Files.toString(
- FileUtils.getResourceAsFile("/store/text/test.json"), Charsets.UTF_8)
- .replace("#{DATA_FILE}", FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString()));
+ DrillFileUtils.getResourceAsFile("/store/text/test.json"), Charsets.UTF_8)
+ .replace("#{DATA_FILE}", DrillFileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString()));
int count = 0;
RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
for(QueryDataBatch b : results) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java
index 407cb7c..6404e27 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestCountDownLatchInjection.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.fail;
import java.util.concurrent.CountDownLatch;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.concurrent.ExtendedLatch;
import org.apache.drill.exec.ops.QueryContext;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java
index 22da20b..44f8727 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestExceptionInjection.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.fail;
import java.io.IOException;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ZookeeperHelper;
import org.apache.drill.exec.exception.DrillbitStartupException;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java
index c618739..2e8aa02 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/TestPauseInjection.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.fail;
import java.util.concurrent.CountDownLatch;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.concurrent.ExtendedLatch;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ZookeeperHelper;
[21/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 5131772..cb62b27 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -62,7 +62,6 @@ import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.expr.ClassGenerator.BlockType;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.fn.AbstractFuncHolder;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
import org.apache.drill.exec.vector.ValueHolderHelper;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
@@ -85,12 +84,7 @@ import com.sun.codemodel.JVar;
public class EvaluationVisitor {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EvaluationVisitor.class);
-
- private final FunctionImplementationRegistry registry;
-
- public EvaluationVisitor(FunctionImplementationRegistry registry) {
- super();
- this.registry = registry;
+ public EvaluationVisitor() {
}
public HoldingContainer addExpr(LogicalExpression e, ClassGenerator<?> generator) {
@@ -497,7 +491,6 @@ public class EvaluationVisitor {
}
if (complex || repeated) {
- MajorType finalType = e.getFieldId().getFinalType();
// //
JVar complexReader = generator.declareClassField("reader", generator.getModel()._ref(FieldReader.class));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
index b83350d..6c186db 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.expr.fn;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import com.google.common.collect.Lists;
@@ -26,7 +25,6 @@ import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
@@ -34,11 +32,9 @@ import org.apache.drill.common.expression.ValueExpressions.IntExpression;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.HoldingContainerExpression;
-import org.apache.calcite.rel.RelFieldCollation.NullDirection;
public class FunctionGenerationHelper {
public static final String COMPARE_TO_NULLS_HIGH = "compare_to_nulls_high";
@@ -58,7 +54,7 @@ public class FunctionGenerationHelper {
* {@code false}) or the highest value (if {@code true})
* @param left ...
* @param right ...
- * @param registry ...
+ * @param functionLookupContext ...
* @return
* FunctionHolderExpression containing the found function implementation
*/
@@ -66,7 +62,7 @@ public class FunctionGenerationHelper {
boolean null_high,
HoldingContainer left,
HoldingContainer right,
- FunctionImplementationRegistry registry) {
+ FunctionLookupContext functionLookupContext) {
final String comparator_name =
null_high ? COMPARE_TO_NULLS_HIGH : COMPARE_TO_NULLS_LOW;
@@ -76,15 +72,14 @@ public class FunctionGenerationHelper {
throw new UnsupportedOperationException(
formatCanNotCompareMsg(left.getMajorType(), right.getMajorType()));
}
- LogicalExpression comparisonFunctionExpression = getFunctionExpression(comparator_name, Types.required(MinorType.INT),
- registry, left, right);
+ LogicalExpression comparisonFunctionExpression = getFunctionExpression(comparator_name, left, right);
ErrorCollector collector = new ErrorCollectorImpl();
if (!isUnionType(left.getMajorType()) && !isUnionType(right.getMajorType())) {
- return ExpressionTreeMaterializer.materialize(comparisonFunctionExpression, null, collector, registry);
+ return ExpressionTreeMaterializer.materialize(comparisonFunctionExpression, null, collector, functionLookupContext);
} else {
LogicalExpression typeComparisonFunctionExpression = getTypeComparisonFunction(comparisonFunctionExpression, left, right);
- return ExpressionTreeMaterializer.materialize(typeComparisonFunctionExpression, null, collector, registry);
+ return ExpressionTreeMaterializer.materialize(typeComparisonFunctionExpression, null, collector, functionLookupContext);
}
}
@@ -107,8 +102,7 @@ public class FunctionGenerationHelper {
return getOrderingComparator(true, left, right, registry);
}
- private static LogicalExpression getFunctionExpression(
- String name, MajorType returnType, FunctionImplementationRegistry registry, HoldingContainer... args) {
+ private static LogicalExpression getFunctionExpression(String name, HoldingContainer... args) {
List<MajorType> argTypes = new ArrayList<MajorType>(args.length);
List<LogicalExpression> argExpressions = new ArrayList<LogicalExpression>(args.length);
for(HoldingContainer c : args) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java
index 9ca6dbd..20b1d12 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.expr.fn;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
@@ -24,7 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.codehaus.commons.compiler.CompileException;
import org.codehaus.janino.Java.CompilationUnit;
import org.codehaus.janino.Parser;
@@ -128,7 +129,7 @@ public class FunctionInitializer {
private CompilationUnit convertToCompilationUnit(Class<?> clazz) throws IOException {
String path = clazz.getName();
path = path.replaceFirst("\\$.*", "");
- path = path.replace(".", FileUtils.separator);
+ path = path.replace(".", DrillFileUtils.SEPARATOR);
path = "/" + path + ".java";
logger.trace("Loading function code from the {}", path);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueue.java
index c583664..9d9020a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueue.java
@@ -20,22 +20,58 @@ package org.apache.drill.exec.physical.impl.TopN;
import org.apache.drill.exec.compile.TemplateClassDefinition;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.selection.SelectionVector4;
public interface PriorityQueue {
- public void add(FragmentContext context, RecordBatchData batch) throws SchemaChangeException;
- public void init(int limit, FragmentContext context, BufferAllocator allocator, boolean hasSv2) throws SchemaChangeException;
- public void generate() throws SchemaChangeException;
- public VectorContainer getHyperBatch();
- public SelectionVector4 getHeapSv4();
- public SelectionVector4 getFinalSv4();
- public boolean validate();
- public void resetQueue(VectorContainer container, SelectionVector4 vector4) throws SchemaChangeException;
- public void cleanup();
-
- public static TemplateClassDefinition<PriorityQueue> TEMPLATE_DEFINITION = new TemplateClassDefinition<PriorityQueue>(PriorityQueue.class, PriorityQueueTemplate.class);
+ /**
+ * The elements in the given batch are added to the priority queue. Note that the priority queue
+ * only retains the top elements that fit within the size specified by the {@link #init(int, BufferAllocator, boolean)}
+ * method.
+ * @param batch The batch containing elements we want to add.
+ * @throws SchemaChangeException
+ */
+ void add(RecordBatchData batch) throws SchemaChangeException;
+ /**
+ * Initializes the priority queue. This method must be called before any other methods on the priority
+ * queue are called.
+ * @param limit The size of the priority queue.
+ * @param allocator The {@link BufferAllocator} to use when creating the priority queue.
+ * @param hasSv2 True when incoming batches have 2 byte selection vectors. False otherwise.
+ * @throws SchemaChangeException
+ */
+ void init(int limit, BufferAllocator allocator, boolean hasSv2) throws SchemaChangeException;
+
+ /**
+ * This method must be called before fetching the final priority queue hyper batch and final Sv4 vector.
+ * @throws SchemaChangeException
+ */
+ void generate() throws SchemaChangeException;
+
+ /**
+ * Retrieves the final priority queue HyperBatch containing the results. <b>Note:</b> this should be called
+ * after {@link #generate()}.
+ * @return The final priority queue HyperBatch containing the results.
+ */
+ VectorContainer getHyperBatch();
+
+ SelectionVector4 getSv4();
+
+ /**
+ * Retrieves the selection vector used to select the elements in the priority queue from the hyper batch
+ * provided by the {@link #getHyperBatch()} method. <b>Note:</b> this should be called after {@link #generate()}.
+ * @return The selection vector used to select the elements in the priority queue.
+ */
+ SelectionVector4 getFinalSv4();
+
+ void resetQueue(VectorContainer container, SelectionVector4 vector4) throws SchemaChangeException;
+
+ /**
+ * Releases all the memory consumed by the priority queue.
+ */
+ void cleanup();
+
+ TemplateClassDefinition<PriorityQueue> TEMPLATE_DEFINITION = new TemplateClassDefinition<>(PriorityQueue.class, PriorityQueueTemplate.class);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
index b82dfc8..7f9aca4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java
@@ -44,7 +44,6 @@ public abstract class PriorityQueueTemplate implements PriorityQueue {
private SelectionVector4 heapSv4; //This holds the heap
private SelectionVector4 finalSv4; //This is for final sorted output
private ExpandableHyperContainer hyperBatch;
- private FragmentContext context;
private BufferAllocator allocator;
private int limit;
private int queueSize = 0;
@@ -52,9 +51,8 @@ public abstract class PriorityQueueTemplate implements PriorityQueue {
private boolean hasSv2;
@Override
- public void init(int limit, FragmentContext context, BufferAllocator allocator, boolean hasSv2) throws SchemaChangeException {
+ public void init(int limit, BufferAllocator allocator, boolean hasSv2) throws SchemaChangeException {
this.limit = limit;
- this.context = context;
this.allocator = allocator;
@SuppressWarnings("resource")
final DrillBuf drillBuf = allocator.buffer(4 * (limit + 1));
@@ -63,9 +61,6 @@ public abstract class PriorityQueueTemplate implements PriorityQueue {
}
@Override
- public boolean validate() { return true; }
-
- @Override
public void resetQueue(VectorContainer container, SelectionVector4 v4) throws SchemaChangeException {
assert container.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.FOUR_BYTE;
BatchSchema schema = container.getSchema();
@@ -89,12 +84,12 @@ public abstract class PriorityQueueTemplate implements PriorityQueue {
++queueSize;
}
v4.clear();
- doSetup(context, hyperBatch, null);
+ doSetup(hyperBatch, null);
}
@SuppressWarnings("resource")
@Override
- public void add(FragmentContext context, RecordBatchData batch) throws SchemaChangeException{
+ public void add(RecordBatchData batch) throws SchemaChangeException{
Stopwatch watch = Stopwatch.createStarted();
if (hyperBatch == null) {
hyperBatch = new ExpandableHyperContainer(batch.getContainer());
@@ -102,7 +97,7 @@ public abstract class PriorityQueueTemplate implements PriorityQueue {
hyperBatch.addBatch(batch.getContainer());
}
- doSetup(context, hyperBatch, null); // may not need to do this every time
+ doSetup(hyperBatch, null); // may not need to do this every time
int count = 0;
SelectionVector2 sv2 = null;
@@ -146,7 +141,7 @@ public abstract class PriorityQueueTemplate implements PriorityQueue {
}
@Override
- public SelectionVector4 getHeapSv4() {
+ public SelectionVector4 getSv4() {
return heapSv4;
}
@@ -226,8 +221,7 @@ public abstract class PriorityQueueTemplate implements PriorityQueue {
return doEval(sv1, sv2);
}
- public abstract void doSetup(@Named("context") FragmentContext context,
- @Named("incoming") VectorContainer incoming,
+ public abstract void doSetup(@Named("incoming") VectorContainer incoming,
@Named("outgoing") RecordBatch outgoing)
throws SchemaChangeException;
public abstract int doEval(@Named("leftIndex") int leftIndex,
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index dcf67d4..34c0f94 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -18,18 +18,18 @@
package org.apache.drill.exec.physical.impl.TopN;
import java.io.IOException;
-import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.calcite.rel.RelFieldCollation.Direction;
import org.apache.drill.common.DrillAutoCloseables;
+import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.compile.CodeCompiler;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.exception.OutOfMemoryException;
@@ -40,6 +40,8 @@ import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
+import org.apache.drill.exec.expr.fn.FunctionLookupContext;
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.TopN;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
@@ -53,13 +55,13 @@ import org.apache.drill.exec.record.ExpandableHyperContainer;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.SchemaUtil;
import org.apache.drill.exec.record.SimpleRecordBatch;
-import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.record.WritableBatch;
import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.exec.server.options.OptionSet;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.AbstractContainerVector;
@@ -67,17 +69,15 @@ import com.google.common.base.Stopwatch;
import com.sun.codemodel.JConditional;
import com.sun.codemodel.JExpr;
-import static org.bouncycastle.asn1.x500.style.RFC4519Style.l;
-
public class TopNBatch extends AbstractRecordBatch<TopN> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopNBatch.class);
- private final int batchPurgeThreshold;
-
- public final MappingSet MAIN_MAPPING = new MappingSet( (String) null, null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
- public final MappingSet LEFT_MAPPING = new MappingSet("leftIndex", null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
- public final MappingSet RIGHT_MAPPING = new MappingSet("rightIndex", null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
+ public final MappingSet mainMapping = createMainMappingSet();
+ public final MappingSet leftMapping = createLeftMappingSet();
+ public final MappingSet rightMapping = createRightMappingSet();
+ private final int batchPurgeThreshold;
+ private final boolean codegenDump;
private final RecordBatch incoming;
private BatchSchema schema;
@@ -95,7 +95,9 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
super(popConfig, context);
this.incoming = incoming;
this.config = popConfig;
- batchPurgeThreshold = context.getConfig().getInt(ExecConstants.BATCH_PURGE_THRESHOLD);
+ DrillConfig drillConfig = context.getConfig();
+ batchPurgeThreshold = drillConfig.getInt(ExecConstants.BATCH_PURGE_THRESHOLD);
+ codegenDump = drillConfig.getBoolean(CodeCompiler.ENABLE_SAVE_CODE_FOR_DEBUG_TOPN);
}
@Override
@@ -181,7 +183,6 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
}
}
-
try{
outer: while (true) {
Stopwatch watch = Stopwatch.createStarted();
@@ -240,9 +241,9 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
try {
if (priorityQueue == null) {
assert !schemaChanged;
- priorityQueue = createNewPriorityQueue(context, config.getOrderings(), new ExpandableHyperContainer(batch.getContainer()), MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
+ priorityQueue = createNewPriorityQueue(new ExpandableHyperContainer(batch.getContainer()), config.getLimit());
}
- priorityQueue.add(context, batch);
+ priorityQueue.add(batch);
if (countSincePurge > config.getLimit() && batchCount > batchPurgeThreshold) {
purge();
countSincePurge = 0;
@@ -290,7 +291,7 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
VectorContainer c = priorityQueue.getHyperBatch();
VectorContainer newContainer = new VectorContainer(oContext);
@SuppressWarnings("resource")
- SelectionVector4 selectionVector4 = priorityQueue.getHeapSv4();
+ SelectionVector4 selectionVector4 = priorityQueue.getSv4();
SimpleSV4RecordBatch batch = new SimpleSV4RecordBatch(c, selectionVector4, context);
SimpleSV4RecordBatch newBatch = new SimpleSV4RecordBatch(newContainer, null, context);
if (copier == null) {
@@ -332,20 +333,42 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
logger.debug("Took {} us to purge", watch.elapsed(TimeUnit.MICROSECONDS));
}
- public PriorityQueue createNewPriorityQueue(FragmentContext context, List<Ordering> orderings,
- VectorAccessible batch, MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping)
- throws ClassTransformationException, IOException, SchemaChangeException{
- CodeGenerator<PriorityQueue> cg = CodeGenerator.get(PriorityQueue.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ private PriorityQueue createNewPriorityQueue(VectorAccessible batch, int limit)
+ throws SchemaChangeException, ClassTransformationException, IOException {
+ return createNewPriorityQueue(
+ mainMapping, leftMapping, rightMapping, context.getOptionSet(), context.getFunctionRegistry(), context.getDrillbitContext().getCompiler(),
+ config.getOrderings(), batch, unionTypeEnabled, codegenDump, limit, oContext.getAllocator(), schema.getSelectionVectorMode());
+ }
+
+ public static MappingSet createMainMappingSet() {
+ return new MappingSet((String) null, null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
+ }
+
+ public static MappingSet createLeftMappingSet() {
+ return new MappingSet("leftIndex", null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
+ }
+
+ public static MappingSet createRightMappingSet() {
+ return new MappingSet("rightIndex", null, ClassGenerator.DEFAULT_SCALAR_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
+ }
+
+ public static PriorityQueue createNewPriorityQueue(
+ MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping,
+ OptionSet optionSet, FunctionLookupContext functionLookupContext, CodeCompiler codeCompiler,
+ List<Ordering> orderings, VectorAccessible batch, boolean unionTypeEnabled, boolean codegenDump,
+ int limit, BufferAllocator allocator, SelectionVectorMode mode)
+ throws ClassTransformationException, IOException, SchemaChangeException {
+ CodeGenerator<PriorityQueue> cg = CodeGenerator.get(PriorityQueue.TEMPLATE_DEFINITION, optionSet);
cg.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// cg.saveCodeForDebugging(true);
+ cg.saveCodeForDebugging(codegenDump);
ClassGenerator<PriorityQueue> g = cg.getRoot();
g.setMappingSet(mainMapping);
for (Ordering od : orderings) {
// first, we rewrite the evaluation stack for each side of the comparison.
ErrorCollector collector = new ErrorCollectorImpl();
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector, context.getFunctionRegistry(), unionTypeEnabled);
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector, functionLookupContext, unionTypeEnabled);
if (collector.hasErrors()) {
throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
}
@@ -357,8 +380,7 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
// next we wrap the two comparison sides and add the expression block for the comparison.
LogicalExpression fh =
- FunctionGenerationHelper.getOrderingComparator(od.nullsSortHigh(), left, right,
- context.getFunctionRegistry());
+ FunctionGenerationHelper.getOrderingComparator(od.nullsSortHigh(), left, right, functionLookupContext);
HoldingContainer out = g.addExpr(fh, ClassGenerator.BlkCreateMode.FALSE);
JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
@@ -373,8 +395,8 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
g.rotateBlock();
g.getEvalBlock()._return(JExpr.lit(0));
- PriorityQueue q = context.getImplementationClass(cg);
- q.init(config.getLimit(), context, oContext.getAllocator(), schema.getSelectionVectorMode() == BatchSchema.SelectionVectorMode.TWO_BYTE);
+ PriorityQueue q = codeCompiler.createInstance(cg);
+ q.init(limit, allocator, mode == BatchSchema.SelectionVectorMode.TWO_BYTE);
return q;
}
@@ -390,7 +412,7 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
final VectorContainer c = priorityQueue.getHyperBatch();
final VectorContainer newContainer = new VectorContainer(oContext);
@SuppressWarnings("resource")
- final SelectionVector4 selectionVector4 = priorityQueue.getHeapSv4();
+ final SelectionVector4 selectionVector4 = priorityQueue.getSv4();
final SimpleSV4RecordBatch batch = new SimpleSV4RecordBatch(c, selectionVector4, context);
final SimpleSV4RecordBatch newBatch = new SimpleSV4RecordBatch(newContainer, null, context);
copier = RemovingRecordBatch.getGenerated4Copier(batch, context, oContext.getAllocator(), newContainer, newBatch, null);
@@ -417,7 +439,7 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
final VectorContainer newSchemaContainer = SchemaUtil.coerceContainer(oldSchemaContainer, this.schema, oContext);
newSchemaContainer.buildSchema(SelectionVectorMode.FOUR_BYTE);
priorityQueue.cleanup();
- priorityQueue = createNewPriorityQueue(context, config.getOrderings(), newSchemaContainer, MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
+ priorityQueue = createNewPriorityQueue(newSchemaContainer, config.getLimit());
priorityQueue.resetQueue(newSchemaContainer, builder.getSv4().createNewWrapperCurrent());
} finally {
builder.clear();
@@ -436,7 +458,6 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
incoming.kill(sendUpstream);
}
-
public static class SimpleSV4RecordBatch extends SimpleRecordBatch {
private SelectionVector4 sv4;
@@ -459,5 +480,4 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
return sv4;
}
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
index 8006276..b3d68d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
@@ -223,8 +223,7 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
private HashAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException,
IOException {
- CodeGenerator<HashAggregator> top =
- CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ CodeGenerator<HashAggregator> top = CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getOptions());
ClassGenerator<HashAggregator> cg = top.getRoot();
ClassGenerator<HashAggregator> cgInner = cg.getInnerGenerator("BatchHolder");
top.plainJavaCapable(true);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
index e5ba98f..b33dbd6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
@@ -273,11 +273,10 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
}
private StreamingAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException{
- ClassGenerator<StreamingAggregator> cg = CodeGenerator.getRoot(StreamingAggTemplate.TEMPLATE_DEFINITION,
- context.getFunctionRegistry(), context.getOptions());
+ ClassGenerator<StreamingAggregator> cg = CodeGenerator.getRoot(StreamingAggTemplate.TEMPLATE_DEFINITION, context.getOptions());
cg.getCodeGenerator().plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// cg.getCodeGenerator().saveCodeForDebugging(true);
+ // cg.getCodeGenerator().saveCodeForDebugging(true);
container.clear();
LogicalExpression[] keyExprs = new LogicalExpression[popConfig.getKeys().size()];
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
index db9622f..dd4d76e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -135,7 +135,7 @@ public class ChainedHashTable {
public HashTable createAndSetupHashTable(TypedFieldId[] outKeyFieldIds, int numPartitions) throws ClassTransformationException,
IOException, SchemaChangeException {
- CodeGenerator<HashTable> top = CodeGenerator.get(HashTable.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ CodeGenerator<HashTable> top = CodeGenerator.get(HashTable.TEMPLATE_DEFINITION, context.getOptions());
top.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
// This code is called from generated code, so to step into this code,
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
index 6dfd311..1bdd097 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterRecordBatch.java
@@ -144,7 +144,7 @@ public class FilterRecordBatch extends AbstractSingleRecordBatch<Filter>{
protected Filterer generateSV4Filterer() throws SchemaChangeException {
final ErrorCollector collector = new ErrorCollectorImpl();
final List<TransferPair> transfers = Lists.newArrayList();
- final ClassGenerator<Filterer> cg = CodeGenerator.getRoot(Filterer.TEMPLATE_DEFINITION4, context.getFunctionRegistry(), context.getOptions());
+ final ClassGenerator<Filterer> cg = CodeGenerator.getRoot(Filterer.TEMPLATE_DEFINITION4, context.getOptions());
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(popConfig.getExpr(), incoming, collector, context.getFunctionRegistry());
if (collector.hasErrors()) {
@@ -178,7 +178,7 @@ public class FilterRecordBatch extends AbstractSingleRecordBatch<Filter>{
protected Filterer generateSV2Filterer() throws SchemaChangeException {
final ErrorCollector collector = new ErrorCollectorImpl();
final List<TransferPair> transfers = Lists.newArrayList();
- final ClassGenerator<Filterer> cg = CodeGenerator.getRoot(Filterer.TEMPLATE_DEFINITION2, context.getFunctionRegistry(), context.getOptions());
+ final ClassGenerator<Filterer> cg = CodeGenerator.getRoot(Filterer.TEMPLATE_DEFINITION2, context.getOptions());
final LogicalExpression expr = ExpressionTreeMaterializer.materialize(popConfig.getExpr(), incoming, collector,
context.getFunctionRegistry(), false, unionTypeEnabled);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index e64e919..2aa841b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -301,10 +301,10 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
final ErrorCollector collector = new ErrorCollectorImpl();
final List<TransferPair> transfers = Lists.newArrayList();
- final ClassGenerator<Flattener> cg = CodeGenerator.getRoot(Flattener.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ final ClassGenerator<Flattener> cg = CodeGenerator.getRoot(Flattener.TEMPLATE_DEFINITION, context.getOptions());
cg.getCodeGenerator().plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// cg.getCodeGenerator().saveCodeForDebugging(true);
+ // cg.getCodeGenerator().saveCodeForDebugging(true);
final IntHashSet transferFieldIds = new IntHashSet();
final NamedExpression flattenExpr = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 481bea8..7e2859e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -49,7 +49,6 @@ import org.apache.drill.exec.physical.impl.common.IndexPointer;
import org.apache.drill.exec.physical.impl.common.Comparator;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
import org.apache.drill.exec.record.AbstractBinaryRecordBatch;
-import org.apache.drill.exec.record.AbstractRecordBatch;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import org.apache.drill.exec.record.ExpandableHyperContainer;
@@ -398,10 +397,10 @@ public class HashJoinBatch extends AbstractBinaryRecordBatch<HashJoinPOP> {
}
public HashJoinProbe setupHashJoinProbe() throws ClassTransformationException, IOException {
- final CodeGenerator<HashJoinProbe> cg = CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ final CodeGenerator<HashJoinProbe> cg = CodeGenerator.get(HashJoinProbe.TEMPLATE_DEFINITION, context.getOptions());
cg.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// cg.saveCodeForDebugging(true);
+ // cg.saveCodeForDebugging(true);
final ClassGenerator<HashJoinProbe> g = cg.getRoot();
// Generate the code to project build side records
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index a1b8dc2..6d804c5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -271,10 +271,10 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
private JoinWorker generateNewWorker() throws ClassTransformationException, IOException, SchemaChangeException{
- final ClassGenerator<JoinWorker> cg = CodeGenerator.getRoot(JoinWorker.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ final ClassGenerator<JoinWorker> cg = CodeGenerator.getRoot(JoinWorker.TEMPLATE_DEFINITION, context.getOptions());
cg.getCodeGenerator().plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// cg.getCodeGenerator().saveCodeForDebugging(true);
+ // cg.getCodeGenerator().saveCodeForDebugging(true);
final ErrorCollector collector = new ErrorCollectorImpl();
// Generate members and initialization code
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
index b390e41..fa8c13a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
@@ -43,7 +43,6 @@ import org.apache.drill.exec.physical.config.NestedLoopJoinPOP;
import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
import org.apache.drill.exec.record.AbstractBinaryRecordBatch;
-import org.apache.drill.exec.record.AbstractRecordBatch;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.ExpandableHyperContainer;
import org.apache.drill.exec.record.MaterializedField;
@@ -221,10 +220,10 @@ public class NestedLoopJoinBatch extends AbstractBinaryRecordBatch<NestedLoopJoi
*/
private NestedLoopJoin setupWorker() throws IOException, ClassTransformationException, SchemaChangeException {
final CodeGenerator<NestedLoopJoin> nLJCodeGenerator = CodeGenerator.get(
- NestedLoopJoin.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ NestedLoopJoin.TEMPLATE_DEFINITION, context.getOptions());
nLJCodeGenerator.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// nLJCodeGenerator.saveCodeForDebugging(true);
+ // nLJCodeGenerator.saveCodeForDebugging(true);
final ClassGenerator<NestedLoopJoin> nLJClassGenerator = nLJCodeGenerator.getRoot();
// generate doEval
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index ec945d6..a7d3f39 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -700,10 +700,6 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
}
}
-// private boolean isOutgoingFull() {
-// return outgoingPosition == DEFAULT_ALLOC_RECORD_COUNT;
-// }
-
/**
* Creates a generate class which implements the copy and compare methods.
*
@@ -713,7 +709,7 @@ public class MergingRecordBatch extends AbstractRecordBatch<MergingReceiverPOP>
private MergingReceiverGeneratorBase createMerger() throws SchemaChangeException {
try {
- final CodeGenerator<MergingReceiverGeneratorBase> cg = CodeGenerator.get(MergingReceiverGeneratorBase.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ final CodeGenerator<MergingReceiverGeneratorBase> cg = CodeGenerator.get(MergingReceiverGeneratorBase.TEMPLATE_DEFINITION, context.getOptions());
cg.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
// cg.saveCodeForDebugging(true);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index 63133d4..7f662ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -420,13 +420,12 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
private SampleCopier getCopier(SelectionVector4 sv4, VectorContainer incoming, VectorContainer outgoing,
List<Ordering> orderings, List<ValueVector> localAllocationVectors) throws SchemaChangeException {
final ErrorCollector collector = new ErrorCollectorImpl();
- final ClassGenerator<SampleCopier> cg = CodeGenerator.getRoot(SampleCopier.TEMPLATE_DEFINITION,
- context.getFunctionRegistry(), context.getOptions());
+ final ClassGenerator<SampleCopier> cg = CodeGenerator.getRoot(SampleCopier.TEMPLATE_DEFINITION, context.getOptions());
// Note: disabled for now. This may require some debugging:
// no tests are available for this operator.
-// cg.getCodeGenerator().plainOldJavaCapable(true);
+ // cg.getCodeGenerator().plainOldJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// cg.getCodeGenerator().saveCodeForDebugging(true);
+ // cg.getCodeGenerator().saveCodeForDebugging(true);
int i = 0;
for (Ordering od : orderings) {
@@ -592,7 +591,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
final List<TransferPair> transfers = Lists.newArrayList();
final ClassGenerator<OrderedPartitionProjector> cg = CodeGenerator.getRoot(
- OrderedPartitionProjector.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ OrderedPartitionProjector.TEMPLATE_DEFINITION, context.getOptions());
// Note: disabled for now. This may require some debugging:
// no tests are available for this operator.
// cg.getCodeGenerator().plainOldJavaCapable(true);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
index 92364e8..7684e94 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
@@ -265,10 +265,10 @@ public class PartitionSenderRootExec extends BaseRootExec {
final ErrorCollector collector = new ErrorCollectorImpl();
final ClassGenerator<Partitioner> cg ;
- cg = CodeGenerator.getRoot(Partitioner.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ cg = CodeGenerator.getRoot(Partitioner.TEMPLATE_DEFINITION, context.getOptions());
cg.getCodeGenerator().plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// cg.getCodeGenerator().saveCodeForDebugging(true);
+ // cg.getCodeGenerator().saveCodeForDebugging(true);
ClassGenerator<Partitioner> cgInner = cg.getInnerGenerator("OutgoingRecordBatch");
final LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(expr, incoming, collector, context.getFunctionRegistry());
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 30efeec..3abf0fc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -35,7 +35,6 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.fn.CastFunctions;
import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.ClassTransformationException;
@@ -52,7 +51,6 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Project;
import org.apache.drill.exec.planner.StarColumnHelper;
import org.apache.drill.exec.record.AbstractSingleRecordBatch;
-import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatch;
@@ -61,9 +59,6 @@ import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.record.WritableBatch;
-import org.apache.drill.exec.record.selection.SelectionVector2;
-import org.apache.drill.exec.record.selection.SelectionVector4;
import org.apache.drill.exec.store.ColumnExplorer;
import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.FixedWidthVector;
@@ -73,9 +68,7 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
@@ -326,10 +319,10 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
final ErrorCollector collector = new ErrorCollectorImpl();
final List<TransferPair> transfers = Lists.newArrayList();
- final ClassGenerator<Projector> cg = CodeGenerator.getRoot(Projector.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ final ClassGenerator<Projector> cg = CodeGenerator.getRoot(Projector.TEMPLATE_DEFINITION, context.getOptions());
cg.getCodeGenerator().plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
- // cg.getCodeGenerator().saveCodeForDebugging(true);
+ // cg.getCodeGenerator().saveCodeForDebugging(true);
final IntHashSet transferFieldIds = new IntHashSet();
@@ -492,7 +485,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
CodeGenerator<Projector> codeGen = cg.getCodeGenerator();
codeGen.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
- // codeGen.saveCodeForDebugging(true);
+ // codeGen.saveCodeForDebugging(true);
this.projector = context.getImplementationClass(codeGen);
projector.setup(context, incomingBatch, this, transfers);
} catch (ClassTransformationException | IOException e) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
index 152cabb..d711592 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
@@ -161,13 +161,13 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
public static Sorter createNewSorter(FragmentContext context, List<Ordering> orderings, VectorAccessible batch, MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping)
throws ClassTransformationException, IOException, SchemaChangeException{
- CodeGenerator<Sorter> cg = CodeGenerator.get(Sorter.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ CodeGenerator<Sorter> cg = CodeGenerator.get(Sorter.TEMPLATE_DEFINITION, context.getOptions());
// This operator may be deprecated. No tests exercise it.
// There is no way, at present, to verify if the generated code
// works with Plain-old Java.
-// cg.plainOldJavaCapable(true);
+ // cg.plainOldJavaCapable(true);
// Uncomment out this line to debug the generated code.
-// cg.saveCodeForDebugging(true);
+ // cg.saveCodeForDebugging(true);
ClassGenerator<Sorter> g = cg.getRoot();
g.setMappingSet(mainMapping);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
index b875b66..4304c2c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
@@ -149,21 +149,6 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
throw new IllegalStateException(e);
}
- /*
- StringBuilder builder = new StringBuilder();
- for (VectorWrapper w : container) {
- builder.append(w.getField().getPath());
- builder.append(" Value capacity: ");
- builder.append(w.getValueVector().getValueCapacity());
- if (w.getValueVector() instanceof VariableWidthVector) {
- builder.append(" Byte capacity: ");
- builder.append(((VariableWidthVector) w.getValueVector()).getByteCapacity());
- builder.append("\n");
- }
- }
- logger.debug(builder.toString());
- */
-
if (copiedRecords < remainingRecordCount) {
for(VectorWrapper<?> v : container){
ValueVector.Mutator m = v.getValueVector().getMutator();
@@ -234,7 +219,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
}
try {
- final CodeGenerator<Copier> cg = CodeGenerator.get(Copier.TEMPLATE_DEFINITION2, context.getFunctionRegistry(), context.getOptions());
+ final CodeGenerator<Copier> cg = CodeGenerator.get(Copier.TEMPLATE_DEFINITION2, context.getOptions());
CopyUtil.generateCopies(cg.getRoot(), incoming, false);
Copier copier = context.getImplementationClass(cg);
copier.setupRemover(context, incoming, this);
@@ -262,7 +247,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
}
try {
- final CodeGenerator<Copier> cg = CodeGenerator.get(Copier.TEMPLATE_DEFINITION4, context.getFunctionRegistry(), context.getOptions());
+ final CodeGenerator<Copier> cg = CodeGenerator.get(Copier.TEMPLATE_DEFINITION4, context.getOptions());
CopyUtil.generateCopies(cg.getRoot(), batch, true);
cg.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
index 4d623cf..761e272 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
@@ -165,7 +165,7 @@ public class UnionAllRecordBatch extends AbstractBinaryRecordBatch<UnionAll> {
transfers.clear();
allocationVectors.clear();
- final ClassGenerator<UnionAller> cg = CodeGenerator.getRoot(UnionAller.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ final ClassGenerator<UnionAller> cg = CodeGenerator.getRoot(UnionAller.TEMPLATE_DEFINITION, context.getOptions());
cg.getCodeGenerator().plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
// cg.getCodeGenerator().saveCodeForDebugging(true);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
index 989ea96..f4a9825 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFrameRecordBatch.java
@@ -40,7 +40,6 @@ import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.WindowPOP;
-import org.apache.drill.exec.physical.impl.project.Projector;
import org.apache.drill.exec.record.AbstractRecordBatch;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.RecordBatch;
@@ -333,7 +332,7 @@ public class WindowFrameRecordBatch extends AbstractRecordBatch<WindowPOP> {
TemplateClassDefinition<WindowFramer> definition = useCustomFrame ?
WindowFramer.FRAME_TEMPLATE_DEFINITION : WindowFramer.NOFRAME_TEMPLATE_DEFINITION;
- final ClassGenerator<WindowFramer> cg = CodeGenerator.getRoot(definition, context.getFunctionRegistry(), context.getOptions());
+ final ClassGenerator<WindowFramer> cg = CodeGenerator.getRoot(definition, context.getOptions());
{
// generating framer.isSamePartition()
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index e0cfc7a..c212593 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -693,7 +693,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
private MSorter createNewMSorter(FragmentContext context, List<Ordering> orderings, VectorAccessible batch, MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping)
throws ClassTransformationException, IOException, SchemaChangeException{
- CodeGenerator<MSorter> cg = CodeGenerator.get(MSorter.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ CodeGenerator<MSorter> cg = CodeGenerator.get(MSorter.TEMPLATE_DEFINITION, context.getOptions());
ClassGenerator<MSorter> g = cg.getRoot();
g.setMappingSet(mainMapping);
@@ -736,7 +736,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
public SingleBatchSorter createNewSorter(FragmentContext context, VectorAccessible batch)
throws ClassTransformationException, IOException, SchemaChangeException{
- CodeGenerator<SingleBatchSorter> cg = CodeGenerator.get(SingleBatchSorter.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ CodeGenerator<SingleBatchSorter> cg = CodeGenerator.get(SingleBatchSorter.TEMPLATE_DEFINITION, context.getOptions());
cg.plainJavaCapable(true); // This class can generate plain-old Java.
// Uncomment out this line to debug the generated code.
@@ -783,7 +783,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
private void createCopier(VectorAccessible batch, List<BatchGroup> batchGroupList, VectorContainer outputContainer, boolean spilling) throws SchemaChangeException {
try {
if (copier == null) {
- CodeGenerator<PriorityQueueCopier> cg = CodeGenerator.get(PriorityQueueCopier.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
+ CodeGenerator<PriorityQueueCopier> cg = CodeGenerator.get(PriorityQueueCopier.TEMPLATE_DEFINITION, context.getOptions());
cg.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
// cg.saveCodeForDebugging(true);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/MergeSortWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/MergeSortWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/MergeSortWrapper.java
index f592e44..dee24dc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/MergeSortWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/MergeSortWrapper.java
@@ -142,9 +142,7 @@ public class MergeSortWrapper extends BaseSortWrapper implements SortResults {
}
private MSorter createNewMSorter(List<Ordering> orderings, MappingSet mainMapping, MappingSet leftMapping, MappingSet rightMapping) {
- CodeGenerator<MSorter> cg = CodeGenerator.get(MSorter.TEMPLATE_DEFINITION,
- context.getFragmentContext().getFunctionRegistry(),
- context.getFragmentContext().getOptionSet());
+ CodeGenerator<MSorter> cg = CodeGenerator.get(MSorter.TEMPLATE_DEFINITION, context.getFragmentContext().getOptionSet());
cg.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
index ab8cc9a..4d21b11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java
@@ -80,9 +80,7 @@ public class PriorityQueueCopierWrapper extends BaseSortWrapper {
private PriorityQueueCopier newCopier(VectorAccessible batch) {
// Generate the copier code and obtain the resulting class
- CodeGenerator<PriorityQueueCopier> cg = CodeGenerator.get(PriorityQueueCopier.TEMPLATE_DEFINITION,
- context.getFragmentContext().getFunctionRegistry(),
- context.getFragmentContext().getOptionSet());
+ CodeGenerator<PriorityQueueCopier> cg = CodeGenerator.get(PriorityQueueCopier.TEMPLATE_DEFINITION, context.getFragmentContext().getOptionSet());
ClassGenerator<PriorityQueueCopier> g = cg.getRoot();
cg.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
index e7a78ed..1d43128 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/SorterWrapper.java
@@ -78,8 +78,7 @@ public class SorterWrapper extends BaseSortWrapper {
private SingleBatchSorter newSorter(VectorAccessible batch) {
CodeGenerator<SingleBatchSorter> cg = CodeGenerator.get(
- SingleBatchSorter.TEMPLATE_DEFINITION, context.getFragmentContext().getFunctionRegistry(),
- context.getFragmentContext().getOptionSet());
+ SingleBatchSorter.TEMPLATE_DEFINITION, context.getFragmentContext().getOptionSet());
ClassGenerator<SingleBatchSorter> g = cg.getRoot();
cg.plainJavaCapable(true);
// Uncomment out this line to debug the generated code.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
index 564aaed..0497cfd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.record;
-
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -26,7 +25,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.drill.common.types.TypeProtos.MajorType;
-
public class BatchSchema implements Iterable<MaterializedField> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BatchSchema.class);
@@ -74,7 +72,7 @@ public class BatchSchema implements Iterable<MaterializedField> {
return "BatchSchema [fields=" + fields + ", selectionVector=" + selectionVectorMode + "]";
}
- public static enum SelectionVectorMode {
+ public enum SelectionVectorMode {
NONE(-1, false), TWO_BYTE(2, true), FOUR_BYTE(4, true);
public boolean hasSelectionVector;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
index 3b8dd0d..bd077fb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
@@ -116,6 +116,14 @@ public class SelectionVector4 implements AutoCloseable {
}
}
+ public static int getBatchIndex(int sv4Index) {
+ return (sv4Index >> 16) & 0xFFFF;
+ }
+
+ public static int getRecordIndex(int sv4Index) {
+ return (sv4Index) & 0xFFFF;
+ }
+
@Override
public void close() {
clear();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index cb3bfd1..4cb1b45 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -65,6 +65,7 @@ public class FileSystemPlugin extends AbstractStoragePlugin {
public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String name) throws ExecutionSetupException{
this.config = config;
this.lpPersistance = context.getLpPersistence();
+
try {
fsConf = new Configuration();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
index 6f747ea..73cb616 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
@@ -49,7 +49,7 @@ import com.google.common.collect.Maps;
public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
private static final boolean IS_COMPRESSIBLE = true;
- private static final String DEFAULT_NAME = "json";
+ public static final String DEFAULT_NAME = "json";
public JSONFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig storageConfig) {
this(name, context, fsConf, storageConfig, new JSONFormatConfig());
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/util/StoragePluginTestUtils.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/StoragePluginTestUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/StoragePluginTestUtils.java
new file mode 100644
index 0000000..689a5bf
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/StoragePluginTestUtils.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import java.io.File;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.dfs.FileSystemConfig;
+import org.apache.drill.exec.store.dfs.FileSystemPlugin;
+import org.apache.drill.exec.store.dfs.WorkspaceConfig;
+
+import org.apache.drill.exec.store.easy.sequencefile.SequenceFileFormatConfig;
+import org.apache.drill.exec.store.easy.text.TextFormatPlugin;
+
+/**
+ * This class contains utility methods to speed up tests. Some of the production code currently calls this method
+ * when the production code is executed as part of the test runs. That's the reason why this code has to be in
+ * production module.
+ */
+public class StoragePluginTestUtils {
+ public static final String CP_PLUGIN_NAME = "cp";
+ public static final String DFS_PLUGIN_NAME = "dfs";
+
+ public static final String TMP_SCHEMA = "tmp";
+ public static final String DEFAULT_SCHEMA = "default";
+ public static final String ROOT_SCHEMA = "root";
+
+ public static final String DFS_TMP_SCHEMA = DFS_PLUGIN_NAME + "." + TMP_SCHEMA;
+ public static final String DFS_DEFAULT_SCHEMA = DFS_PLUGIN_NAME + "." + DEFAULT_SCHEMA;
+ public static final String DFS_ROOT_SCHEMA = DFS_PLUGIN_NAME + "." + ROOT_SCHEMA;
+
+ public static final String UNIT_TEST_PROP_PREFIX = "drillJDBCUnitTests";
+ public static final String UNIT_TEST_DFS_TMP_PROP = UNIT_TEST_PROP_PREFIX + "." + DFS_TMP_SCHEMA;
+ public static final String UNIT_TEST_DFS_DEFAULT_PROP = UNIT_TEST_PROP_PREFIX + "." + DFS_DEFAULT_SCHEMA;
+ public static final String UNIT_TEST_DFS_ROOT_PROP = UNIT_TEST_PROP_PREFIX + "." + DFS_ROOT_SCHEMA;
+
+ /**
+ * Update the workspace locations for a plugin.
+ *
+ * @param pluginName The plugin to update.
+ * @param pluginRegistry A plugin registry.
+ * @param tmpDirPath The directory to use.
+ */
+ public static void updateSchemaLocation(final String pluginName,
+ final StoragePluginRegistry pluginRegistry,
+ final File tmpDirPath,
+ String... schemas) throws ExecutionSetupException {
+ @SuppressWarnings("resource")
+ final FileSystemPlugin plugin = (FileSystemPlugin) pluginRegistry.getPlugin(pluginName);
+ final FileSystemConfig pluginConfig = (FileSystemConfig) plugin.getConfig();
+
+ Map<String, WorkspaceConfig> workspaces = Maps.newHashMap();
+
+ if (schemas.length == 0) {
+ schemas = new String[]{TMP_SCHEMA};
+ }
+
+ for (String schema: schemas) {
+ WorkspaceConfig workspaceConfig = pluginConfig.workspaces.get(schema);
+ String inputFormat = workspaceConfig == null ? null: workspaceConfig.getDefaultInputFormat();
+ WorkspaceConfig newWorkspaceConfig = new WorkspaceConfig(tmpDirPath.getAbsolutePath(), true, inputFormat);
+ workspaces.put(schema, newWorkspaceConfig);
+ }
+
+ pluginConfig.workspaces.putAll(workspaces);
+ pluginRegistry.createOrUpdate(pluginName, pluginConfig, true);
+ }
+
+ public static void configureFormatPlugins(StoragePluginRegistry pluginRegistry) throws ExecutionSetupException {
+ configureFormatPlugins(pluginRegistry, CP_PLUGIN_NAME);
+ configureFormatPlugins(pluginRegistry, DFS_PLUGIN_NAME);
+ }
+
+ public static void configureFormatPlugins(StoragePluginRegistry pluginRegistry, String storagePlugin) throws ExecutionSetupException {
+ FileSystemPlugin fileSystemPlugin = (FileSystemPlugin) pluginRegistry.getPlugin(storagePlugin);
+ FileSystemConfig fileSystemConfig = (FileSystemConfig) fileSystemPlugin.getConfig();
+
+ TextFormatPlugin.TextFormatConfig textConfig = new TextFormatPlugin.TextFormatConfig();
+ textConfig.extensions = ImmutableList.of("txt");
+ textConfig.fieldDelimiter = '\u0000';
+ fileSystemConfig.formats.put("txt", textConfig);
+
+ TextFormatPlugin.TextFormatConfig ssvConfig = new TextFormatPlugin.TextFormatConfig();
+ ssvConfig.extensions = ImmutableList.of("ssv");
+ ssvConfig.fieldDelimiter = ' ';
+ fileSystemConfig.formats.put("ssv", ssvConfig);
+
+ TextFormatPlugin.TextFormatConfig psvConfig = new TextFormatPlugin.TextFormatConfig();
+ psvConfig.extensions = ImmutableList.of("tbl");
+ psvConfig.fieldDelimiter = '|';
+ fileSystemConfig.formats.put("psv", psvConfig);
+
+ SequenceFileFormatConfig seqConfig = new SequenceFileFormatConfig();
+ seqConfig.extensions = ImmutableList.of("seq");
+ fileSystemConfig.formats.put("sequencefile", seqConfig);
+
+ TextFormatPlugin.TextFormatConfig csvhtestConfig = new TextFormatPlugin.TextFormatConfig();
+ csvhtestConfig.extensions = ImmutableList.of("csvh-test");
+ csvhtestConfig.fieldDelimiter = ',';
+ csvhtestConfig.extractHeader = true;
+ csvhtestConfig.skipFirstLine = true;
+ fileSystemConfig.formats.put("csvh-test", csvhtestConfig);
+
+ pluginRegistry.createOrUpdate(storagePlugin, fileSystemConfig, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java
deleted file mode 100644
index a9f178a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TestUtilities.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.store.StoragePluginRegistry;
-import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.store.dfs.FileSystemConfig;
-import org.apache.drill.exec.store.dfs.FileSystemPlugin;
-import org.apache.drill.exec.store.dfs.WorkspaceConfig;
-
-import com.google.common.io.Files;
-import org.apache.drill.exec.store.easy.json.JSONRecordReader;
-
-/**
- * This class contains utility methods to speed up tests. Some of the production code currently calls this method
- * when the production code is executed as part of the test runs. That's the reason why this code has to be in
- * production module.
- */
-public class TestUtilities {
- // Below two variable values are derived from
- // <DRILL_SRC_HOME>/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json.
- private static final String dfsPluginName = "dfs";
- private static final String dfsTmpSchema = "tmp";
-
- // Below two variable values are derived from
- // <DRILL_SRC_HOME>/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json.
- private static final String dfsTestPluginName = "dfs_test";
- private static final String dfsTestTmpSchema = "tmp";
-
- /**
- * Create and removes a temporary folder
- *
- * @return absolute path to temporary folder
- */
- public static String createTempDir() {
- final File tmpDir = Files.createTempDir();
- tmpDir.deleteOnExit();
- return tmpDir.getAbsolutePath();
- }
-
- /**
- * Update the location of dfs_test.tmp location. Get the "dfs_test.tmp" workspace and update the location with an
- * exclusive temp directory just for use in the current test jvm.
- *
- * @param pluginRegistry
- * @return JVM exclusive temporary directory location.
- */
- public static void updateDfsTestTmpSchemaLocation(final StoragePluginRegistry pluginRegistry,
- final String tmpDirPath)
- throws ExecutionSetupException {
- @SuppressWarnings("resource")
- final FileSystemPlugin plugin = (FileSystemPlugin) pluginRegistry.getPlugin(dfsTestPluginName);
- final FileSystemConfig pluginConfig = (FileSystemConfig) plugin.getConfig();
- final WorkspaceConfig tmpWSConfig = pluginConfig.workspaces.get(dfsTestTmpSchema);
-
- final WorkspaceConfig newTmpWSConfig = new WorkspaceConfig(tmpDirPath, true, tmpWSConfig.getDefaultInputFormat());
-
- pluginConfig.workspaces.remove(dfsTestTmpSchema);
- pluginConfig.workspaces.put(dfsTestTmpSchema, newTmpWSConfig);
-
- pluginRegistry.createOrUpdate(dfsTestPluginName, pluginConfig, true);
- }
-
- /**
- * Make the dfs.tmp schema immutable, so that tests writers don't use the dfs.tmp to create views.
- * Schema "dfs.tmp" added as part of the default bootstrap plugins file that comes with drill-java-exec jar
- */
- public static void makeDfsTmpSchemaImmutable(final StoragePluginRegistry pluginRegistry) throws ExecutionSetupException {
- @SuppressWarnings("resource")
- final FileSystemPlugin dfsPlugin = (FileSystemPlugin) pluginRegistry.getPlugin(dfsPluginName);
- final FileSystemConfig dfsPluginConfig = (FileSystemConfig) dfsPlugin.getConfig();
- final WorkspaceConfig tmpWSConfig = dfsPluginConfig.workspaces.get(dfsTmpSchema);
-
- final WorkspaceConfig newTmpWSConfig = new WorkspaceConfig(tmpWSConfig.getLocation(), false,
- tmpWSConfig.getDefaultInputFormat());
-
- dfsPluginConfig.workspaces.remove(dfsTmpSchema);
- dfsPluginConfig.workspaces.put(dfsTmpSchema, newTmpWSConfig);
-
- pluginRegistry.createOrUpdate(dfsPluginName, dfsPluginConfig, true);
- }
-
- /**
- * Create JSONRecordReader from input strings.
- * @param jsonBatches : list of input strings, each element represent a batch. Each string could either
- * be in the form of "[{...}, {...}, ..., {...}]", or in the form of "{...}".
- * @param fragContext : fragment context
- * @param columnsToRead : list of schema paths to read from JSON reader.
- * @return
- */
- public static Iterator<RecordReader> getJsonReadersFromBatchString(List<String> jsonBatches, FragmentContext fragContext, List<SchemaPath> columnsToRead) {
- ObjectMapper mapper = new ObjectMapper();
- List<RecordReader> readers = new ArrayList<>();
- for (String batchJason : jsonBatches) {
- JsonNode records;
- try {
- records = mapper.readTree(batchJason);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- readers.add(new JSONRecordReader(fragContext, records, null, columnsToRead));
- }
- return readers.iterator();
- }
-
- /**
- * Create JSONRecordReader from files on a file system.
- * @param fs : file system.
- * @param inputPaths : list of .json file paths.
- * @param fragContext
- * @param columnsToRead
- * @return
- */
- public static Iterator<RecordReader> getJsonReadersFromInputFiles(DrillFileSystem fs, List<String> inputPaths, FragmentContext fragContext, List<SchemaPath> columnsToRead) {
- List<RecordReader> readers = new ArrayList<>();
- for (String inputPath : inputPaths) {
- readers.add(new JSONRecordReader(fragContext, inputPath, fs, columnsToRead));
- }
- return readers.iterator();
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index d95f421..a66fce0 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -206,6 +206,7 @@ drill.exec: {
}
},
compile: {
+ codegen.debug.topn: false,
compiler: "DEFAULT",
debug: true,
janino_maxsize: 262144,
[09/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
index 00a7087..a8eef3c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestInfoSchema.java
@@ -27,18 +27,20 @@ import static org.junit.Assert.assertTrue;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlTest;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.store.dfs.FileSystemConfig;
import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
@@ -51,9 +53,15 @@ import java.util.Map;
*/
@Category(SqlTest.class)
public class TestInfoSchema extends BaseTestQuery {
-
+ public static final String TEST_SUB_DIR = "testSubDir";
private static final ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT);
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data"));
+ dirTestWatcher.makeRootSubDir(Paths.get(TEST_SUB_DIR));
+ }
+
@Test
public void selectFromAllTables() throws Exception{
test("select * from INFORMATION_SCHEMA.SCHEMATA");
@@ -133,9 +141,6 @@ public class TestInfoSchema extends BaseTestQuery {
new String[] { "dfs.tmp" },
new String[] { "cp.default" },
new String[] { "sys" },
- new String[] { "dfs_test.home" },
- new String[] { "dfs_test.default" },
- new String[] { "dfs_test.tmp" },
new String[] { "INFORMATION_SCHEMA" }
);
@@ -161,10 +166,10 @@ public class TestInfoSchema extends BaseTestQuery {
@Test
public void showDatabasesWhere() throws Exception{
testBuilder()
- .sqlQuery("SHOW DATABASES WHERE SCHEMA_NAME='dfs_test.tmp'")
+ .sqlQuery("SHOW DATABASES WHERE SCHEMA_NAME='dfs.tmp'")
.unOrdered()
.baselineColumns("SCHEMA_NAME")
- .baselineValues("dfs_test.tmp")
+ .baselineValues("dfs.tmp")
.go();
}
@@ -207,13 +212,13 @@ public class TestInfoSchema extends BaseTestQuery {
@Test
public void describeWhenSameTableNameExistsInMultipleSchemas() throws Exception{
try {
- test("USE dfs_test.tmp");
+ test("USE dfs.tmp");
test("CREATE OR REPLACE VIEW `TABLES` AS SELECT full_name FROM cp.`employee.json`");
testBuilder()
.sqlQuery("DESCRIBE `TABLES`")
.unOrdered()
- .optionSettingQueriesForTestQuery("USE dfs_test.tmp")
+ .optionSettingQueriesForTestQuery("USE dfs.tmp")
.baselineColumns("COLUMN_NAME", "DATA_TYPE", "IS_NULLABLE")
.baselineValues("full_name", "ANY", "YES")
.go();
@@ -228,7 +233,7 @@ public class TestInfoSchema extends BaseTestQuery {
.baselineValues("TABLE_TYPE", "CHARACTER VARYING", "NO")
.go();
} finally {
- test("DROP VIEW dfs_test.tmp.`TABLES`");
+ test("DROP VIEW dfs.tmp.`TABLES`");
}
}
@@ -280,9 +285,9 @@ public class TestInfoSchema extends BaseTestQuery {
@Test
public void defaultSchemaDfs() throws Exception{
testBuilder()
- .sqlQuery("SELECT R_REGIONKEY FROM `[WORKING_PATH]/../../sample-data/region.parquet` LIMIT 1")
+ .sqlQuery("SELECT R_REGIONKEY FROM `sample-data/region.parquet` LIMIT 1")
.unOrdered()
- .optionSettingQueriesForTestQuery("USE dfs_test")
+ .optionSettingQueriesForTestQuery("USE dfs")
.baselineColumns("R_REGIONKEY")
.baselineValues(0L)
.go();
@@ -305,7 +310,7 @@ public class TestInfoSchema extends BaseTestQuery {
testBuilder()
.sqlQuery("SELECT full_name FROM cp.`employee.json` LIMIT 1")
.unOrdered()
- .optionSettingQueriesForTestQuery("USE dfs_test")
+ .optionSettingQueriesForTestQuery("USE dfs")
.baselineColumns("full_name")
.baselineValues("Sheri Nowmer")
.go();
@@ -314,27 +319,27 @@ public class TestInfoSchema extends BaseTestQuery {
@Test
public void useSchema() throws Exception{
testBuilder()
- .sqlQuery("USE dfs_test.`default`")
+ .sqlQuery("USE dfs.`default`")
.unOrdered()
.baselineColumns("ok", "summary")
- .baselineValues(true, "Default schema changed to [dfs_test.default]")
+ .baselineValues(true, "Default schema changed to [dfs.default]")
.go();
}
@Test
public void useSubSchemaWithinSchema() throws Exception{
testBuilder()
- .sqlQuery("USE dfs_test")
+ .sqlQuery("USE dfs")
.unOrdered()
.baselineColumns("ok", "summary")
- .baselineValues(true, "Default schema changed to [dfs_test]")
+ .baselineValues(true, "Default schema changed to [dfs]")
.go();
testBuilder()
.sqlQuery("USE tmp")
.unOrdered()
.baselineColumns("ok", "summary")
- .baselineValues(true, "Default schema changed to [dfs_test.tmp]")
+ .baselineValues(true, "Default schema changed to [dfs.tmp]")
.go();
testBuilder()
@@ -352,7 +357,7 @@ public class TestInfoSchema extends BaseTestQuery {
}
// Tests using backticks around the complete schema path
- // select * from `dfs_test.tmp`.`/tmp/nation.parquet`;
+ // select * from `dfs.tmp`.`/tmp/nation.parquet`;
@Test
public void completeSchemaRef1() throws Exception {
test("SELECT * FROM `cp.default`.`employee.json` limit 2");
@@ -360,26 +365,26 @@ public class TestInfoSchema extends BaseTestQuery {
@Test
public void showFiles() throws Exception {
- test("show files from dfs_test.`/tmp`");
- test("show files from `dfs_test.default`.`/tmp`");
+ test("show files from dfs.`%s`", TEST_SUB_DIR);
+ test("show files from `dfs.default`.`%s`", TEST_SUB_DIR);
}
@Test
public void showFilesWithDefaultSchema() throws Exception{
- test("USE dfs_test.`default`");
- test("SHOW FILES FROM `/tmp`");
+ test("USE dfs.`default`");
+ test("SHOW FILES FROM `%s`", TEST_SUB_DIR);
}
@Test
public void describeSchemaSyntax() throws Exception {
- test("describe schema dfs_test");
- test("describe schema dfs_test.`default`");
- test("describe database dfs_test.`default`");
+ test("describe schema dfs");
+ test("describe schema dfs.`default`");
+ test("describe database dfs.`default`");
}
@Test
public void describeSchemaOutput() throws Exception {
- final List<QueryDataBatch> result = testSqlWithResults("describe schema dfs_test.tmp");
+ final List<QueryDataBatch> result = testSqlWithResults("describe schema dfs.tmp");
assertTrue(result.size() == 1);
final QueryDataBatch batch = result.get(0);
final RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator());
@@ -390,7 +395,7 @@ public class TestInfoSchema extends BaseTestQuery {
NullableVarCharVector.class,
loader.getValueVectorId(SchemaPath.getCompoundPath("schema")).getFieldIds());
String schema = schemaValueVector.getValueVector().getAccessor().getObject(0).toString();
- assertEquals("dfs_test.tmp", schema);
+ assertEquals("dfs.tmp", schema);
// check properties column value
final VectorWrapper propertiesValueVector = loader.getValueAccessorById(
@@ -408,7 +413,7 @@ public class TestInfoSchema extends BaseTestQuery {
// check some stable properties values
assertEquals("file", configMap.get("type"));
- final FileSystemConfig testConfig = (FileSystemConfig) bits[0].getContext().getStorage().getPlugin("dfs_test").getConfig();
+ final FileSystemConfig testConfig = (FileSystemConfig) bits[0].getContext().getStorage().getPlugin("dfs").getConfig();
final String tmpSchemaLocation = testConfig.workspaces.get("tmp").getLocation();
assertEquals(tmpSchemaLocation, configMap.get("location"));
@@ -420,5 +425,4 @@ public class TestInfoSchema extends BaseTestQuery {
public void describeSchemaInvalid() throws Exception {
errorMsgTestHelper("describe schema invalid.schema", "Invalid schema name [invalid.schema]");
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java
index ea64c7d..00c7af4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestSimpleCastFunctions.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.sql;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
index c2bd215..d571c66 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
@@ -21,23 +21,34 @@ import com.google.common.collect.ImmutableList;
import org.apache.commons.io.FileUtils;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.categories.UnlikelyTest;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.io.File;
+import java.nio.file.Paths;
import java.util.List;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA;
+
@Category(SqlTest.class)
public class TestViewSupport extends TestBaseViewSupport {
+
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("nation"));
+ }
+
@Test
public void referToSchemaInsideAndOutsideView() throws Exception {
- String use = "use dfs_test.tmp;";
+ String use = "use dfs.tmp;";
String selectInto = "create table monkey as select c_custkey, c_nationkey from cp.`tpch/customer.parquet`";
String createView = "create or replace view myMonkeyView as select c_custkey, c_nationkey from monkey";
String selectInside = "select * from myMonkeyView;";
String use2 = "use cp;";
- String selectOutside = "select * from dfs_test.tmp.myMonkeyView;";
+ String selectOutside = "select * from dfs.tmp.myMonkeyView;";
test(use);
test(selectInto);
@@ -56,7 +67,7 @@ public class TestViewSupport extends TestBaseViewSupport {
final String viewName = "testNullabilityPropertyInViewPersistence";
try {
- test("USE dfs_test.tmp");
+ test("USE dfs.tmp");
test(String.format("CREATE OR REPLACE VIEW %s AS SELECT " +
"CAST(customer_id AS BIGINT) as cust_id, " +
"CAST(fname AS VARCHAR(25)) as fname, " +
@@ -88,7 +99,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Test
public void viewWithStarInDef_StarInQuery() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT * FROM cp.`region.json` ORDER BY `region_id`",
"SELECT * FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 1",
@@ -101,7 +112,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Test
public void viewWithSelectFieldsInDef_StarInQuery() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT region_id, sales_city FROM cp.`region.json` ORDER BY `region_id`",
"SELECT * FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 2",
@@ -116,7 +127,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Test
public void viewWithSelectFieldsInDef_SelectFieldsInView_StarInQuery() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
"(regionid, salescity)",
"SELECT region_id, sales_city FROM cp.`region.json` ORDER BY `region_id`",
"SELECT * FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 2",
@@ -131,7 +142,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Test
public void viewWithStarInDef_SelectFieldsInQuery() throws Exception{
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT * FROM cp.`region.json` ORDER BY `region_id`",
"SELECT region_id, sales_city FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 2",
@@ -146,7 +157,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Test
public void viewWithSelectFieldsInDef_SelectFieldsInQuery1() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT region_id, sales_city FROM cp.`region.json` ORDER BY `region_id`",
"SELECT region_id, sales_city FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 2",
@@ -161,7 +172,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Test
public void viewWithSelectFieldsInDef_SelectFieldsInQuery2() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT region_id, sales_city FROM cp.`region.json` ORDER BY `region_id`",
"SELECT sales_city FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 2",
@@ -176,7 +187,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Test
public void viewWithSelectFieldsInDef_SelectFieldsInView_SelectFieldsInQuery1() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
"(regionid, salescity)",
"SELECT region_id, sales_city FROM cp.`region.json` ORDER BY `region_id` LIMIT 2",
"SELECT regionid, salescity FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 2",
@@ -191,7 +202,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Test
public void viewWithSelectFieldsInDef_SelectFieldsInView_SelectFieldsInQuery2() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
"(regionid, salescity)",
"SELECT region_id, sales_city FROM cp.`region.json` ORDER BY `region_id` DESC",
"SELECT regionid FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 2",
@@ -207,7 +218,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Ignore("DRILL-1921")
public void viewWithUnionWithSelectFieldsInDef_StarInQuery() throws Exception{
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT region_id FROM cp.`region.json` UNION SELECT employee_id FROM cp.`employee.json`",
"SELECT regionid FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 2",
@@ -225,20 +236,20 @@ public class TestViewSupport extends TestBaseViewSupport {
final String outerView = generateViewName();
try {
- createViewHelper(TEMP_SCHEMA, innerView, TEMP_SCHEMA, null,
+ createViewHelper(DFS_TMP_SCHEMA, innerView, DFS_TMP_SCHEMA, null,
"SELECT region_id, sales_city FROM cp.`region.json` ORDER BY `region_id`");
- createViewHelper(TEMP_SCHEMA, outerView, TEMP_SCHEMA, null,
- String.format("SELECT region_id FROM %s.`%s`", TEMP_SCHEMA, innerView));
+ createViewHelper(DFS_TMP_SCHEMA, outerView, DFS_TMP_SCHEMA, null,
+ String.format("SELECT region_id FROM %s.`%s`", DFS_TMP_SCHEMA, innerView));
queryViewHelper(
- String.format("SELECT region_id FROM %s.`%s` LIMIT 1", TEMP_SCHEMA, outerView),
+ String.format("SELECT region_id FROM %s.`%s` LIMIT 1", DFS_TMP_SCHEMA, outerView),
new String[] { "region_id" },
ImmutableList.of(new Object[] { 0L })
);
} finally {
- dropViewHelper(TEMP_SCHEMA, outerView, TEMP_SCHEMA);
- dropViewHelper(TEMP_SCHEMA, innerView, TEMP_SCHEMA);
+ dropViewHelper(DFS_TMP_SCHEMA, outerView, DFS_TMP_SCHEMA);
+ dropViewHelper(DFS_TMP_SCHEMA, innerView, DFS_TMP_SCHEMA);
}
}
@@ -250,10 +261,10 @@ public class TestViewSupport extends TestBaseViewSupport {
"cast(columns[1] AS CHAR(25)) n_name, " +
"cast(columns[2] AS INT) n_regionkey, " +
"cast(columns[3] AS VARCHAR(152)) n_comment " +
- "FROM dfs_test.`[WORKING_PATH]/src/test/resources/nation`";
+ "FROM dfs.`nation`";
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
viewDef,
"SELECT * FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 1",
@@ -272,33 +283,31 @@ public class TestViewSupport extends TestBaseViewSupport {
final String viewDef1 = "SELECT region_id, sales_city FROM cp.`region.json`";
// Create the view
- createViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA, null, viewDef1);
+ createViewHelper(DFS_TMP_SCHEMA, viewName, DFS_TMP_SCHEMA, null, viewDef1);
// Try to create the view with same name in same schema.
- final String createViewSql = String.format("CREATE VIEW %s.`%s` AS %s", TEMP_SCHEMA, viewName, viewDef1);
+ final String createViewSql = String.format("CREATE VIEW %s.`%s` AS %s", DFS_TMP_SCHEMA, viewName, viewDef1);
errorMsgTestHelper(createViewSql,
- String.format("A view with given name [%s] already exists in schema [%s]", viewName, TEMP_SCHEMA));
+ String.format("A view with given name [%s] already exists in schema [%s]", viewName, DFS_TMP_SCHEMA));
// Try creating the view with same name in same schema, but with CREATE OR REPLACE VIEW clause
final String viewDef2 = "SELECT sales_state_province FROM cp.`region.json` ORDER BY `region_id`";
- final String createOrReplaceViewSql = String.format("CREATE OR REPLACE VIEW %s.`%s` AS %s", TEMP_SCHEMA,
- viewName, viewDef2);
testBuilder()
- .sqlQuery(createOrReplaceViewSql)
+ .sqlQuery("CREATE OR REPLACE VIEW %s.`%s` AS %s", DFS_TMP_SCHEMA, viewName, viewDef2)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true,
- String.format("View '%s' replaced successfully in '%s' schema", viewName, TEMP_SCHEMA))
+ String.format("View '%s' replaced successfully in '%s' schema", viewName, DFS_TMP_SCHEMA))
.go();
// Make sure the new view created returns the data expected.
- queryViewHelper(String.format("SELECT * FROM %s.`%s` LIMIT 1", TEMP_SCHEMA, viewName),
+ queryViewHelper(String.format("SELECT * FROM %s.`%s` LIMIT 1", DFS_TMP_SCHEMA, viewName),
new String[]{"sales_state_province"},
ImmutableList.of(new Object[]{"None"})
);
} finally {
- dropViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA);
+ dropViewHelper(DFS_TMP_SCHEMA, viewName, DFS_TMP_SCHEMA);
}
}
@@ -310,19 +319,19 @@ public class TestViewSupport extends TestBaseViewSupport {
try {
final String tableDef1 = "SELECT region_id, sales_city FROM cp.`region.json`";
- test(String.format("CREATE TABLE %s.%s as %s", TEMP_SCHEMA, tableName, tableDef1));
+ test("CREATE TABLE %s.%s as %s", DFS_TMP_SCHEMA, tableName, tableDef1);
// Try to create the view with same name in same schema.
- final String createViewSql = String.format("CREATE VIEW %s.`%s` AS %s", TEMP_SCHEMA, tableName, tableDef1);
+ final String createViewSql = String.format("CREATE VIEW %s.`%s` AS %s", DFS_TMP_SCHEMA, tableName, tableDef1);
errorMsgTestHelper(createViewSql,
- String.format("A non-view table with given name [%s] already exists in schema [%s]", tableName, TEMP_SCHEMA));
+ String.format("A non-view table with given name [%s] already exists in schema [%s]", tableName, DFS_TMP_SCHEMA));
// Try creating the view with same name in same schema, but with CREATE OR REPLACE VIEW clause
final String viewDef2 = "SELECT sales_state_province FROM cp.`region.json` ORDER BY `region_id`";
- errorMsgTestHelper(String.format("CREATE OR REPLACE VIEW %s.`%s` AS %s", TEMP_SCHEMA, tableName, viewDef2),
- String.format("A non-view table with given name [%s] already exists in schema [%s]", tableName, TEMP_SCHEMA));
+ errorMsgTestHelper(String.format("CREATE OR REPLACE VIEW %s.`%s` AS %s", DFS_TMP_SCHEMA, tableName, viewDef2),
+ String.format("A non-view table with given name [%s] already exists in schema [%s]", tableName, DFS_TMP_SCHEMA));
} finally {
- FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), tableName));
+ FileUtils.deleteQuietly(new File(dirTestWatcher.getDfsTestTmpDir(), tableName));
}
}
@@ -331,44 +340,44 @@ public class TestViewSupport extends TestBaseViewSupport {
final String viewName = generateViewName();
try {
- test("USE " + TEMP_SCHEMA);
- createViewHelper(null /*pass no schema*/, viewName, TEMP_SCHEMA, null,
+ test("USE " + DFS_TMP_SCHEMA);
+ createViewHelper(null /*pass no schema*/, viewName, DFS_TMP_SCHEMA, null,
"SELECT cast(`employee_id` as integer) employeeid FROM cp.`employee.json`");
// Test SHOW TABLES on view
testBuilder()
- .sqlQuery(String.format("SHOW TABLES like '%s'", viewName))
+ .sqlQuery("SHOW TABLES like '%s'", viewName)
.unOrdered()
.baselineColumns("TABLE_SCHEMA", "TABLE_NAME")
- .baselineValues(TEMP_SCHEMA, viewName)
+ .baselineValues(DFS_TMP_SCHEMA, viewName)
.go();
// Test record in INFORMATION_SCHEMA.VIEWS
testBuilder()
- .sqlQuery(String.format("SELECT * FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_NAME = '%s'", viewName))
+ .sqlQuery("SELECT * FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_NAME = '%s'", viewName)
.unOrdered()
.baselineColumns("TABLE_CATALOG", "TABLE_SCHEMA", "TABLE_NAME", "VIEW_DEFINITION")
- .baselineValues("DRILL", TEMP_SCHEMA, viewName,
- "SELECT CAST(`employee_id` AS INTEGER) AS `employeeid`\n" + "FROM `cp`.`employee.json`")
+ .baselineValues("DRILL", DFS_TMP_SCHEMA, viewName,
+ "SELECT CAST(`employee_id` AS INTEGER) AS `employeeid`\nFROM `cp`.`employee.json`")
.go();
// Test record in INFORMATION_SCHEMA.TABLES
testBuilder()
- .sqlQuery(String.format("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME = '%s'", viewName))
+ .sqlQuery("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME = '%s'", viewName)
.unOrdered()
.baselineColumns("TABLE_CATALOG", "TABLE_SCHEMA", "TABLE_NAME", "TABLE_TYPE")
- .baselineValues("DRILL", TEMP_SCHEMA, viewName, "VIEW")
+ .baselineValues("DRILL", DFS_TMP_SCHEMA, viewName, "VIEW")
.go();
// Test DESCRIBE view
testBuilder()
- .sqlQuery(String.format("DESCRIBE `%s`", viewName))
+ .sqlQuery("DESCRIBE `%s`", viewName)
.unOrdered()
.baselineColumns("COLUMN_NAME", "DATA_TYPE", "IS_NULLABLE")
.baselineValues("employeeid", "INTEGER", "YES")
.go();
} finally {
- dropViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA);
+ dropViewHelper(DFS_TMP_SCHEMA, viewName, DFS_TMP_SCHEMA);
}
}
@@ -377,35 +386,35 @@ public class TestViewSupport extends TestBaseViewSupport {
final String viewName = generateViewName();
try {
- // Change default schema to just "dfs_test". View is actually created in "dfs_test.tmp" schema.
- test("USE dfs_test");
+ // Change default schema to just "dfs". View is actually created in "dfs.tmp" schema.
+ test("USE dfs");
// Create a view with with "tmp" schema identifier
- createViewHelper("tmp", viewName, TEMP_SCHEMA, null,
+ createViewHelper("tmp", viewName, DFS_TMP_SCHEMA, null,
"SELECT CAST(`employee_id` AS INTEGER) AS `employeeid`\n" + "FROM `cp`.`employee.json`");
final String[] baselineColumns = new String[] { "employeeid" };
final List<Object[]> baselineValues = ImmutableList.of(new Object[] { 1156 });
- // Query view from current schema "dfs_test" by referring to the view using "tmp.viewName"
+ // Query view from current schema "dfs" by referring to the view using "tmp.viewName"
queryViewHelper(
String.format("SELECT * FROM %s.`%s` ORDER BY `employeeid` DESC LIMIT 1", "tmp", viewName),
baselineColumns, baselineValues);
- // Change the default schema to "dfs_test.tmp" and query view by referring to it using "viewName"
- test("USE dfs_test.tmp");
+ // Change the default schema to "dfs.tmp" and query view by referring to it using "viewName"
+ test("USE dfs.tmp");
queryViewHelper(
String.format("SELECT * FROM `%s` ORDER BY `employeeid` DESC LIMIT 1", viewName),
baselineColumns, baselineValues);
- // Change the default schema to "cp" and query view by referring to it using "dfs_test.tmp.viewName";
+ // Change the default schema to "cp" and query view by referring to it using "dfs.tmp.viewName";
test("USE cp");
queryViewHelper(
- String.format("SELECT * FROM %s.`%s` ORDER BY `employeeid` DESC LIMIT 1", "dfs_test.tmp", viewName),
+ String.format("SELECT * FROM %s.`%s` ORDER BY `employeeid` DESC LIMIT 1", "dfs.tmp", viewName),
baselineColumns, baselineValues);
} finally {
- dropViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA);
+ dropViewHelper(DFS_TMP_SCHEMA, viewName, DFS_TMP_SCHEMA);
}
}
@@ -419,24 +428,24 @@ public class TestViewSupport extends TestBaseViewSupport {
test("USE cp");
// Create a view with full schema identifier and refer the "region.json" as without schema.
- createViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA, null, "SELECT region_id, sales_city FROM `region.json`");
+ createViewHelper(DFS_TMP_SCHEMA, viewName, DFS_TMP_SCHEMA, null, "SELECT region_id, sales_city FROM `region.json`");
final String[] baselineColumns = new String[] { "region_id", "sales_city" };
final List<Object[]> baselineValues = ImmutableList.of(new Object[]{109L, "Santa Fe"});
// Query the view
queryViewHelper(
- String.format("SELECT * FROM %s.`%s` ORDER BY region_id DESC LIMIT 1", "dfs_test.tmp", viewName),
+ String.format("SELECT * FROM %s.`%s` ORDER BY region_id DESC LIMIT 1", DFS_TMP_SCHEMA, viewName),
baselineColumns, baselineValues);
- // Change default schema to "dfs_test" and query by referring to the view using "tmp.viewName"
- test("USE dfs_test");
+ // Change default schema to "dfs" and query by referring to the view using "tmp.viewName"
+ test("USE dfs");
queryViewHelper(
- String.format("SELECT * FROM %s.`%s` ORDER BY region_id DESC LIMIT 1", "tmp", viewName),
+ String.format("SELECT * FROM %s.`%s` ORDER BY region_id DESC LIMIT 1", TMP_SCHEMA, viewName),
baselineColumns, baselineValues);
} finally {
- dropViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA);
+ dropViewHelper(DFS_TMP_SCHEMA, viewName, DFS_TMP_SCHEMA);
}
}
@@ -448,8 +457,8 @@ public class TestViewSupport extends TestBaseViewSupport {
final String viewName = generateViewName();
try {
- test("USE " + TEMP_SCHEMA);
- createViewHelper(null, viewName, TEMP_SCHEMA, "(id, name, bday)",
+ test("use %s", DFS_TMP_SCHEMA);
+ createViewHelper(null, viewName, DFS_TMP_SCHEMA, "(id, name, bday)",
"SELECT " +
"cast(`region_id` as integer), " +
"cast(`full_name` as varchar(100)), " +
@@ -458,7 +467,7 @@ public class TestViewSupport extends TestBaseViewSupport {
// Test DESCRIBE view
testBuilder()
- .sqlQuery(String.format("DESCRIBE `%s`", viewName))
+ .sqlQuery("DESCRIBE `%s`", viewName)
.unOrdered()
.baselineColumns("COLUMN_NAME", "DATA_TYPE", "IS_NULLABLE")
.baselineValues("id", "INTEGER", "YES")
@@ -466,7 +475,7 @@ public class TestViewSupport extends TestBaseViewSupport {
.baselineValues("bday", "DATE", "YES")
.go();
} finally {
- dropViewHelper(TEMP_SCHEMA, viewName, TEMP_SCHEMA);
+ dropViewHelper(DFS_TMP_SCHEMA, viewName, DFS_TMP_SCHEMA);
}
}
@@ -531,7 +540,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Category(UnlikelyTest.class)
public void createViewWithUniqueColsInFieldListDuplicateColsInQuery1() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
"(regionid1, regionid2)",
"SELECT region_id, region_id FROM cp.`region.json` LIMIT 1",
"SELECT * FROM TEST_SCHEMA.TEST_VIEW_NAME",
@@ -546,7 +555,7 @@ public class TestViewSupport extends TestBaseViewSupport {
@Category(UnlikelyTest.class)
public void createViewWithUniqueColsInFieldListDuplicateColsInQuery2() throws Exception {
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
"(regionid1, regionid2)",
"SELECT t1.region_id, t2.region_id FROM cp.`region.json` t1 JOIN cp.`region.json` t2 " +
"ON t1.region_id = t2.region_id LIMIT 1",
@@ -578,15 +587,15 @@ public class TestViewSupport extends TestBaseViewSupport {
}
private static void createViewErrorTestHelper(final String viewSql, final String expErrorMsg) throws Exception {
- final String createViewSql = String.format(viewSql, TEMP_SCHEMA, "duplicateColumnsInViewDef");
+ final String createViewSql = String.format(viewSql, DFS_TMP_SCHEMA, "duplicateColumnsInViewDef");
errorMsgTestHelper(createViewSql, expErrorMsg);
}
@Test // DRILL-2423
@Category(UnlikelyTest.class)
public void showProperMsgWhenDroppingNonExistentView() throws Exception{
- errorMsgTestHelper("DROP VIEW dfs_test.tmp.nonExistentView",
- "Unknown view [nonExistentView] in schema [dfs_test.tmp].");
+ errorMsgTestHelper("DROP VIEW dfs.tmp.nonExistentView",
+ "Unknown view [nonExistentView] in schema [dfs.tmp].");
}
@Test // DRILL-2423
@@ -601,13 +610,13 @@ public class TestViewSupport extends TestBaseViewSupport {
public void showProperMsgWhenTryingToDropANonViewTable() throws Exception{
final String testTableName = "testTableShowErrorMsg";
try {
- test(String.format("CREATE TABLE %s.%s AS SELECT c_custkey, c_nationkey from cp.`tpch/customer.parquet`",
- TEMP_SCHEMA, testTableName));
+ test("CREATE TABLE %s.%s AS SELECT c_custkey, c_nationkey from cp.`tpch/customer.parquet`",
+ DFS_TMP_SCHEMA, testTableName);
- errorMsgTestHelper(String.format("DROP VIEW %s.%s", TEMP_SCHEMA, testTableName),
- "[testTableShowErrorMsg] is not a VIEW in schema [dfs_test.tmp]");
+ errorMsgTestHelper(String.format("DROP VIEW %s.%s", DFS_TMP_SCHEMA, testTableName),
+ "[testTableShowErrorMsg] is not a VIEW in schema [dfs.tmp]");
} finally {
- File tblPath = new File(getDfsTestTmpSchemaLocation(), testTableName);
+ File tblPath = new File(dirTestWatcher.getDfsTestTmpDir(), testTableName);
FileUtils.deleteQuietly(tblPath);
}
}
@@ -617,9 +626,9 @@ public class TestViewSupport extends TestBaseViewSupport {
final String existentViewName = generateViewName();
// successful dropping of existent view
- createViewHelper(TEMP_SCHEMA, existentViewName, TEMP_SCHEMA, null,
+ createViewHelper(DFS_TMP_SCHEMA, existentViewName, DFS_TMP_SCHEMA, null,
"SELECT c_custkey, c_nationkey from cp.`tpch/customer.parquet`");
- dropViewIfExistsHelper(TEMP_SCHEMA, existentViewName, TEMP_SCHEMA, true);
+ dropViewIfExistsHelper(DFS_TMP_SCHEMA, existentViewName, DFS_TMP_SCHEMA, true);
}
@Test // DRILL-4673
@@ -627,7 +636,7 @@ public class TestViewSupport extends TestBaseViewSupport {
final String nonExistentViewName = generateViewName();
// dropping of non existent view without error
- dropViewIfExistsHelper(TEMP_SCHEMA, nonExistentViewName, TEMP_SCHEMA, false);
+ dropViewIfExistsHelper(DFS_TMP_SCHEMA, nonExistentViewName, DFS_TMP_SCHEMA, false);
}
@Test // DRILL-4673
@@ -635,11 +644,11 @@ public class TestViewSupport extends TestBaseViewSupport {
final String tableName = "table_name";
try{
// dropping of non existent view without error if the table with such name is existed
- test(String.format("CREATE TABLE %s.%s as SELECT region_id, sales_city FROM cp.`region.json`",
- TEMP_SCHEMA, tableName));
- dropViewIfExistsHelper(TEMP_SCHEMA, tableName, TEMP_SCHEMA, false);
+ test("CREATE TABLE %s.%s as SELECT region_id, sales_city FROM cp.`region.json`",
+ DFS_TMP_SCHEMA, tableName);
+ dropViewIfExistsHelper(DFS_TMP_SCHEMA, tableName, DFS_TMP_SCHEMA, false);
} finally {
- test(String.format("DROP TABLE IF EXISTS %s.%s ", TEMP_SCHEMA, tableName));
+ test("DROP TABLE IF EXISTS %s.%s ", DFS_TMP_SCHEMA, tableName);
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java
index b901120..51821e9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.sql;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlTest;
import org.junit.Ignore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
index d1a16df..324fc73 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
@@ -19,30 +19,31 @@ package org.apache.drill.exec.store;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.util.Text;
import org.apache.drill.test.rowSet.SchemaBuilder;
-import org.apache.hadoop.fs.Path;
-import org.junit.Before;
-import org.junit.Rule;
+import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
public class TestImplicitFileColumns extends BaseTestQuery {
-
+ public static final String CSV = "csv";
public static final String MAIN = "main";
public static final String NESTED = "nested";
- public static final String CSV = "csv";
- public final String JSON_TBL = "/scan/jsonTbl"; // 1990/1.json : {id:100, name: "John"}, 1991/2.json : {id: 1000, name : "Joe"}
- public final String PARQUET_TBL = "/multilevel/parquet/"; // 1990/Q1/orders_1990_q1.parquet, ...
- public final String CSV_TBL = "/multilevel/csv"; // 1990/Q1/orders_1990_q1.csv, ..
+ public static final String MAIN_FILE = MAIN + "." + CSV;
+ public static final String NESTED_FILE = NESTED + "." + CSV;
+ public static final Path FILES = Paths.get("files");
+ public static final Path NESTED_DIR = FILES.resolve(NESTED);
+ public static final Path JSON_TBL = Paths.get("scan", "jsonTbl"); // 1990/1.json : {id:100, name: "John"}, 1991/2.json : {id: 1000, name : "Joe"}
+ public static final Path PARQUET_TBL = Paths.get("multilevel", "parquet"); // 1990/Q1/orders_1990_q1.parquet, ...
+ public static final Path PARQUET_CHANGE_TBL = Paths.get("multilevel", "parquetWithSchemaChange");
+ public static final Path CSV_TBL = Paths.get("multilevel", "csv"); // 1990/Q1/orders_1990_q1.csv, ..
private static final JsonStringArrayList<Text> mainColumnValues = new JsonStringArrayList<Text>() {{
add(new Text(MAIN));
@@ -51,37 +52,40 @@ public class TestImplicitFileColumns extends BaseTestQuery {
add(new Text(NESTED));
}};
- @Rule
- public TemporaryFolder testFolder = new TemporaryFolder();
+ private static File mainFile;
+ private static File nestedFile;
- private File mainFile;
- private File nestedFolder;
- private File nestedFile;
-
- @Before
- public void setup() throws Exception {
- mainFile = testFolder.newFile(MAIN + "." + CSV);
+ @BeforeClass
+ public static void setup() throws Exception {
+ File files = dirTestWatcher.makeRootSubDir(FILES);
+ mainFile = new File(files, MAIN_FILE);
Files.write(MAIN, mainFile, Charsets.UTF_8);
- nestedFolder = testFolder.newFolder(NESTED);
- nestedFile = new File(nestedFolder, NESTED + "." + CSV);
+ File nestedFolder = new File(files, NESTED);
+ nestedFolder.mkdirs();
+ nestedFile = new File(nestedFolder, NESTED_FILE);
Files.write(NESTED, nestedFile, Charsets.UTF_8);
+
+ dirTestWatcher.copyResourceToRoot(JSON_TBL);
+ dirTestWatcher.copyResourceToRoot(PARQUET_TBL);
+ dirTestWatcher.copyResourceToRoot(CSV_TBL);
+ dirTestWatcher.copyResourceToRoot(PARQUET_CHANGE_TBL);
}
@Test
public void testImplicitColumns() throws Exception {
testBuilder()
- .sqlQuery("select *, filename, suffix, fqn, filepath from dfs.`%s` order by filename", testFolder.getRoot().getPath())
+ .sqlQuery("select *, filename, suffix, fqn, filepath from dfs.`%s` order by filename", FILES)
.ordered()
.baselineColumns("columns", "dir0", "filename", "suffix", "fqn", "filepath")
- .baselineValues(mainColumnValues, null, mainFile.getName(), CSV, new Path(mainFile.getPath()).toString(), new Path(mainFile.getParent()).toString())
- .baselineValues(nestedColumnValues, NESTED, nestedFile.getName(), CSV, new Path(nestedFile.getPath()).toString(), new Path(nestedFile.getParent()).toString())
+ .baselineValues(mainColumnValues, null, mainFile.getName(), CSV, mainFile.getCanonicalPath(), mainFile.getParentFile().getCanonicalPath())
+ .baselineValues(nestedColumnValues, NESTED, NESTED_FILE, CSV, nestedFile.getCanonicalPath(), nestedFile.getParentFile().getCanonicalPath())
.go();
}
@Test
public void testImplicitColumnInWhereClause() throws Exception {
testBuilder()
- .sqlQuery("select * from dfs.`%s` where filename = '%s'", nestedFolder.getPath(), nestedFile.getName())
+ .sqlQuery("select * from dfs.`%s` where filename = '%s'", NESTED_DIR, NESTED_FILE)
.unOrdered()
.baselineColumns("columns")
.baselineValues(nestedColumnValues)
@@ -91,27 +95,27 @@ public class TestImplicitFileColumns extends BaseTestQuery {
@Test
public void testImplicitColumnAlone() throws Exception {
testBuilder()
- .sqlQuery("select filename from dfs.`%s`", nestedFolder.getPath())
+ .sqlQuery("select filename from dfs.`%s`", NESTED_DIR)
.unOrdered()
.baselineColumns("filename")
- .baselineValues(nestedFile.getName())
+ .baselineValues(NESTED_FILE)
.go();
}
@Test
public void testImplicitColumnWithTableColumns() throws Exception {
testBuilder()
- .sqlQuery("select columns, filename from dfs.`%s`", nestedFolder.getPath())
+ .sqlQuery("select columns, filename from dfs.`%s`", NESTED_DIR)
.unOrdered()
.baselineColumns("columns", "filename")
- .baselineValues(nestedColumnValues, nestedFile.getName())
+ .baselineValues(nestedColumnValues, NESTED_FILE)
.go();
}
@Test
public void testCountStarWithImplicitColumnsInWhereClause() throws Exception {
testBuilder()
- .sqlQuery("select count(*) as cnt from dfs.`%s` where filename = '%s'", nestedFolder.getPath(), nestedFile.getName())
+ .sqlQuery("select count(*) as cnt from dfs.`%s` where filename = '%s'", NESTED_DIR, NESTED_FILE)
.unOrdered()
.baselineColumns("cnt")
.baselineValues(1L)
@@ -121,10 +125,11 @@ public class TestImplicitFileColumns extends BaseTestQuery {
@Test
public void testImplicitAndPartitionColumnsInSelectClause() throws Exception {
testBuilder()
- .sqlQuery("select dir0, filename from dfs.`%s` order by filename", testFolder.getRoot().getPath()).ordered()
+ .sqlQuery("select dir0, filename from dfs.`%s` order by filename", FILES)
+ .ordered()
.baselineColumns("dir0", "filename")
- .baselineValues(null, mainFile.getName())
- .baselineValues(NESTED, nestedFile.getName())
+ .baselineValues(null, MAIN_FILE)
+ .baselineValues(NESTED, NESTED_FILE)
.go();
}
@@ -143,8 +148,7 @@ public class TestImplicitFileColumns extends BaseTestQuery {
try {
test("alter session set `planner.enable_decimal_data_type` = true");
testBuilder()
- .sqlQuery(String.format("select max(dir0) as max_dir from dfs_test.`%s/src/test/resources/multilevel/parquetWithSchemaChange`",
- TestTools.getWorkingPath()))
+ .sqlQuery("select max(dir0) as max_dir from dfs.`%s`", PARQUET_CHANGE_TBL)
.unOrdered()
.baselineColumns("max_dir")
.baselineValues("voter50")
@@ -156,9 +160,6 @@ public class TestImplicitFileColumns extends BaseTestQuery {
@Test
public void testStarColumnJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(JSON_TBL).toURI().toString();
- final String query1 = String.format("select * from dfs_test.`%s` ", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.addNullable("dir0", TypeProtos.MinorType.VARCHAR)
.addNullable("id", TypeProtos.MinorType.BIGINT)
@@ -166,7 +167,7 @@ public class TestImplicitFileColumns extends BaseTestQuery {
.build();
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select * from dfs.`%s` ", JSON_TBL)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -174,9 +175,6 @@ public class TestImplicitFileColumns extends BaseTestQuery {
@Test
public void testStarColumnParquet() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(PARQUET_TBL).toURI().toString();
- final String query1 = String.format("select * from dfs_test.`%s` ", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.addNullable("dir0", TypeProtos.MinorType.VARCHAR)
.addNullable("dir1", TypeProtos.MinorType.VARCHAR)
@@ -192,7 +190,7 @@ public class TestImplicitFileColumns extends BaseTestQuery {
.build();
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select * from dfs.`%s` ", PARQUET_TBL)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -200,9 +198,6 @@ public class TestImplicitFileColumns extends BaseTestQuery {
@Test
public void testStarColumnCsv() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(CSV_TBL).toURI().toString();
- final String query1 = String.format("select * from dfs_test.`%s` ", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.addNullable("dir0", TypeProtos.MinorType.VARCHAR)
.addNullable("dir1", TypeProtos.MinorType.VARCHAR)
@@ -210,10 +205,9 @@ public class TestImplicitFileColumns extends BaseTestQuery {
.build();
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select * from dfs.`%s` ", CSV_TBL)
.schemaBaseLine(expectedSchema)
.build()
.run();
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java
index 4848a9d..3e6f118 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestTimedRunnable.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.store;
import com.google.common.collect.Lists;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.test.DrillTest;
import org.apache.drill.categories.SlowTest;
import org.junit.Rule;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
index f804e88..6436c1c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
@@ -18,8 +18,8 @@
package org.apache.drill.exec.store.avro;
import com.google.common.collect.Lists;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.util.JsonStringHashMap;
@@ -29,7 +29,21 @@ import org.junit.Test;
import java.util.List;
import java.util.Map;
-import static org.apache.drill.TestBuilder.listOf;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateDoubleNestedSchema_NoNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateLinkedList;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateMapSchemaComplex_withNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateMapSchema_withNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateNestedArraySchema;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateSimpleArraySchema_NoNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateSimpleEnumSchema_NoNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateSimpleNestedSchema_NoNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateStringAndUtf8Data;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateUnionNestedArraySchema_withNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateUnionNestedSchema_withNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateUnionSchema_WithNonNullValues;
+import static org.apache.drill.exec.store.avro.AvroTestUtil.generateUnionSchema_WithNullValues;
+import static org.apache.drill.test.TestBuilder.listOf;
/**
* Unit tests for Avro record reader.
@@ -42,15 +56,12 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testBatchCutoff() throws Exception {
-
- final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues(5000);
- final String file = testSetup.getFilePath();
- final String sql =
- "select a_string, b_int, c_long, d_float, e_double, f_bytes, h_boolean, g_null " +
- "from dfs_test.`" + file + "`";
- test(sql);
+ final AvroTestUtil.AvroTestRecordWriter testSetup = generateSimplePrimitiveSchema_NoNullValues(5000);
+ final String file = testSetup.getFileName();
+ final String sql = "select a_string, b_int, c_long, d_float, e_double, f_bytes, h_boolean, g_null from dfs.`%s`";
+ test(sql, file);
testBuilder()
- .sqlQuery(sql)
+ .sqlQuery(sql, file)
.unOrdered()
.expectsNumBatches(2)
.baselineRecords(testSetup.getExpectedRecords())
@@ -67,24 +78,18 @@ public class AvroFormatTest extends BaseTestQuery {
*/
@Test
public void testFiltersOnVarchar() throws Exception {
-
- final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues(5000);
- final String file = testSetup.getFilePath();
- final String sql =
- "select a_string " +
- "from dfs_test.`" + file + "` where a_string = 'a_1'";
+ final String file = generateSimplePrimitiveSchema_NoNullValues(5000).getFileName();
+ final String sql = "select a_string from dfs.`%s` where a_string = 'a_1'";
testBuilder()
- .sqlQuery(sql)
+ .sqlQuery(sql, file)
.unOrdered()
.baselineColumns("a_string")
.baselineValues("a_1")
.go();
- final String sql2 =
- "select a_string " +
- "from dfs_test.`" + file + "` where a_string IN ('a_1')";
+ final String sql2 = "select a_string from dfs.`%s` where a_string IN ('a_1')";
testBuilder()
- .sqlQuery(sql2)
+ .sqlQuery(sql2, file)
.unOrdered()
.baselineColumns("a_string")
.baselineValues("a_1")
@@ -93,13 +98,10 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testFiltersOnVarBinary() throws Exception {
- final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues(5000);
- final String file = testSetup.getFilePath();
- final String sql =
- "select f_bytes " +
- "from dfs_test.`" + file + "` where f_bytes = BINARY_STRING('\\x61\\x31')";
+ final String file = generateSimplePrimitiveSchema_NoNullValues(5000).getFileName();
+ final String sql = "select f_bytes from dfs.`%s` where f_bytes = BINARY_STRING('\\x61\\x31')";
TestBuilder testBuilder = testBuilder()
- .sqlQuery(sql)
+ .sqlQuery(sql, file)
.unOrdered()
.baselineColumns("f_bytes");
@@ -108,11 +110,9 @@ public class AvroFormatTest extends BaseTestQuery {
}
testBuilder.go();
- final String sql2 =
- "select f_bytes " +
- "from dfs_test.`" + file + "` where f_bytes IN (BINARY_STRING('\\x61\\x31'))";
+ final String sql2 = "select f_bytes from dfs.`%s` where f_bytes IN (BINARY_STRING('\\x61\\x31'))";
testBuilder = testBuilder()
- .sqlQuery(sql2)
+ .sqlQuery(sql2, file)
.unOrdered()
.baselineColumns("f_bytes");
@@ -124,15 +124,12 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testSimplePrimitiveSchema_NoNullValues() throws Exception {
-
- final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues();
- final String file = testSetup.getFilePath();
- final String sql =
- "select a_string, b_int, c_long, d_float, e_double, f_bytes, h_boolean, g_null " +
- "from dfs_test.`" + file + "`";
- test(sql);
+ final AvroTestUtil.AvroTestRecordWriter testSetup = generateSimplePrimitiveSchema_NoNullValues();
+ final String file = testSetup.getFileName();
+ final String sql = "select a_string, b_int, c_long, d_float, e_double, f_bytes, h_boolean, g_null from dfs.`%s`";
+ test(sql, file);
testBuilder()
- .sqlQuery(sql)
+ .sqlQuery(sql, file)
.unOrdered()
.baselineRecords(testSetup.getExpectedRecords())
.go();
@@ -140,7 +137,7 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testSimplePrimitiveSchema_StarQuery() throws Exception {
- simpleAvroTestHelper(AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues(), "select * from dfs_test.`%s`");
+ simpleAvroTestHelper(generateSimplePrimitiveSchema_NoNullValues(), "select * from dfs.`%s`");
}
private List<Map<String, Object>> project(
@@ -161,13 +158,11 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testSimplePrimitiveSchema_SelectColumnSubset() throws Exception {
-
- final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues();
- final String file = testSetup.getFilePath();
- final String sql = "select h_boolean, e_double from dfs_test.`" + file + "`";
+ final AvroTestUtil.AvroTestRecordWriter testSetup = generateSimplePrimitiveSchema_NoNullValues();
+ final String file = testSetup.getFileName();
List<String> projectList = Lists.newArrayList("`h_boolean`", "`e_double`");
testBuilder()
- .sqlQuery(sql)
+ .sqlQuery("select h_boolean, e_double from dfs.`%s`", file)
.unOrdered()
.baselineRecords(project(testSetup.getExpectedRecords(), projectList))
.go();
@@ -175,11 +170,9 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testSimplePrimitiveSchema_NoColumnsExistInTheSchema() throws Exception {
-
- final String file = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues().getFilePath();
- final String sql = "select h_dummy1, e_dummy2 from dfs_test.`" + file + "`";
+ final String file = generateSimplePrimitiveSchema_NoNullValues().getFileName();
try {
- test(sql);
+ test("select h_dummy1, e_dummy2 from dfs.`%s`", file);
Assert.fail("Test should fail as h_dummy1 and e_dummy2 does not exist.");
} catch(UserException ue) {
Assert.assertTrue("Test should fail as h_dummy1 and e_dummy2 does not exist.",
@@ -189,11 +182,9 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testSimplePrimitiveSchema_OneExistAndOneDoesNotExistInTheSchema() throws Exception {
-
- final String file = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues().getFilePath();
- final String sql = "select h_boolean, e_dummy2 from dfs_test.`" + file + "`";
+ final String file = generateSimplePrimitiveSchema_NoNullValues().getFileName();
try {
- test(sql);
+ test("select h_boolean, e_dummy2 from dfs.`%s`", file);
Assert.fail("Test should fail as e_dummy2 does not exist.");
} catch(UserException ue) {
Assert.assertTrue("Test should fail as e_dummy2 does not exist.", true);
@@ -202,80 +193,68 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testSimpleArraySchema_NoNullValues() throws Exception {
- final String file = AvroTestUtil.generateSimpleArraySchema_NoNullValues().getFilePath();
- final String sql = "select a_string, c_string_array[0], e_float_array[2] " +
- "from dfs_test.`" + file + "`";
- test(sql);
+ final String file = generateSimpleArraySchema_NoNullValues().getFileName();
+ final String sql = "select a_string, c_string_array[0], e_float_array[2] from dfs.`%s`";
+ test(sql, file);
}
@Test
public void testSimpleArraySchema_StarQuery() throws Exception {
- simpleAvroTestHelper(AvroTestUtil.generateSimpleArraySchema_NoNullValues(), "select * from dfs_test.`%s`");
+ simpleAvroTestHelper(generateSimpleArraySchema_NoNullValues(), "select * from dfs.`%s`");
}
@Test
public void testDoubleNestedSchema_NoNullValues_NotAllColumnsProjected() throws Exception {
- final String file = AvroTestUtil.generateDoubleNestedSchema_NoNullValues().getFilePath();
- final String sql = "select t.c_record.nested_1_int, " +
- "t.c_record.nested_1_record.double_nested_1_int " +
- "from dfs_test.`" + file + "` t";
- test(sql);
+ final String file = generateDoubleNestedSchema_NoNullValues().getFileName();
+ final String sql = "select t.c_record.nested_1_int, t.c_record.nested_1_record.double_nested_1_int from dfs.`%s` t";
+ test(sql, file);
}
@Test
public void testSimpleNestedSchema_NoNullValues() throws Exception {
-
- final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimpleNestedSchema_NoNullValues();
- final String file = testSetup.getFilePath();
- final String sql = "select a_string, b_int, t.c_record.nested_1_string, t.c_record.nested_1_int " +
- "from dfs_test.`" + file + "` t";
- test(sql);
+ final String file = generateSimpleNestedSchema_NoNullValues().getFileName();
+ final String sql = "select a_string, b_int, t.c_record.nested_1_string, t.c_record.nested_1_int from dfs.`%s` t";
+ test(sql, file);
}
@Test
public void testSimpleNestedSchema_StarQuery() throws Exception {
-
- final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimpleNestedSchema_NoNullValues();
- final String file = testSetup.getFilePath();
- final String sql = "select * from dfs_test.`" + file + "`";
+ final AvroTestUtil.AvroTestRecordWriter testSetup = generateSimpleNestedSchema_NoNullValues();
+ final String file = testSetup.getFileName();
testBuilder()
- .sqlQuery(sql)
+ .sqlQuery("select * from dfs.`%s`", file)
.unOrdered()
.baselineRecords(testSetup.getExpectedRecords())
.go();
}
@Test
public void testDoubleNestedSchema_NoNullValues() throws Exception {
- final String file = AvroTestUtil.generateDoubleNestedSchema_NoNullValues().getFilePath();
+ final String file = generateDoubleNestedSchema_NoNullValues().getFileName();
final String sql = "select a_string, b_int, t.c_record.nested_1_string, t.c_record.nested_1_int, " +
"t.c_record.nested_1_record.double_nested_1_string, " +
"t.c_record.nested_1_record.double_nested_1_int " +
- "from dfs_test.`" + file + "` t";
- test(sql);
+ "from dfs.`%s` t";
+ test(sql, file);
- final String sql2 = "select t.c_record.nested_1_string " +
- "from dfs_test.`" + file + "` t limit 1";
+ final String sql2 = "select t.c_record.nested_1_string from dfs.`%s` t limit 1";
TestBuilder testBuilder = testBuilder()
- .sqlQuery(sql2)
+ .sqlQuery(sql2, file)
.unOrdered()
.baselineColumns("EXPR$0");
for (int i = 0; i < 1; i++) {
- testBuilder
- .baselineValues("nested_1_string_" + i);
+ testBuilder.baselineValues("nested_1_string_" + i);
}
testBuilder.go();
}
@Test
public void testDoubleNestedSchema_StarQuery() throws Exception {
- simpleAvroTestHelper(AvroTestUtil.generateDoubleNestedSchema_NoNullValues(), "select * from dfs_test.`%s`");
+ simpleAvroTestHelper(generateDoubleNestedSchema_NoNullValues(), "select * from dfs.`%s`");
}
private static void simpleAvroTestHelper(AvroTestUtil.AvroTestRecordWriter testSetup, final String sql) throws Exception {
- final String file = testSetup.getFilePath();
- final String sqlWithTable = String.format(sql, file);
testBuilder()
- .sqlQuery(sqlWithTable)
+ .sqlQuery(sql, testSetup.getFileName())
.unOrdered()
.baselineRecords(testSetup.getExpectedRecords())
.go();
@@ -283,12 +262,12 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testSimpleEnumSchema_NoNullValues() throws Exception {
- final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimpleEnumSchema_NoNullValues();
- final String file = testSetup.getFilePath();
- final String sql = "select a_string, b_enum from dfs_test.`" + file + "`";
+ final AvroTestUtil.AvroTestRecordWriter testSetup = generateSimpleEnumSchema_NoNullValues();
+ final String file = testSetup.getFileName();
+ final String sql = "select a_string, b_enum from dfs.`%s`";
List<String> projectList = Lists.newArrayList("`a_string`", "`b_enum`");
testBuilder()
- .sqlQuery(sql)
+ .sqlQuery(sql, file)
.unOrdered()
.baselineRecords(project(testSetup.getExpectedRecords(), projectList))
.go();
@@ -296,21 +275,19 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testSimpleEnumSchema_StarQuery() throws Exception {
- simpleAvroTestHelper(AvroTestUtil.generateSimpleEnumSchema_NoNullValues(), "select * from dfs_test.`%s`");
+ simpleAvroTestHelper(generateSimpleEnumSchema_NoNullValues(), "select * from dfs.`%s`");
}
@Test
public void testSimpleUnionSchema_StarQuery() throws Exception {
- simpleAvroTestHelper(AvroTestUtil.generateUnionSchema_WithNullValues(), "select * from dfs_test.`%s`");
+ simpleAvroTestHelper(generateUnionSchema_WithNullValues(), "select * from dfs.`%s`");
}
@Test
public void testShouldFailSimpleUnionNonNullSchema_StarQuery() throws Exception {
-
- final String file = AvroTestUtil.generateUnionSchema_WithNonNullValues().getFilePath();
- final String sql = "select * from dfs_test.`" + file + "`";
+ final String file = generateUnionSchema_WithNonNullValues().getFileName();
try {
- test(sql);
+ test("select * from dfs.`%s`", file);
Assert.fail("Test should fail as union is only supported for optional fields");
} catch(UserRemoteException e) {
String message = e.getMessage();
@@ -320,25 +297,21 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testNestedUnionSchema_withNullValues() throws Exception {
-
- final String file = AvroTestUtil.generateUnionNestedSchema_withNullValues().getFilePath();
- final String sql = "select t.c_record.nested_1_string,t.c_record.nested_1_int from dfs_test.`" + file + "` t";
- test(sql);
+ final String file = generateUnionNestedSchema_withNullValues().getFileName();
+ final String sql = "select t.c_record.nested_1_string,t.c_record.nested_1_int from dfs.`%s` t";
+ test(sql, file);
}
- /**
- * See <a href="https://issues.apache.org/jira/browse/DRILL-4574"></a>
- *
- */
+ // DRILL-4574"></a>
@Test
public void testFlattenPrimitiveArray() throws Exception {
- final String file = AvroTestUtil.generateSimpleArraySchema_NoNullValues().getFilePath();
+ final String file = generateSimpleArraySchema_NoNullValues().getFileName();
+ final String sql = "select a_string, flatten(c_string_array) as array_item from dfs.`%s` t";
- final String sql = "select a_string, flatten(c_string_array) as array_item "
- + "from dfs_test.`" + file + "` t";
-
- TestBuilder testBuilder = testBuilder().sqlQuery(sql).unOrdered()
- .baselineColumns("a_string", "array_item");
+ TestBuilder testBuilder = testBuilder()
+ .sqlQuery(sql, file)
+ .unOrdered()
+ .baselineColumns("a_string", "array_item");
for (int i = 0; i < AvroTestUtil.RECORD_COUNT; i++) {
@@ -347,30 +320,25 @@ public class AvroFormatTest extends BaseTestQuery {
}
}
-
testBuilder.go();
-
}
private TestBuilder nestedArrayQueryTestBuilder(String file) {
-
final String sql = "select rec_nr, array_item['nested_1_int'] as array_item_nested_int from "
- + "(select a_int as rec_nr, flatten(t.b_array) as array_item " + "from dfs_test.`" + file + "` t) a";
+ + "(select a_int as rec_nr, flatten(t.b_array) as array_item from dfs.`%s` t) a";
- TestBuilder testBuilder = testBuilder().sqlQuery(sql).unOrdered().baselineColumns("rec_nr",
- "array_item_nested_int");
+ TestBuilder testBuilder = testBuilder()
+ .sqlQuery(sql, file)
+ .unOrdered()
+ .baselineColumns("rec_nr", "array_item_nested_int");
return testBuilder;
-
}
-
- /**
- * See <a href="https://issues.apache.org/jira/browse/DRILL-4574"></a>
- */
+ //DRILL-4574
@Test
public void testFlattenComplexArray() throws Exception {
- final String file = AvroTestUtil.generateNestedArraySchema().getFilePath();
+ final String file = generateNestedArraySchema().getFileName();
TestBuilder testBuilder = nestedArrayQueryTestBuilder(file);
for (int i = 0; i < AvroTestUtil.RECORD_COUNT; i++) {
@@ -381,40 +349,36 @@ public class AvroFormatTest extends BaseTestQuery {
testBuilder.go();
}
- /**
- * See <a href="https://issues.apache.org/jira/browse/DRILL-4574"></a>
- */
+
+ //DRILL-4574
@Test
public void testFlattenEmptyComplexArrayMustYieldNoResults() throws Exception {
- final String file = AvroTestUtil.generateNestedArraySchema(AvroTestUtil.RECORD_COUNT, 0).getFilePath();
+ final String file = generateNestedArraySchema(AvroTestUtil.RECORD_COUNT, 0).getFilePath();
TestBuilder testBuilder = nestedArrayQueryTestBuilder(file);
testBuilder.expectsEmptyResultSet();
}
@Test
public void testNestedUnionArraySchema_withNullValues() throws Exception {
-
- final String file = AvroTestUtil.generateUnionNestedArraySchema_withNullValues().getFilePath();
- final String sql = "select t.c_array[0].nested_1_string,t.c_array[0].nested_1_int from dfs_test.`" + file + "` t";
- test(sql);
+ final String file = generateUnionNestedArraySchema_withNullValues().getFileName();
+ final String sql = "select t.c_array[0].nested_1_string,t.c_array[0].nested_1_int from dfs.`%s` t";
+ test(sql, file);
}
@Test
public void testMapSchema_withNullValues() throws Exception {
-
- final String file = AvroTestUtil.generateMapSchema_withNullValues().getFilePath();
- final String sql = "select c_map['key1'],c_map['key2'] from dfs_test.`" + file + "`";
- test(sql);
+ final String file = generateMapSchema_withNullValues().getFileName();
+ final String sql = "select c_map['key1'],c_map['key2'] from dfs.`%s`";
+ test(sql, file);
}
@Test
public void testMapSchemaComplex_withNullValues() throws Exception {
-
- final String file = AvroTestUtil.generateMapSchemaComplex_withNullValues().getFilePath();
- final String sql = "select d_map['key1'] nested_key1, d_map['key2'] nested_key2 from dfs_test.`" + file + "`";
+ final String file = generateMapSchemaComplex_withNullValues().getFileName();
+ final String sql = "select d_map['key1'] nested_key1, d_map['key2'] nested_key2 from dfs.`%s`";
TestBuilder testBuilder = testBuilder()
- .sqlQuery(sql)
+ .sqlQuery(sql, file)
.unOrdered()
.baselineColumns("nested_key1", "nested_key2");
@@ -432,26 +396,25 @@ public class AvroFormatTest extends BaseTestQuery {
@Test
public void testStringAndUtf8Data() throws Exception {
- simpleAvroTestHelper(AvroTestUtil.generateStringAndUtf8Data(), "select * from dfs_test.`%s`");
+ simpleAvroTestHelper(generateStringAndUtf8Data(), "select * from dfs.`%s`");
}
@Test
public void testLinkedList() throws Exception {
- final String file = AvroTestUtil.generateLinkedList();
- final String sql = "select * from dfs_test.`" + file + "`";
- test(sql);
+ final String file = generateLinkedList();
+ final String sql = "select * from dfs.`%s`";
+ test(sql, file);
}
@Test
public void testCountStar() throws Exception {
- final String file = AvroTestUtil.generateStringAndUtf8Data().getFilePath();
- final String sql = "select count(*) as row_count from dfs_test.`" + file + "`";
+ final String file = generateStringAndUtf8Data().getFileName();
+ final String sql = "select count(*) as row_count from dfs.`%s`";
testBuilder()
- .sqlQuery(sql)
+ .sqlQuery(sql, file)
.ordered()
.baselineColumns("row_count")
.baselineValues((long)AvroTestUtil.RECORD_COUNT)
.go();
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java
index 86d29ae..1cb9284 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java
@@ -37,10 +37,10 @@ import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
-import com.google.common.base.Charsets;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.util.JsonStringHashMap;
import org.apache.drill.exec.util.Text;
+import org.apache.drill.test.BaseTestQuery;
/**
* Utilities for generating Avro test data.
@@ -62,6 +62,7 @@ public class AvroTestUtil {
private Schema schema;
private final DataFileWriter<GenericData.Record> writer;
private final String filePath;
+ private final String fileName;
private AvroTestRecordWriter(Schema schema, File file) {
writer = new DataFileWriter<GenericData.Record>(new GenericDatumWriter<GenericData.Record>(schema));
@@ -74,6 +75,7 @@ public class AvroTestUtil {
currentExpectedRecord = new TreeMap<>();
expectedRecords = new ArrayList<>();
filePath = file.getAbsolutePath();
+ fileName = file.getName();
}
public void startRecord() {
@@ -134,6 +136,10 @@ public class AvroTestUtil {
return filePath;
}
+ public String getFileName() {
+ return fileName;
+ }
+
public List<Map<String, Object>>getExpectedRecords() {
return expectedRecords;
}
@@ -145,7 +151,6 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateSimplePrimitiveSchema_NoNullValues(int numRecords) throws Exception {
-
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -159,10 +164,9 @@ public class AvroTestUtil {
.name("h_boolean").type().booleanType().noDefault()
.endRecord();
- final File file = File.createTempFile("avro-primitive-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-primitive-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final AvroTestRecordWriter record = new AvroTestRecordWriter(schema, file);
+
try {
ByteBuffer bb = ByteBuffer.allocate(2);
bb.put(0, (byte) 'a');
@@ -189,7 +193,6 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateUnionSchema_WithNullValues() throws Exception {
-
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -204,12 +207,10 @@ public class AvroTestUtil {
.name("i_union").type().optional().doubleType()
.endRecord();
- final File file = File.createTempFile("avro-primitive-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-primitive-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final AvroTestRecordWriter record = new AvroTestRecordWriter(schema, file);
- try {
+ try {
ByteBuffer bb = ByteBuffer.allocate(1);
bb.put(0, (byte) 1);
@@ -234,7 +235,6 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateUnionSchema_WithNonNullValues() throws Exception {
-
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -249,9 +249,7 @@ public class AvroTestUtil {
.name("i_union").type().unionOf().doubleType().and().longType().endUnion().noDefault()
.endRecord();
- final File file = File.createTempFile("avro-primitive-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-primitive-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final AvroTestRecordWriter record = new AvroTestRecordWriter(schema, file);
try {
@@ -279,7 +277,6 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateSimpleEnumSchema_NoNullValues() throws Exception {
-
final String[] symbols = { "E_SYM_A", "E_SYM_B", "E_SYM_C", "E_SYM_D" };
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
@@ -289,9 +286,7 @@ public class AvroTestUtil {
.name("b_enum").type().enumeration("my_enum").symbols(symbols).noDefault()
.endRecord();
- final File file = File.createTempFile("avro-primitive-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-primitive-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema enumSchema = schema.getField("b_enum").schema();
final AvroTestRecordWriter record = new AvroTestRecordWriter(schema, file);
@@ -313,10 +308,7 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateSimpleArraySchema_NoNullValues() throws Exception {
-
- final File file = File.createTempFile("avro-array-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-array-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -364,10 +356,7 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateSimpleNestedSchema_NoNullValues() throws Exception {
-
- final File file = File.createTempFile("avro-nested-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-nested-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -406,10 +395,7 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateUnionNestedArraySchema_withNullValues() throws Exception {
-
- final File file = File.createTempFile("avro-nested-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-nested-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -455,10 +441,7 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateNestedArraySchema(int numRecords, int numArrayItems) throws IOException {
-
- final File file = File.createTempFile("avro-nested-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-nested-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest").namespace("org.apache.drill.exec.store.avro")
.fields().name("a_int").type().intType().noDefault().name("b_array").type().array().items()
.record("my_record_1").namespace("foo.blah.org").fields().name("nested_1_int").type().optional().intType()
@@ -490,10 +473,7 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateMapSchema_withNullValues() throws Exception {
-
- final File file = File.createTempFile("avro-nested-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-nested-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -525,10 +505,7 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateMapSchemaComplex_withNullValues() throws Exception {
-
- final File file = File.createTempFile("avro-nested-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-nested-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -574,10 +551,7 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateUnionNestedSchema_withNullValues() throws Exception {
-
- final File file = File.createTempFile("avro-nested-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-nested-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -616,10 +590,7 @@ public class AvroTestUtil {
}
public static AvroTestRecordWriter generateDoubleNestedSchema_NoNullValues() throws Exception {
-
- final File file = File.createTempFile("avro-double-nested-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-double-nested-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("AvroRecordReaderTest")
.namespace("org.apache.drill.exec.store.avro")
.fields()
@@ -672,10 +643,7 @@ public class AvroTestUtil {
}
public static String generateLinkedList() throws Exception {
-
- final File file = File.createTempFile("avro-linkedlist", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-linkedlist", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final Schema schema = SchemaBuilder.record("LongList")
.namespace("org.apache.drill.exec.store.avro")
.aliases("LinkedLongs")
@@ -703,7 +671,7 @@ public class AvroTestUtil {
writer.close();
}
- return file.getAbsolutePath();
+ return file.getName();
}
public static AvroTestRecordWriter generateStringAndUtf8Data() throws Exception {
@@ -715,9 +683,7 @@ public class AvroTestUtil {
.name("b_utf8").type().stringType().noDefault()
.endRecord();
- final File file = File.createTempFile("avro-primitive-test", ".avro");
- file.deleteOnExit();
-
+ final File file = File.createTempFile("avro-primitive-test", ".avro", BaseTestQuery.dirTestWatcher.getRootDir());
final AvroTestRecordWriter record = new AvroTestRecordWriter(schema, file);
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java
index 1919184..1b2801d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.store.TestOutputMutator;
import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFileSelection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFileSelection.java
index d23cd1f..f74bc55 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFileSelection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFileSelection.java
@@ -20,11 +20,11 @@ package org.apache.drill.exec.store.dfs;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import java.nio.file.Paths;
import java.util.List;
import com.google.common.collect.ImmutableList;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.hadoop.fs.FileStatus;
import org.junit.Test;
@@ -46,15 +46,14 @@ public class TestFileSelection extends BaseTestQuery {
}
}
-
@Test(expected = Exception.class)
public void testEmptyFolderThrowsTableNotFound() throws Exception {
- final String table = String.format("%s/empty", TestTools.getTestResourcesPath());
- final String query = String.format("select * from dfs.`%s`", table);
+ final String emptyDirPath = dirTestWatcher.makeRootSubDir(Paths.get("empty")).getAbsolutePath();
+ final String query = String.format("select * from dfs.`%s`", emptyDirPath);
try {
testNoResult(query);
} catch (Exception ex) {
- final String pattern = String.format("%s' not found", table).toLowerCase();
+ final String pattern = String.format("%s' not found", emptyDirPath).toLowerCase();
final boolean isTableNotFound = ex.getMessage().toLowerCase().contains(pattern);
assertTrue(isTableNotFound);
throw ex;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
index 59cb22b..8dae43f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestGlob.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,54 +17,62 @@
*/
package org.apache.drill.exec.store.dfs;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
-import org.apache.drill.common.util.TestTools;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
@Category(UnlikelyTest.class)
public class TestGlob extends BaseTestQuery {
- String MULTILEVEL = TestTools.getWorkingPath() + "/../java-exec/src/test/resources/multilevel";
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel"));
+ }
- @Test
- public void testGlobSet() throws Exception {
- testBuilder()
- .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/{1994,1995}`", MULTILEVEL))
- .unOrdered()
- .baselineColumns("EXPR$0")
- .baselineValues(80L)
- .build().run();
- }
+ @Test
+ public void testGlobSet() throws Exception {
+ testBuilder()
+ .sqlQuery("select count(*) from dfs.`multilevel/parquet/{1994,1995}`")
+ .unOrdered().baselineColumns("EXPR$0")
+ .baselineValues(80L)
+ .build()
+ .run();
+ }
- @Test
- public void testGlobWildcard() throws Exception {
- testBuilder()
- .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/1994/*`", MULTILEVEL))
- .unOrdered()
- .baselineColumns("EXPR$0")
- .baselineValues(40L)
- .build().run();
- }
+ @Test
+ public void testGlobWildcard() throws Exception {
+ testBuilder()
+ .sqlQuery("select count(*) from dfs.`multilevel/parquet/1994/*`")
+ .unOrdered()
+ .baselineColumns("EXPR$0")
+ .baselineValues(40L)
+ .build()
+ .run();
+ }
- @Test
- public void testGlobSingleCharacter() throws Exception {
- testBuilder()
- .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/199?/*`", MULTILEVEL))
- .unOrdered()
- .baselineColumns("EXPR$0")
- .baselineValues(120L)
- .build().run();
- }
+ @Test
+ public void testGlobSingleCharacter() throws Exception {
+ testBuilder()
+ .sqlQuery("select count(*) from dfs.`multilevel/parquet/199?/*`")
+ .unOrdered()
+ .baselineColumns("EXPR$0")
+ .baselineValues(120L)
+ .build()
+ .run();
+ }
- @Test
- public void testGlobSingleCharacterRange() throws Exception {
- testBuilder()
- .sqlQuery(String.format("select count(*) from dfs_test.`%s/parquet/199[4-5]/*`", MULTILEVEL))
- .unOrdered()
- .baselineColumns("EXPR$0")
- .baselineValues(80L)
- .build().run();
- }
+ @Test
+ public void testGlobSingleCharacterRange() throws Exception {
+ testBuilder()
+ .sqlQuery("select count(*) from dfs.`multilevel/parquet/199[4-5]/*`")
+ .unOrdered()
+ .baselineColumns("EXPR$0")
+ .baselineValues(80L)
+ .build()
+ .run();
+ }
}
[22/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
DRILL-5783, DRILL-5841, DRILL-5894: Rationalize test temp directories
This change includes:
DRILL-5783:
- A unit test is created for the priority queue in the TopN operator.
- The code generation classes passed around a completely unused function registry reference in some places so it is removed.
- The priority queue had unused parameters for some of its methods so it is removed.
DRILL-5841:
- Created standardized temp directory classes DirTestWatcher, SubDirTestWatcher, and BaseDirTestWatcher. And updated all unit tests to use them.
DRILL-5894:
- Removed the dfs_test storage plugin for tests and replaced it with the already existing dfs storage plugin.
Misc:
- General code cleanup.
- Removed unnecessary use of String.format in the tests.
This closes #984
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/acc5ed92
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/acc5ed92
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/acc5ed92
Branch: refs/heads/master
Commit: acc5ed927e1fa4011ac1c3724d15197484b9f45b
Parents: c56de2f
Author: Timothy Farkas <ti...@apache.org>
Authored: Thu Nov 9 14:25:53 2017 -0800
Committer: Paul Rogers <pr...@maprtech.com>
Committed: Tue Nov 14 17:26:41 2017 -0800
----------------------------------------------------------------------
.../drill/common/util/DrillFileUtils.java | 46 +
.../org/apache/drill/common/util/FileUtils.java | 47 --
.../org/apache/drill/common/util/TestTools.java | 75 --
.../org/apache/drill/test/DirTestWatcher.java | 167 ++++
.../java/org/apache/drill/test/DrillTest.java | 19 +-
.../apache/drill/test/SubDirTestWatcher.java | 193 +++++
.../java/org/apache/drill/test/TestTools.java | 104 +++
.../apache/drill/testutils/DirTestWatcher.java | 61 --
.../drill/maprdb/tests/json/BaseJsonTest.java | 2 +-
.../expr/fn/impl/gis/TestGeometryFunctions.java | 10 +-
.../org/apache/drill/hbase/BaseHBaseTest.java | 6 +-
.../drill/hbase/HBaseRecordReaderTest.java | 2 +-
.../hbase/TestOrderedBytesConvertFunctions.java | 2 +-
.../apache/drill/exec/fn/hive/TestHiveUDFs.java | 2 +-
.../drill/exec/fn/hive/TestInbuiltHiveUDFs.java | 4 +-
.../apache/drill/exec/hive/HiveTestBase.java | 4 +-
.../apache/drill/exec/hive/TestHiveStorage.java | 1 -
.../exec/hive/TestInfoSchemaOnHiveStorage.java | 5 +-
.../hive/BaseTestHiveImpersonation.java | 14 +-
.../sql/hive/TestViewSupportOnHiveTables.java | 16 +-
.../exec/store/hive/HiveTestDataGenerator.java | 46 +-
.../apache/drill/store/kudu/TestKuduPlugin.java | 2 +-
.../store/mongo/TestMongoProjectPushDown.java | 4 +-
.../drill/exec/compile/ClassTransformer.java | 11 +-
.../apache/drill/exec/compile/CodeCompiler.java | 6 +-
.../apache/drill/exec/expr/CodeGenerator.java | 38 +-
.../drill/exec/expr/EvaluationVisitor.java | 9 +-
.../exec/expr/fn/FunctionGenerationHelper.java | 18 +-
.../drill/exec/expr/fn/FunctionInitializer.java | 5 +-
.../exec/physical/impl/TopN/PriorityQueue.java | 60 +-
.../impl/TopN/PriorityQueueTemplate.java | 18 +-
.../exec/physical/impl/TopN/TopNBatch.java | 78 +-
.../physical/impl/aggregate/HashAggBatch.java | 3 +-
.../impl/aggregate/StreamingAggBatch.java | 5 +-
.../physical/impl/common/ChainedHashTable.java | 2 +-
.../physical/impl/filter/FilterRecordBatch.java | 4 +-
.../impl/flatten/FlattenRecordBatch.java | 4 +-
.../exec/physical/impl/join/HashJoinBatch.java | 5 +-
.../exec/physical/impl/join/MergeJoinBatch.java | 4 +-
.../physical/impl/join/NestedLoopJoinBatch.java | 5 +-
.../impl/mergereceiver/MergingRecordBatch.java | 6 +-
.../OrderedPartitionRecordBatch.java | 9 +-
.../PartitionSenderRootExec.java | 4 +-
.../impl/project/ProjectRecordBatch.java | 13 +-
.../exec/physical/impl/sort/SortBatch.java | 6 +-
.../impl/svremover/RemovingRecordBatch.java | 19 +-
.../impl/union/UnionAllRecordBatch.java | 2 +-
.../impl/window/WindowFrameRecordBatch.java | 3 +-
.../physical/impl/xsort/ExternalSortBatch.java | 6 +-
.../impl/xsort/managed/MergeSortWrapper.java | 4 +-
.../managed/PriorityQueueCopierWrapper.java | 4 +-
.../impl/xsort/managed/SorterWrapper.java | 3 +-
.../apache/drill/exec/record/BatchSchema.java | 4 +-
.../exec/record/selection/SelectionVector4.java | 8 +
.../drill/exec/store/dfs/FileSystemPlugin.java | 1 +
.../exec/store/easy/json/JSONFormatPlugin.java | 2 +-
.../drill/exec/util/StoragePluginTestUtils.java | 125 +++
.../apache/drill/exec/util/TestUtilities.java | 148 ----
.../src/main/resources/drill-module.conf | 1 +
.../java/org/apache/drill/BaseTestQuery.java | 713 ----------------
.../java/org/apache/drill/DrillTestWrapper.java | 808 ------------------
.../java/org/apache/drill/PlanTestBase.java | 2 +
.../java/org/apache/drill/PlanningBase.java | 2 +-
.../java/org/apache/drill/QueryTestUtil.java | 221 -----
.../java/org/apache/drill/TestAggNullable.java | 23 +-
.../org/apache/drill/TestAltSortQueries.java | 22 +-
.../java/org/apache/drill/TestBugFixes.java | 27 +-
.../test/java/org/apache/drill/TestBuilder.java | 689 ---------------
.../java/org/apache/drill/TestCTASJson.java | 16 +-
.../apache/drill/TestCTASPartitionFilter.java | 45 +-
.../org/apache/drill/TestCaseSensitivity.java | 1 +
.../apache/drill/TestDisabledFunctionality.java | 56 +-
.../java/org/apache/drill/TestDropTable.java | 62 +-
.../org/apache/drill/TestDynamicUDFSupport.java | 196 +++--
.../org/apache/drill/TestExampleQueries.java | 195 ++---
.../org/apache/drill/TestFrameworkTest.java | 14 +-
.../org/apache/drill/TestFunctionsQuery.java | 1 +
.../drill/TestFunctionsWithTypeExpoQueries.java | 172 ++--
.../test/java/org/apache/drill/TestInList.java | 3 +-
.../java/org/apache/drill/TestJoinNullable.java | 157 ++--
.../org/apache/drill/TestMergeFilterPlan.java | 64 +-
.../apache/drill/TestOutOfMemoryOutcome.java | 3 +-
.../org/apache/drill/TestPartitionFilter.java | 258 +++---
.../org/apache/drill/TestProjectPushDown.java | 20 +-
.../java/org/apache/drill/TestSchemaChange.java | 15 +-
.../org/apache/drill/TestSelectWithOption.java | 13 +-
.../java/org/apache/drill/TestSelectivity.java | 1 +
.../java/org/apache/drill/TestStarQueries.java | 30 +-
.../java/org/apache/drill/TestTextJoin.java | 25 +-
.../org/apache/drill/TestTpchDistributed.java | 1 +
.../drill/TestTpchDistributedConcurrent.java | 4 +-
.../drill/TestTpchDistributedStreaming.java | 3 +-
.../java/org/apache/drill/TestTpchExplain.java | 3 +-
.../java/org/apache/drill/TestTpchLimit0.java | 3 +-
.../java/org/apache/drill/TestTpchPlanning.java | 2 +-
.../org/apache/drill/TestTpchSingleMode.java | 3 +-
.../java/org/apache/drill/TestUnionAll.java | 343 ++++----
.../org/apache/drill/TestUnionDistinct.java | 338 +++-----
.../drill/TestUtf8SupportInQueryString.java | 1 +
.../drill/exec/DrillSeparatePlanningTest.java | 44 +-
.../java/org/apache/drill/exec/ExecTest.java | 20 -
.../apache/drill/exec/TestEmptyInputSql.java | 52 +-
.../drill/exec/TestQueriesOnLargeFile.java | 32 +-
.../apache/drill/exec/TestRepeatedReaders.java | 41 +-
.../apache/drill/exec/TestWindowFunctions.java | 47 +-
.../drill/exec/cache/TestWriteToDisk.java | 2 +-
.../apache/drill/exec/client/DumpCatTest.java | 4 +-
.../exec/compile/TestClassTransformation.java | 4 +-
.../exec/compile/TestEvaluationVisitor.java | 11 +-
.../exec/compile/TestLargeFileCompilation.java | 17 +-
.../apache/drill/exec/expr/ExpressionTest.java | 8 +-
.../drill/exec/expr/TestLogicalExprSerDe.java | 2 +-
.../expr/TestSchemaPathMaterialization.java | 14 +-
.../exec/expr/fn/FunctionInitializerTest.java | 10 +-
.../exec/expr/fn/impl/TestStringFunctions.java | 21 +-
.../exec/fn/impl/TestAggregateFunction.java | 4 +-
.../exec/fn/impl/TestAggregateFunctions.java | 70 +-
.../exec/fn/impl/TestCastEmptyStrings.java | 143 ++--
.../drill/exec/fn/impl/TestCastFunctions.java | 76 +-
.../exec/fn/impl/TestContextFunctions.java | 10 +-
.../drill/exec/fn/impl/TestCryptoFunctions.java | 2 +-
.../exec/fn/impl/TestDateAddFunctions.java | 2 +-
.../drill/exec/fn/impl/TestDateFunctions.java | 7 +-
.../exec/fn/impl/TestDateTruncFunctions.java | 2 +-
.../drill/exec/fn/impl/TestMathFunctions.java | 4 +-
.../drill/exec/fn/impl/TestMultiInputAdd.java | 4 +-
.../exec/fn/impl/TestNetworkFunctions.java | 2 +-
.../exec/fn/impl/TestNewAggregateFunctions.java | 4 +-
.../exec/fn/impl/TestNewDateFunctions.java | 2 +-
.../fn/impl/TestNewSimpleRepeatedFunctions.java | 2 +-
.../exec/fn/impl/TestRepeatedFunction.java | 4 +-
.../drill/exec/fn/impl/TestTrigFunctions.java | 2 +-
.../fn/impl/testing/TestDateConversions.java | 117 +--
.../exec/fn/interp/TestConstantFolding.java | 38 +-
.../impersonation/BaseTestImpersonation.java | 11 +-
.../TestImpersonationDisabledWithMiniDFS.java | 16 +-
.../TestImpersonationMetadata.java | 16 +-
.../impersonation/TestImpersonationQueries.java | 24 +-
.../impersonation/TestInboundImpersonation.java | 8 +-
.../TestInboundImpersonationPrivileges.java | 4 +-
.../drill/exec/memory/TestAllocators.java | 4 +-
.../exec/nested/TestFastComplexSchema.java | 2 +-
.../exec/nested/TestNestedComplexSchema.java | 2 +-
.../drill/exec/opt/BasicOptimizerTest.java | 4 +-
.../physical/config/TestParsePhysicalPlan.java | 4 +-
.../physical/impl/TestBroadcastExchange.java | 10 +-
.../exec/physical/impl/TestCastFunctions.java | 24 +-
.../physical/impl/TestCastVarCharToBigInt.java | 4 +-
.../physical/impl/TestConvertFunctions.java | 58 +-
.../drill/exec/physical/impl/TestDecimal.java | 14 +-
.../impl/TestDistributedFragmentRun.java | 10 +-
.../physical/impl/TestExtractFunctions.java | 4 +-
.../physical/impl/TestHashToRandomExchange.java | 4 +-
.../exec/physical/impl/TestLocalExchange.java | 47 +-
.../exec/physical/impl/TestOptiqPlans.java | 6 +-
.../physical/impl/TestReverseImplicitCast.java | 4 +-
.../physical/impl/TestSimpleFragmentRun.java | 8 +-
.../exec/physical/impl/TestSimpleFunctions.java | 8 +-
.../exec/physical/impl/TestStringFunctions.java | 1 -
.../exec/physical/impl/TestUnionExchange.java | 4 +-
.../exec/physical/impl/TopN/TestSimpleTopN.java | 102 ---
.../impl/TopN/TestTopNSchemaChanges.java | 77 +-
.../exec/physical/impl/TopN/TopNBatchTest.java | 173 ++++
.../drill/exec/physical/impl/agg/TestAgg.java | 4 +-
.../exec/physical/impl/agg/TestHashAggr.java | 2 +-
.../physical/impl/agg/TestHashAggrSpill.java | 153 ++--
.../impl/broadcastsender/TestBroadcast.java | 25 +-
.../physical/impl/filter/TestLargeInClause.java | 2 +-
.../physical/impl/filter/TestSimpleFilter.java | 6 +-
.../exec/physical/impl/flatten/TestFlatten.java | 221 +++--
.../impl/flatten/TestFlattenPlanning.java | 6 +-
.../exec/physical/impl/join/TestHashJoin.java | 24 +-
.../impl/join/TestHashJoinAdvanced.java | 45 +-
.../exec/physical/impl/join/TestMergeJoin.java | 32 +-
.../impl/join/TestMergeJoinAdvanced.java | 42 +-
.../impl/join/TestMergeJoinMulCondition.java | 10 +-
.../join/TestMergeJoinWithSchemaChanges.java | 123 +--
.../physical/impl/join/TestNestedLoopJoin.java | 17 +-
.../impl/limit/TestEarlyLimit0Optimization.java | 36 +-
.../impl/limit/TestLimitWithExchanges.java | 45 +-
.../physical/impl/limit/TestSimpleLimit.java | 6 +-
.../impl/mergereceiver/TestMergingReceiver.java | 12 +-
.../TestOrderedPartitionExchange.java | 4 +-
.../partitionsender/TestPartitionSender.java | 31 +-
.../impl/project/TestSimpleProjection.java | 4 +-
.../exec/physical/impl/sort/TestSimpleSort.java | 6 +-
.../drill/exec/physical/impl/sort/TestSort.java | 2 +-
.../physical/impl/svremover/TestSVRemover.java | 2 +-
.../impl/trace/TestTraceMultiRecordBatch.java | 4 +-
.../impl/trace/TestTraceOutputDump.java | 4 +-
.../physical/impl/union/TestSimpleUnion.java | 4 +-
.../impl/validate/TestValidationOptions.java | 21 +-
.../physical/impl/window/GenerateTestData.java | 24 +-
.../physical/impl/window/TestWindowFrame.java | 85 +-
.../TestCorruptParquetDateCorrection.java | 85 +-
.../physical/impl/writer/TestParquetWriter.java | 146 ++--
.../writer/TestParquetWriterEmptyFiles.java | 50 +-
.../exec/physical/impl/writer/TestWriter.java | 110 ++-
.../physical/impl/xsort/TestExternalSort.java | 171 ++--
.../impl/xsort/TestSimpleExternalSort.java | 29 +-
.../impl/xsort/TestSortSpillWithException.java | 17 +-
.../physical/unit/BasicPhysicalOpUnitTest.java | 2 +-
.../physical/unit/MiniPlanUnitTestBase.java | 8 +-
.../physical/unit/PhysicalOpUnitTestBase.java | 61 +-
.../drill/exec/physical/unit/TestMiniPlan.java | 4 +-
.../physical/unit/TestNullInputMiniPlan.java | 6 +-
.../exec/planner/TestDirectoryExplorerUDFs.java | 93 +--
.../planner/logical/TestCaseNullableTypes.java | 2 +-
.../logical/TestConvertCountToDirectScan.java | 12 +-
.../exec/planner/sql/TestDrillSQLWorker.java | 2 +-
.../apache/drill/exec/pop/PopUnitTestBase.java | 6 +-
.../drill/exec/pop/TestInjectionValue.java | 4 +-
.../drill/exec/record/TestRecordIterator.java | 6 +-
.../drill/exec/record/vector/TestDateTypes.java | 14 +-
.../exec/rpc/control/TestCustomTunnel.java | 2 +-
.../drill/exec/rpc/data/TestBitBitKerberos.java | 4 +-
.../drill/exec/rpc/security/KerberosHelper.java | 6 +-
.../user/TemporaryTablesAutomaticDropTest.java | 67 +-
.../security/TestCustomUserAuthenticator.java | 2 +-
.../rpc/user/security/TestUserBitKerberos.java | 4 +-
.../security/TestUserBitKerberosEncryption.java | 4 +-
.../exec/rpc/user/security/TestUserBitSSL.java | 5 +-
.../rpc/user/security/TestUserBitSSLServer.java | 20 +-
.../security/TestUserBitSaslCompatibility.java | 3 +-
.../exec/server/TestDrillbitResilience.java | 4 +-
.../apache/drill/exec/server/TestOptions.java | 2 +-
.../exec/server/TestOptionsAuthEnabled.java | 8 +-
.../drill/exec/server/TestTpcdsSf1Leaks.java | 4 +-
.../options/PersistedOptionValueTest.java | 11 +-
.../exec/server/options/TestConfigLinkage.java | 332 ++++++++
.../exec/server/rest/StatusResourcesTest.java | 14 +-
.../drill/exec/sql/TestBaseViewSupport.java | 16 +-
.../org/apache/drill/exec/sql/TestCTAS.java | 207 ++---
.../org/apache/drill/exec/sql/TestCTTAS.java | 82 +-
.../apache/drill/exec/sql/TestInfoSchema.java | 66 +-
.../drill/exec/sql/TestSimpleCastFunctions.java | 2 +-
.../apache/drill/exec/sql/TestViewSupport.java | 173 ++--
.../apache/drill/exec/sql/TestWithClause.java | 2 +-
.../exec/store/TestImplicitFileColumns.java | 94 +--
.../drill/exec/store/TestTimedRunnable.java | 2 +-
.../drill/exec/store/avro/AvroFormatTest.java | 257 +++---
.../drill/exec/store/avro/AvroTestUtil.java | 82 +-
.../exec/store/bson/TestBsonRecordReader.java | 2 +-
.../drill/exec/store/dfs/TestFileSelection.java | 11 +-
.../apache/drill/exec/store/dfs/TestGlob.java | 92 +-
.../store/dfs/TestSchemaNotFoundException.java | 12 +-
.../exec/store/easy/text/compliant/TestCsv.java | 18 +-
.../Drill2283InfoSchemaVarchar1BugTest.java | 2 +-
.../exec/store/json/TestJsonRecordReader.java | 24 +-
.../store/parquet/ParquetInternalsTest.java | 37 +-
.../store/parquet/ParquetRecordReaderTest.java | 100 ++-
.../exec/store/parquet/TestFixedlenDecimal.java | 15 +-
.../exec/store/parquet/TestParquetComplex.java | 2 +-
.../parquet/TestParquetFilterPushDown.java | 227 ++---
.../store/parquet/TestParquetGroupScan.java | 44 +-
.../store/parquet/TestParquetMetadataCache.java | 393 +++++----
.../exec/store/parquet/TestParquetScan.java | 40 +-
.../columnreaders/TestColumnReaderFactory.java | 2 +-
.../parquet/columnreaders/TestDateReader.java | 2 +-
.../store/parquet2/TestDrillParquetReader.java | 6 +-
.../drill/exec/store/pcap/TestPcapDecoder.java | 3 +-
.../exec/store/pcap/TestPcapRecordReader.java | 22 +-
.../sequencefile/TestSequenceFileReader.java | 9 +-
.../drill/exec/store/sys/PStoreTestUtil.java | 30 +-
.../exec/store/sys/TestPStoreProviders.java | 27 +-
.../drill/exec/store/sys/TestSystemTable.java | 2 +-
.../drill/exec/store/text/TestCsvHeader.java | 68 +-
.../exec/store/text/TestNewTextReader.java | 31 +-
.../drill/exec/store/text/TestTextColumn.java | 11 +-
.../exec/store/text/TextRecordReaderTest.java | 6 +-
.../testing/TestCountDownLatchInjection.java | 2 +-
.../exec/testing/TestExceptionInjection.java | 2 +-
.../drill/exec/testing/TestPauseInjection.java | 2 +-
.../drill/exec/testing/TestResourceLeak.java | 2 +-
.../vector/complex/TestEmptyPopulation.java | 6 +-
.../fn/TestJsonReaderWithSparseFiles.java | 2 +-
.../complex/writer/TestComplexToJson.java | 8 +-
.../complex/writer/TestComplexTypeReader.java | 59 +-
.../complex/writer/TestComplexTypeWriter.java | 2 +-
.../complex/writer/TestExtendedTypes.java | 46 +-
.../vector/complex/writer/TestJsonReader.java | 241 +++---
.../exec/work/batch/TestSpoolingBuffer.java | 6 +-
.../fragment/TestFragmentExecutorCancel.java | 2 +-
.../work/metadata/TestMetadataProvider.java | 19 +-
.../work/metadata/TestServerMetaProvider.java | 2 +-
.../work/prepare/PreparedStatementTestBase.java | 2 +-
.../apache/drill/test/BaseDirTestWatcher.java | 278 +++++++
.../org/apache/drill/test/BaseTestQuery.java | 613 ++++++++++++++
.../org/apache/drill/test/ClientFixture.java | 4 +-
.../org/apache/drill/test/ClusterFixture.java | 215 ++---
.../drill/test/ClusterFixtureBuilder.java | 61 +-
.../java/org/apache/drill/test/ClusterTest.java | 2 -
.../org/apache/drill/test/DrillTestWrapper.java | 832 +++++++++++++++++++
.../java/org/apache/drill/test/ExampleTest.java | 98 ++-
.../java/org/apache/drill/test/HadoopUtils.java | 44 +
.../org/apache/drill/test/QueryBuilder.java | 1 -
.../org/apache/drill/test/QueryTestUtil.java | 220 +++++
.../java/org/apache/drill/test/TestBuilder.java | 695 ++++++++++++++++
.../apache/drill/test/TestConfigLinkage.java | 319 -------
.../org/apache/drill/test/package-info.java | 117 ++-
.../org/apache/drill/test/rowSet/RowSet.java | 16 +-
.../apache/drill/test/rowSet/RowSetBuilder.java | 2 +-
.../drill/test/rowSet/RowSetComparison.java | 37 +
.../drill/test/rowSet/file/JsonFileBuilder.java | 159 ++++
.../resources/bootstrap-storage-plugins.json | 75 --
.../resources/parquet/parquet_nullable.json | 6 +-
.../parquet/parquet_nullable_varlen.json | 6 +-
.../resources/parquet/parquet_scan_screen.json | 6 +-
.../src/test/resources/queries/tpch/15.sql | 2 +-
.../src/test/resources/store/text/test.json | 2 +-
.../src/test/resources/topN/one_key_sort.json | 4 +-
.../src/test/resources/tpcds-sf1/q73.sql | 10 +-
.../src/test/resources/window/3604.sql | 2 +-
.../src/test/resources/window/3605.sql | 2 +-
.../src/test/resources/window/3606.sql | 2 +-
.../src/test/resources/window/3648.sql | 2 +-
.../src/test/resources/window/3668.sql | 2 +-
.../src/test/resources/window/fval.alltypes.sql | 2 +-
.../src/test/resources/window/fval.pby.sql | 2 +-
.../src/test/resources/window/lag.oby.sql | 2 +-
.../src/test/resources/window/lag.pby.oby.sql | 2 +-
.../src/test/resources/window/lead.oby.sql | 2 +-
.../src/test/resources/window/lead.pby.oby.sql | 2 +-
.../src/test/resources/window/lval.alltypes.sql | 2 +-
.../src/test/resources/window/lval.pby.oby.sql | 2 +-
.../src/test/resources/window/ntile.sql | 2 +-
exec/java-exec/src/test/resources/window/q1.sql | 2 +-
exec/java-exec/src/test/resources/window/q2.sql | 2 +-
exec/java-exec/src/test/resources/window/q3.sql | 2 +-
.../src/test/resources/window/q3218.sql | 2 +-
.../src/test/resources/window/q3220.sql | 2 +-
exec/java-exec/src/test/resources/window/q4.sql | 2 +-
.../resources/writer/simple_csv_writer.json | 6 +-
.../org/apache/drill/jdbc/ITTestShadedJar.java | 134 +--
.../drill/jdbc/impl/DrillConnectionImpl.java | 42 +-
.../jdbc/DatabaseMetaDataGetColumnsTest.java | 79 +-
.../apache/drill/jdbc/DrillResultSetTest.java | 13 +-
.../org/apache/drill/jdbc/JdbcTestBase.java | 334 +++++++-
.../LegacyDatabaseMetaDataGetColumnsTest.java | 6 +-
.../drill/jdbc/ResultSetMetaDataTest.java | 11 +-
.../jdbc/test/Bug1735ConnectionCloseTest.java | 8 +-
...ug1735ResultSetCloseReleasesBuffersTest.java | 5 +-
...etColumnsDataTypeNotTypeCodeIntBugsTest.java | 2 +-
...ill2288GetColumnsMetadataWhenNoRowsTest.java | 8 +-
...39GetBooleanFailsSayingWrongTypeBugTest.java | 2 +-
...rill2461IntervalsBreakInfoSchemaBugTest.java | 4 +-
...2463GetNullsFailedWithAssertionsBugTest.java | 2 +-
...l2489CallsAfterCloseThrowExceptionsTest.java | 8 +-
...69UnsupportedReportsUseSqlExceptionTest.java | 7 +-
.../org/apache/drill/jdbc/test/JdbcAssert.java | 361 --------
.../test/JdbcConnectTriesTestEmbeddedBits.java | 20 +-
.../apache/drill/jdbc/test/JdbcDataTest.java | 133 +--
.../test/JdbcNullOrderingAndGroupingTest.java | 266 ++----
.../drill/jdbc/test/JdbcTestActionBase.java | 62 +-
.../drill/jdbc/test/JdbcTestQueryBase.java | 8 +-
.../jdbc/test/TestAggregateFunctionsQuery.java | 34 +-
.../jdbc/test/TestInformationSchemaColumns.java | 76 +-
.../drill/jdbc/test/TestJdbcDistQuery.java | 111 ++-
.../drill/jdbc/test/TestJdbcMetadata.java | 7 +-
.../apache/drill/jdbc/test/TestJdbcQuery.java | 84 +-
.../drill/jdbc/test/TestLegacyJdbcMetadata.java | 2 +-
.../drill/exec/memory/ChildAllocator.java | 11 +-
.../drill/exec/vector/BaseValueVector.java | 2 -
.../apache/drill/exec/vector/ValueVector.java | 1 +
.../exec/vector/complex/RepeatedMapVector.java | 4 -
.../drill/storage/CheckStorageConfig.java | 4 +-
366 files changed, 9004 insertions(+), 9170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/common/src/main/java/org/apache/drill/common/util/DrillFileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/DrillFileUtils.java b/common/src/main/java/org/apache/drill/common/util/DrillFileUtils.java
new file mode 100644
index 0000000..f20fef7
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/util/DrillFileUtils.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.util;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URL;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+
+public class DrillFileUtils {
+ // These two seperators are intentionally used instead of Path.SEPERATPR or File.seperator
+ // for loading classes since both alternatives return '\' on Windows. However, in the context
+ // of classpath scanning and loading only '/' is valid.
+ public static final char SEPARATOR_CHAR = '/';
+ public static final String SEPARATOR = "/";
+
+ public static File getResourceAsFile(String fileName) throws IOException {
+ URL u = DrillFileUtils.class.getResource(fileName);
+ if (u == null) {
+ throw new FileNotFoundException(String.format("Unable to find file on path %s", fileName));
+ }
+ return new File(u.getPath());
+ }
+
+ public static String getResourceAsString(String fileName) throws IOException {
+ return Files.toString(getResourceAsFile(fileName), Charsets.UTF_8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/common/src/main/java/org/apache/drill/common/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/FileUtils.java b/common/src/main/java/org/apache/drill/common/util/FileUtils.java
deleted file mode 100644
index cd846f8..0000000
--- a/common/src/main/java/org/apache/drill/common/util/FileUtils.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.util;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URL;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-
-public class FileUtils {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileUtils.class);
-
- public static final char separatorChar = '/';
-
- public static final String separator = "" + separatorChar;
-
- public static File getResourceAsFile(String fileName) throws IOException {
- URL u = FileUtils.class.getResource(fileName);
- if (u == null) {
- throw new FileNotFoundException(String.format("Unable to find file on path %s", fileName));
- }
- return new File(u.getPath());
- }
-
- public static String getResourceAsString(String fileName) throws IOException {
- return Files.toString(getResourceAsFile(fileName), Charsets.UTF_8);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/common/src/main/java/org/apache/drill/common/util/TestTools.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/TestTools.java b/common/src/main/java/org/apache/drill/common/util/TestTools.java
deleted file mode 100644
index 30d073e..0000000
--- a/common/src/main/java/org/apache/drill/common/util/TestTools.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.util;
-
-import java.nio.file.Paths;
-
-import org.junit.rules.TestName;
-import org.junit.rules.TestRule;
-import org.junit.rules.Timeout;
-
-public class TestTools {
- // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTools.class);
-
- static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments()
- .toString().indexOf("-agentlib:jdwp") > 0;
- static final String WORKING_PATH = Paths.get("").toAbsolutePath().toString();
-
- public static TestRule getTimeoutRule() {
- return getTimeoutRule(10000);
- }
-
- public static TestRule getTimeoutRule(int timeout) {
- return IS_DEBUG ? new TestName() : new Timeout(timeout);
- }
-
- /**
- * If not enforced, the repeat rule applies only if the test is run in non-debug mode.
- */
- public static TestRule getRepeatRule(final boolean enforce) {
- return enforce || !IS_DEBUG ? new RepeatTestRule() : new TestName();
- }
-
- public static String getWorkingPath() {
- return WORKING_PATH;
- }
-
- private static final String PATH_SEPARATOR = System.getProperty("file.separator");
- private static final String[] STRUCTURE = {"drill", "exec", "java-exec", "src", "test", "resources"};
-
- /**
- * Returns fully qualified path where test resources reside if current working directory is at any level in the
- * following root->exec->java-exec->src->test->resources, throws an {@link IllegalStateException} otherwise.
- */
- public static String getTestResourcesPath() {
- final StringBuilder builder = new StringBuilder(WORKING_PATH);
- for (int i=0; i< STRUCTURE.length; i++) {
- if (WORKING_PATH.endsWith(STRUCTURE[i])) {
- for (int j=i+1; j< STRUCTURE.length; j++) {
- builder.append(PATH_SEPARATOR).append(STRUCTURE[j]);
- }
- return builder.toString();
- }
- }
- final String msg = String.format("Unable to recognize working directory[%s]. The workspace must be root or exec " +
- "module.", WORKING_PATH);
- throw new IllegalStateException(msg);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/test/DirTestWatcher.java b/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
new file mode 100644
index 0000000..c741b80
--- /dev/null
+++ b/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.test;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * <p>
+ * This class is used to create consistently named and safe temp directories for unit tests.
+ * </p>
+ *
+ * <p>
+ * A {@link DirTestWatcher} is added to a test by declaring it as a JUnit {@link org.junit.Rule}. A {@link org.junit.Rule} is
+ * a piece of code that is run before and after every JUnit test marked with the {@link org.junit.Test} annotation. When the
+ * {@link DirTestWatcher} is added to a test class the {@link DirTestWatcher} will create a temp directory before each of your
+ * {@link org.junit.Test}s and optionally delete the temp directory after each of your {@link org.junit.Test}s. The temp directory
+ * created by the {@link DirTestWatcher} is in the <b>target</b> folder of the maven project and has the form
+ * <b>(my test class fully qualified name)/(my test method name)</b>. So in the context of the code example below, the temp directory created for
+ * each test in <b>target</b> will be <b>my.proj.MyTestClass/myTestMethod1</b> and <b>my.proj.MyTestClass/myTestMethod2</b> respectively.
+ * </p>
+ *
+ * <p>
+ * The temp directory created by the {@link DirTestWatcher} can be used within a test by simply calling the {@link DirTestWatcher#getDir()}
+ * method on the {@link DirTestWatcher} within your unit test.
+ * </p>
+ *
+ * <p>
+ * By default, the {@link DirTestWatcher} deletes the temp directory it creates at the end of each {@link org.junit.Test}. However, you can create a {@link DirTestWatcher}
+ * by doing {@code new DirTestWatcher(false)} to disable the deletion of temp directories after a test. This is useful if you want to examine files after a test runs.
+ * </p>
+ *
+ * <pre>
+ * package my.proj;
+ *
+ * public class MyTestClass {
+ * @org.junit.Rule
+ * public final DirTestWatcher dirTestWatcher = new DirTestWatcher();
+ *
+ * @org.junit.Test
+ * public void myTestMethod1() {
+ * File dir = dirTestWatcher.getDir();
+ * // Do stuff in the temp directory
+ * }
+ *
+ * @org.junit.Test
+ * public void myTestMethod2() {
+ * File dir = dirTestWatcher.getDir();
+ * // Do stuff in the temp directory
+ * }
+ * }
+ * </pre>
+ *
+ * <p>
+ * <b>Note:</b> In the code sample above, the directories returned by {@link DirTestWatcher#getDir()} in myTestMethod1 and myTestMethod2 are
+ * <b>my.proj.MyTestClass/myTestMethod1</b> and <b>my.proj.MyTestClass/myTestMethod2</b> respectively.
+ * </p>
+ */
+public class DirTestWatcher extends TestWatcher {
+ private static final int TEMP_DIR_ATTEMPTS = 10000;
+
+ private String dirPath;
+ private File dir;
+ private boolean deleteDirAtEnd = true;
+
+ /**
+ * Creates a {@link DirTestWatcher} that deletes temp directories after the {@link TestWatcher} completes.
+ */
+ public DirTestWatcher() {
+ }
+
+ /**
+ * Creates a {@link DirTestWatcher} which can delete or keep the temp directory after the {@link TestWatcher} completes.
+ * @param deleteDirAtEnd When true the temp directory created by the {@link DirTestWatcher} is deleted. When false the
+ * temp directory created by the {@link DirTestWatcher} is not deleted.
+ */
+ public DirTestWatcher(boolean deleteDirAtEnd) {
+ this.deleteDirAtEnd = deleteDirAtEnd;
+ }
+
+ @Override
+ protected void starting(Description description) {
+ if (description.getMethodName() != null) {
+ dirPath = Paths.get(".","target", description.getClassName(), description.getMethodName()).toString();
+ } else {
+ dirPath = Paths.get(".","target", description.getClassName()).toString();
+ }
+
+ dir = new File(dirPath);
+ FileUtils.deleteQuietly(dir);
+ dir.mkdirs();
+ }
+
+ @Override
+ protected void finished(Description description) {
+ deleteDir();
+ }
+
+ @Override
+ protected void failed(Throwable e, Description description) {
+ deleteDir();
+ }
+
+ /**
+ * Creates a sub directory with the given relative path in current temp directory
+ * @param relativeDirPath The relative path of the sub directory to create in the current temp directory.
+ * @return The {@link java.io.File} object of the newly created sub directory.
+ */
+ public File makeSubDir(Path relativeDirPath) {
+ File subDir = dir
+ .toPath()
+ .resolve(relativeDirPath)
+ .toFile();
+ subDir.mkdirs();
+ return subDir;
+ }
+
+ private void deleteDir() {
+ if (deleteDirAtEnd) {
+ FileUtils.deleteQuietly(dir);
+ }
+ }
+
+ /**
+ * Gets the {@link java.io.File} object of the current temp directory.
+ * @return The {@link java.io.File} object of the current temp directory.
+ */
+ public File getDir() {
+ return dir;
+ }
+
+ public static File createTempDir(File baseDir) {
+ String baseName = System.currentTimeMillis() + "-";
+
+ for (int counter = 0; counter < TEMP_DIR_ATTEMPTS; counter++) {
+ File tempDir = new File(baseDir, baseName + counter);
+ if (tempDir.mkdirs()) {
+ return tempDir;
+ }
+ }
+
+ String message = String.format("Failed to create directory within %s attempts (tried %s0 to %s)",
+ TEMP_DIR_ATTEMPTS, baseName, baseName + (TEMP_DIR_ATTEMPTS - 1));
+ throw new IllegalStateException(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/common/src/test/java/org/apache/drill/test/DrillTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/test/DrillTest.java b/common/src/test/java/org/apache/drill/test/DrillTest.java
index 64df178..de26470 100644
--- a/common/src/test/java/org/apache/drill/test/DrillTest.java
+++ b/common/src/test/java/org/apache/drill/test/DrillTest.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.commons.io.output.NullOutputStream;
import org.apache.drill.common.util.DrillStringUtils;
-import org.apache.drill.common.util.TestTools;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -41,8 +40,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public class DrillTest {
-// private static final Logger logger = org.slf4j.LoggerFactory.getLogger(DrillTest.class);
-
protected static final ObjectMapper objectMapper;
static {
System.setProperty("line.separator", "\n");
@@ -104,7 +101,8 @@ public class DrillTest {
@AfterClass
public static void finiDrillTest() throws InterruptedException{
testReporter.info(String.format("Test Class done (%s): %s.", memWatcher.getMemString(true), className));
- LOG_OUTCOME.sleepIfFailure();
+ // Clear interrupts for next test
+ Thread.interrupted();
}
protected static class MemWatcher {
@@ -138,9 +136,7 @@ public class DrillTest {
}
private static class TestLogReporter extends TestWatcher {
-
private MemWatcher memWatcher;
- private int failureCount = 0;
@Override
protected void starting(Description description) {
@@ -152,23 +148,12 @@ public class DrillTest {
@Override
protected void failed(Throwable e, Description description) {
testReporter.error(String.format("Test Failed (%s): %s", memWatcher.getMemString(), description.getDisplayName()), e);
- failureCount++;
}
@Override
public void succeeded(Description description) {
testReporter.info(String.format("Test Succeeded (%s): %s", memWatcher.getMemString(), description.getDisplayName()));
}
-
- public void sleepIfFailure() throws InterruptedException {
- if(failureCount > 0){
- Thread.sleep(2000);
- failureCount = 0;
- } else {
- // pause to get logger to catch up.
- Thread.sleep(250);
- }
- }
}
public static String escapeJsonString(String original) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/common/src/test/java/org/apache/drill/test/SubDirTestWatcher.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/test/SubDirTestWatcher.java b/common/src/test/java/org/apache/drill/test/SubDirTestWatcher.java
new file mode 100644
index 0000000..bffb605
--- /dev/null
+++ b/common/src/test/java/org/apache/drill/test/SubDirTestWatcher.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.test;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.List;
+
+/**
+ * This is a {@link org.junit.rules.TestWatcher} which is used to create and delete sub directories before and after unit tests respectively.
+ * Ideally a {@link SubDirTestWatcher} would be used in conjunction with a {@link DirTestWatcher} or a sub class of {@link DirTestWatcher}. The
+ * {@link DirTestWatcher} would be used to create the base directory used by the {@link SubDirTestWatcher} to create all its sub directories. Here
+ * is an example of using the {@link SubDirTestWatcher} in this way.
+ *
+ * <pre>
+ * package my.proj;
+ *
+ * public class MyTestClass {
+ * public static final String TEST_SUB_DIR = "testSubDir";
+ * private static File testSubDir;
+ *
+ * @org.junit.ClassRule
+ * public static final DirTestWatcher dirTestWatcher = new DirTestWatcher();
+ *
+ * @org.junit.BeforeClass
+ * public static void setupFiles() {
+ * testSubDir = new File(dirTestWatcher.getDir(), TEST_SUB_DIR);
+ * }
+ *
+ * @org.junit.Rule
+ * public final SubDirTestWatcher subDirTestWatcher =
+ * new SubDirTestWatcher.Builder(dirTestWatcher.getDir())
+ * .addSubDir(TEST_SUB_DIR)
+ * .build();
+ *
+ * @org.junit.Test
+ * public void myTestMethod1() {
+ * // The testSubDir should be created for us by the SubDirTestWatcher
+ * org.junit.Assert.assertTrue(testSubDir.exists());
+ *
+ * // Lets make another sub directory in our sub directory
+ * File newSubDirectory = new File(testSubDir, "a");
+ * newSubDirectory.mkdirs();
+ * org.junit.Assert.assertTrue(newSubDirectory.exists());
+ * }
+ *
+ * @org.junit.Test
+ * public void myTestMethod2() {
+ * // The testSubDir should be created for us by the SubDirTestWatcher
+ * org.junit.Assert.assertTrue(testSubDir.exists());
+ *
+ * // The directory we created in the previous test should be gone
+ * org.junit.Assert.assertFalse(new File(testSubDir, "a").exists());
+ * }
+ * }
+ * </pre>
+ */
+public class SubDirTestWatcher extends TestWatcher {
+ private File baseDir;
+ private boolean createAtBeginning;
+ private boolean deleteAtEnd;
+ private List<Path> subDirs;
+
+ protected SubDirTestWatcher(File baseDir, boolean createAtBeginning, boolean deleteAtEnd, List<Path> subDirs) {
+ this.baseDir = Preconditions.checkNotNull(baseDir);
+ this.createAtBeginning = createAtBeginning;
+ this.deleteAtEnd = deleteAtEnd;
+ this.subDirs = Preconditions.checkNotNull(subDirs);
+
+ Preconditions.checkArgument(!subDirs.isEmpty(), "The list of subDirs is empty.");
+ }
+
+ @Override
+ protected void starting(Description description) {
+ deleteDirs(true);
+
+ if (!createAtBeginning) {
+ return;
+ }
+
+ for (Path subDir: subDirs) {
+ baseDir.toPath()
+ .resolve(subDir)
+ .toFile()
+ .mkdirs();
+ }
+ }
+
+ @Override
+ protected void finished(Description description) {
+ deleteDirs(deleteAtEnd);
+ }
+
+ @Override
+ protected void failed(Throwable e, Description description) {
+ deleteDirs(deleteAtEnd);
+ }
+
+ private void deleteDirs(boolean delete) {
+ if (!delete) {
+ return;
+ }
+
+ for (Path subDir: subDirs) {
+ FileUtils.deleteQuietly(baseDir.toPath().resolve(subDir).toFile());
+ }
+ }
+
+ /**
+ * This is a builder for a {@link SubDirTestWatcher}. Create an instance of the builder, configure it, and then call the {@link Builder#build()} to create a
+ * {@link SubDirTestWatcher}.
+ */
+ public static class Builder {
+ private File baseDir;
+ private boolean createAtBeginning = true;
+ private boolean deleteAtEnd = true;
+ private List<Path> subDirs = Lists.newArrayList();
+
+ /**
+ * Initializes a builder for a {@link SubDirTestWatcher} with the given baseDir.
+ * @param baseDir The baseDir is the parent directory in which all sub directories are created.
+ */
+ public Builder(File baseDir) {
+ this.baseDir = Preconditions.checkNotNull(baseDir);
+ }
+
+ /**
+ * Sets the createAtBeginning flag.
+ * @param createAtBeginning This flag determines whether the {@link SubDirTestWatcher} creates the configured sub directories at a beginning of a test. Setting
+ * This to true will cause the {@link SubDirTestWatcher} to create the configured sub directories at the beginning of each test. Setting
+ * This to false will prevent the {@link SubDirTestWatcher} from attempting to create the configured sub directories at the beginning of
+ * at test. Setting this to false is useful for scenarios where you only want to delete directories created in the middle of your tests. By
+ * default this flag is true.
+ * @return The {@link SubDirTestWatcher.Builder}.
+ */
+ public Builder setCreateAtBeginning(boolean createAtBeginning) {
+ this.createAtBeginning = createAtBeginning;
+ return this;
+ }
+
+ /**
+ * Sets the deleteAtEnd flag.
+ * @param deleteAtEnd This flag determines whether the {@link SubDirTestWatcher} deletes the sub directories it manages at the end of a test.
+ * @return The {@link SubDirTestWatcher.Builder}.
+ */
+ public Builder setDeleteAtEnd(boolean deleteAtEnd) {
+ this.deleteAtEnd = deleteAtEnd;
+ return this;
+ }
+
+ /**
+ * Adds a sub directory to be managed by the {@link SubDirTestWatcher}.
+ * @param subDir The relative path of the sub directory to be added.
+ * @return The {@link SubDirTestWatcher.Builder}.
+ */
+ public Builder addSubDir(Path subDir) {
+ Preconditions.checkNotNull(subDir);
+ subDirs.add(subDir);
+ return this;
+ }
+
+ /**
+ * Builds a {@link SubDirTestWatcher}.
+ * @return A configured {@link SubDirTestWatcher}.
+ */
+ public SubDirTestWatcher build() {
+ Preconditions.checkState(!subDirs.isEmpty(), "The list of subDirs is empty.");
+ return new SubDirTestWatcher(baseDir, createAtBeginning, deleteAtEnd, subDirs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/common/src/test/java/org/apache/drill/test/TestTools.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/test/TestTools.java b/common/src/test/java/org/apache/drill/test/TestTools.java
new file mode 100644
index 0000000..661b978
--- /dev/null
+++ b/common/src/test/java/org/apache/drill/test/TestTools.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.drill.common.util.RepeatTestRule;
+import org.junit.rules.TestName;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+
+public class TestTools {
+ public enum FileSource {
+ RESOURCE,
+ PROJECT
+ }
+
+ public static final Path TEST_RESOURCES = Paths.get("src", "test", "resources");
+ public static final Path PROJECT_ROOT = Paths.get("..", "..");
+ public static final Path WORKING_PATH = new File(".").toPath();
+
+ static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean()
+ .getInputArguments()
+ .toString()
+ .indexOf("-agentlib:jdwp") > 0;
+
+ public static TestRule getTimeoutRule(int timeout) {
+ return IS_DEBUG ? new TestName() : new Timeout(timeout);
+ }
+
+ /**
+ * If not enforced, the repeat rule applies only if the test is run in non-debug mode.
+ */
+ public static TestRule getRepeatRule(final boolean enforce) {
+ return enforce || !IS_DEBUG ? new RepeatTestRule() : new TestName();
+ }
+
+ public static File getResourceFile(Path relPath) {
+ return WORKING_PATH
+ .resolve(TEST_RESOURCES)
+ .resolve(relPath)
+ .toFile();
+ }
+
+ public static File getProjectFile(Path relPath) {
+ return WORKING_PATH
+ .resolve(PROJECT_ROOT)
+ .resolve(relPath)
+ .toFile();
+ }
+
+ public static File getFile(Path relPath, FileSource fileSource) {
+ switch (fileSource) {
+ case RESOURCE:
+ return getResourceFile(relPath);
+ case PROJECT:
+ return getProjectFile(relPath);
+ default: {
+ throw new IllegalArgumentException(String.format("Unkown data type %s", fileSource));
+ }
+ }
+ }
+
+ public static void copyDirToDest(Path relPath, File destDir, FileSource fileSource) {
+ File srcDir;
+
+ switch (fileSource) {
+ case RESOURCE:
+ srcDir = getResourceFile(relPath);
+ break;
+ case PROJECT:
+ srcDir = getProjectFile(relPath);
+ break;
+ default: {
+ throw new IllegalArgumentException(String.format("Unkown data type %s", fileSource));
+ }
+ }
+
+ try {
+ FileUtils.copyDirectory(srcDir, destDir);
+ } catch (IOException e) {
+ throw new RuntimeException("This should never happen", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/common/src/test/java/org/apache/drill/testutils/DirTestWatcher.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/testutils/DirTestWatcher.java b/common/src/test/java/org/apache/drill/testutils/DirTestWatcher.java
deleted file mode 100644
index a5a8806..0000000
--- a/common/src/test/java/org/apache/drill/testutils/DirTestWatcher.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.testutils;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-
-import java.io.File;
-
-/**
- * This JUnit {@link TestWatcher} creates a unique directory for each JUnit test in the project's
- * target folder at the start of each test. This directory can be used as a temporary directory to store
- * files for the test. The directory and its contents are deleted at the end of the test.
- */
-public class DirTestWatcher extends TestWatcher {
- private String dirPath;
- private File dir;
-
- @Override
- protected void starting(Description description) {
- dirPath = String.format("%s/target/%s/%s", new File(".").getAbsolutePath(),
- description.getClassName(), description.getMethodName());
- dir = new File(dirPath);
- dir.mkdirs();
- }
-
- @Override
- protected void finished(Description description) {
- FileUtils.deleteQuietly(dir);
- }
-
- @Override
- protected void failed(Throwable e, Description description) {
- FileUtils.deleteQuietly(dir);
- }
-
- public String getDirPath() {
- return dirPath;
- }
-
- public File getDir() {
- return dir;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java
index 2760ffe..27dd7ce 100644
--- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java
@@ -19,7 +19,7 @@ package com.mapr.drill.maprdb.tests.json;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/gis/src/test/java/org/apache/drill/exec/expr/fn/impl/gis/TestGeometryFunctions.java
----------------------------------------------------------------------
diff --git a/contrib/gis/src/test/java/org/apache/drill/exec/expr/fn/impl/gis/TestGeometryFunctions.java b/contrib/gis/src/test/java/org/apache/drill/exec/expr/fn/impl/gis/TestGeometryFunctions.java
index 059fb36..63c9e07 100644
--- a/contrib/gis/src/test/java/org/apache/drill/exec/expr/fn/impl/gis/TestGeometryFunctions.java
+++ b/contrib/gis/src/test/java/org/apache/drill/exec/expr/fn/impl/gis/TestGeometryFunctions.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.expr.fn.impl.gis;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
public class TestGeometryFunctions extends BaseTestQuery {
@@ -29,7 +29,7 @@ public class TestGeometryFunctions extends BaseTestQuery {
testBuilder()
.sqlQuery("select ST_AsText(ST_GeomFromText('" + wktPoint + "')) "
- + "from cp.`/sample-data/CA-cities.csv` limit 1")
+ + "from cp.`sample-data/CA-cities.csv` limit 1")
.ordered().baselineColumns("EXPR$0")
.baselineValues(wktPoint)
.build()
@@ -41,7 +41,7 @@ public class TestGeometryFunctions extends BaseTestQuery {
testBuilder()
.sqlQuery("select ST_AsText(ST_Point(-121.895, 37.339)) "
- + "from cp.`/sample-data/CA-cities.csv` limit 1")
+ + "from cp.`sample-data/CA-cities.csv` limit 1")
.ordered().baselineColumns("EXPR$0")
.baselineValues(wktPoint)
.build()
@@ -55,7 +55,7 @@ public class TestGeometryFunctions extends BaseTestQuery {
.sqlQuery("select ST_Within(ST_Point(columns[4], columns[3]),"
+ "ST_GeomFromText('POLYGON((-121.95 37.28, -121.94 37.35, -121.84 37.35, -121.84 37.28, -121.95 37.28))')"
+ ") "
- + "from cp.`/sample-data/CA-cities.csv` where columns[2] = 'San Jose'")
+ + "from cp.`sample-data/CA-cities.csv` where columns[2] = 'San Jose'")
.ordered().baselineColumns("EXPR$0")
.baselineValues(true)
.build()
@@ -66,7 +66,7 @@ public class TestGeometryFunctions extends BaseTestQuery {
.sqlQuery("select ST_Within(" + "ST_Point(columns[4], columns[3]),"
+ "ST_GeomFromText('POLYGON((-121.95 37.28, -121.94 37.35, -121.84 37.35, -121.84 37.28, -121.95 37.28))')"
+ ") "
- + "from cp.`/sample-data/CA-cities.csv` where columns[2] = 'San Francisco'")
+ + "from cp.`sample-data/CA-cities.csv` where columns[2] = 'San Francisco'")
.ordered().baselineColumns("EXPR$0")
.baselineValues(false)
.build()
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
index b957347..9d4b4cd 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
@@ -20,8 +20,8 @@ package org.apache.drill.hbase;
import java.io.IOException;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.store.StoragePluginRegistry;
@@ -71,7 +71,7 @@ public class BaseHBaseTest extends BaseTestQuery {
}
protected String getPlanText(String planFile, String tableName) throws IOException {
- return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8)
+ return Files.toString(DrillFileUtils.getResourceAsFile(planFile), Charsets.UTF_8)
.replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\" : " + HBaseTestsSuite.getZookeeperPort())
.replace("[TABLE_NAME]", tableName);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
index 5d94d24..f1bcf3c 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
@@ -22,7 +22,7 @@ import org.apache.drill.categories.SlowTest;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import static org.apache.drill.TestBuilder.mapOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
@Category({SlowTest.class, HbaseStorageTest.class})
public class HBaseRecordReaderTest extends BaseHBaseTest {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestOrderedBytesConvertFunctions.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestOrderedBytesConvertFunctions.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestOrderedBytesConvertFunctions.java
index 6d1e302..2ef6d68 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestOrderedBytesConvertFunctions.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestOrderedBytesConvertFunctions.java
@@ -25,7 +25,7 @@ import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.HbaseStorageTest;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
index 584e339..3872563 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestHiveUDFs.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.HiveStorageTest;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.exec.record.RecordBatchLoader;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
index 8242f3f..5654ff0 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
@@ -20,8 +20,8 @@ package org.apache.drill.exec.fn.hive;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.drill.categories.HiveStorageTest;
-import org.apache.drill.QueryTestUtil;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.QueryTestUtil;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestBase.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestBase.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestBase.java
index c33de5d..ddc808e 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestBase.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestBase.java
@@ -30,8 +30,8 @@ public class HiveTestBase extends PlanTestBase {
protected static HiveTestDataGenerator hiveTest;
@BeforeClass
- public static void generateHive() throws Exception{
- hiveTest = HiveTestDataGenerator.getInstance();
+ public static void generateHive() throws Exception {
+ hiveTest = HiveTestDataGenerator.getInstance(dirTestWatcher.getRootDir());
hiveTest.addHiveTestPlugin(getDrillbitContext().getStorage());
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index cf98797..33b8ec0 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -56,7 +56,6 @@ public class TestHiveStorage extends HiveTestBase {
test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
}
-
@Test // DRILL-4083
public void testNativeScanWhenNoColumnIsRead() throws Exception {
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
index 228fa24..8e5c77b 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.hive;
import com.google.common.base.Strings;
import org.apache.drill.categories.HiveStorageTest;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.categories.SlowTest;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.junit.Test;
@@ -86,9 +86,6 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
.baselineValues("dfs.root")
.baselineValues("dfs.tmp")
.baselineValues("sys")
- .baselineValues("dfs_test.home")
- .baselineValues("dfs_test.default")
- .baselineValues("dfs_test.tmp")
.baselineValues("cp.default")
.baselineValues("INFORMATION_SCHEMA")
.go();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java
index 66c595f..3862dc6 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.impersonation.hive;
import org.apache.calcite.schema.Schema.TableType;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.dotdrill.DotDrillType;
import org.apache.drill.exec.impersonation.BaseTestImpersonation;
@@ -31,9 +31,11 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
+import java.io.File;
import java.util.List;
import java.util.Map;
+import static org.apache.drill.exec.store.hive.HiveTestDataGenerator.createFileWithPermissions;
import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS;
@@ -60,12 +62,16 @@ public class BaseTestHiveImpersonation extends BaseTestImpersonation {
protected static void prepHiveConfAndData() throws Exception {
hiveConf = new HiveConf();
+ File scratchDir = createFileWithPermissions(dirTestWatcher.getRootDir(), "scratch_dir");
+ File localScratchDir = createFileWithPermissions(dirTestWatcher.getRootDir(), "local_scratch_dir");
+ File metaStoreDBDir = new File(dirTestWatcher.getRootDir(), "metastore_db");
+
// Configure metastore persistence db location on local filesystem
- final String dbUrl = String.format("jdbc:derby:;databaseName=%s;create=true", getTempDir("metastore_db"));
+ final String dbUrl = String.format("jdbc:derby:;databaseName=%s;create=true", metaStoreDBDir.getAbsolutePath());
hiveConf.set(ConfVars.METASTORECONNECTURLKEY.varname, dbUrl);
- hiveConf.set(ConfVars.SCRATCHDIR.varname, "file:///" + getTempDir("scratch_dir"));
- hiveConf.set(ConfVars.LOCALSCRATCHDIR.varname, getTempDir("local_scratch_dir"));
+ hiveConf.set(ConfVars.SCRATCHDIR.varname, "file://" + scratchDir.getAbsolutePath());
+ hiveConf.set(ConfVars.LOCALSCRATCHDIR.varname, localScratchDir.getAbsolutePath());
// Set MiniDFS conf in HiveConf
hiveConf.set(FS_DEFAULT_NAME_KEY, dfsConf.get(FS_DEFAULT_NAME_KEY));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/sql/hive/TestViewSupportOnHiveTables.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/sql/hive/TestViewSupportOnHiveTables.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/sql/hive/TestViewSupportOnHiveTables.java
index 2076c3a..3e70731 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/sql/hive/TestViewSupportOnHiveTables.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/sql/hive/TestViewSupportOnHiveTables.java
@@ -27,22 +27,22 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
+
@Category({SlowTest.class, HiveStorageTest.class})
public class TestViewSupportOnHiveTables extends TestBaseViewSupport {
- private static final String TEMP_SCHEMA = "dfs_test.tmp";
-
protected static HiveTestDataGenerator hiveTest;
@BeforeClass
public static void generateHive() throws Exception{
- hiveTest = HiveTestDataGenerator.getInstance();
+ hiveTest = HiveTestDataGenerator.getInstance(dirTestWatcher.getRootDir());
hiveTest.addHiveTestPlugin(getDrillbitContext().getStorage());
}
@Test
public void viewWithStarInDef_StarInQuery() throws Exception{
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT * FROM hive.kv",
"SELECT * FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 1",
@@ -54,7 +54,7 @@ public class TestViewSupportOnHiveTables extends TestBaseViewSupport {
@Test
public void viewWithStarInDef_SelectFieldsInQuery1() throws Exception{
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT * FROM hive.kv",
"SELECT key, `value` FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 1",
@@ -66,7 +66,7 @@ public class TestViewSupportOnHiveTables extends TestBaseViewSupport {
@Test
public void viewWithStarInDef_SelectFieldsInQuery2() throws Exception{
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT * FROM hive.kv",
"SELECT `value` FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 1",
@@ -78,7 +78,7 @@ public class TestViewSupportOnHiveTables extends TestBaseViewSupport {
@Test
public void viewWithSelectFieldsInDef_StarInQuery() throws Exception{
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT key, `value` FROM hive.kv",
"SELECT * FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 1",
@@ -90,7 +90,7 @@ public class TestViewSupportOnHiveTables extends TestBaseViewSupport {
@Test
public void viewWithSelectFieldsInDef_SelectFieldsInQuery() throws Exception{
testViewHelper(
- TEMP_SCHEMA,
+ DFS_TMP_SCHEMA,
null,
"SELECT key, `value` FROM hive.kv",
"SELECT key, `value` FROM TEST_SCHEMA.TEST_VIEW_NAME LIMIT 1",
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 580cf78..4fafadb 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -19,15 +19,20 @@
package org.apache.drill.exec.store.hive;
import java.io.File;
+import java.io.IOException;
import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.attribute.PosixFilePermission;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.Map;
+import java.util.Set;
+import com.google.common.collect.Sets;
import com.google.common.io.Resources;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.exceptions.DrillException;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.hadoop.fs.FileSystem;
@@ -39,21 +44,26 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import com.google.common.collect.Maps;
import org.apache.hadoop.hive.serde.serdeConstants;
-import static org.apache.drill.BaseTestQuery.getTempDir;
import static org.apache.drill.exec.hive.HiveTestUtilities.executeQuery;
public class HiveTestDataGenerator {
private static final String HIVE_TEST_PLUGIN_NAME = "hive";
private static HiveTestDataGenerator instance;
+ private static File baseDir;
private final String dbDir;
private final String whDir;
private final Map<String, String> config;
- public static synchronized HiveTestDataGenerator getInstance() throws Exception {
- if (instance == null) {
- final String dbDir = getTempDir("metastore_db");
- final String whDir = getTempDir("warehouse");
+ public static synchronized HiveTestDataGenerator getInstance(File baseDir) throws Exception {
+ if (instance == null || !HiveTestDataGenerator.baseDir.equals(baseDir)) {
+ HiveTestDataGenerator.baseDir = baseDir;
+
+ File dbDirFile = new File(baseDir, "metastore_db");
+ File whDirFile = new File(baseDir, "warehouse");
+
+ final String dbDir = dbDirFile.getAbsolutePath();
+ final String whDir = whDirFile.getAbsolutePath();
instance = new HiveTestDataGenerator(dbDir, whDir);
instance.generateTestData();
@@ -111,15 +121,33 @@ public class HiveTestDataGenerator {
pluginRegistry.deletePlugin(HIVE_TEST_PLUGIN_NAME);
}
+ public static File createFileWithPermissions(File baseDir, String name) {
+ Set<PosixFilePermission> perms = Sets.newHashSet(PosixFilePermission.values());
+ File dir = new File(baseDir, name);
+ dir.mkdirs();
+
+ try {
+ Files.setPosixFilePermissions(dir.toPath(), perms);
+ } catch (IOException e) {
+ new RuntimeException(e);
+ }
+
+ return dir;
+ }
+
private void generateTestData() throws Exception {
HiveConf conf = new HiveConf(SessionState.class);
+ File scratchDir = createFileWithPermissions(baseDir, "scratch_dir");
+ File localScratchDir = createFileWithPermissions(baseDir, "local_scratch_dir");
+ File part1Dir = createFileWithPermissions(baseDir, "part1");
+
conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true", dbDir));
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
conf.set("hive.metastore.warehouse.dir", whDir);
conf.set("mapred.job.tracker", "local");
- conf.set(ConfVars.SCRATCHDIR.varname, getTempDir("scratch_dir"));
- conf.set(ConfVars.LOCALSCRATCHDIR.varname, getTempDir("local_scratch_dir"));
+ conf.set(ConfVars.SCRATCHDIR.varname, scratchDir.getAbsolutePath());
+ conf.set(ConfVars.LOCALSCRATCHDIR.varname, localScratchDir.getAbsolutePath());
conf.set(ConfVars.DYNAMICPARTITIONINGMODE.varname, "nonstrict");
SessionState ss = new SessionState(conf);
@@ -450,7 +478,7 @@ public class HiveTestDataGenerator {
// Add a partition with custom location
executeQuery(hiveDriver,
String.format("ALTER TABLE partition_pruning_test ADD PARTITION (c=99, d=98, e=97) LOCATION '%s'",
- getTempDir("part1")));
+ part1Dir.getAbsolutePath()));
executeQuery(hiveDriver,
String.format("INSERT INTO TABLE partition_pruning_test PARTITION(c=99, d=98, e=97) " +
"SELECT '%s', '%s' FROM kv LIMIT 1",
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduPlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduPlugin.java b/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduPlugin.java
index c9265d5..b94033a 100644
--- a/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduPlugin.java
+++ b/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduPlugin.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.store.kudu;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.KuduStorageTest;
import org.junit.Ignore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoProjectPushDown.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoProjectPushDown.java
index 848a6a7..4d228c8 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoProjectPushDown.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoProjectPushDown.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.store.mongo;
-import static org.apache.drill.TestBuilder.listOf;
-import static org.apache.drill.TestBuilder.mapOf;
+import static org.apache.drill.test.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
import org.apache.drill.categories.MongoStorageTest;
import org.apache.drill.categories.SlowTest;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
index 3f01a5a..d8da9d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassTransformer.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.compile;
+import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.Map;
@@ -24,8 +25,8 @@ import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.common.util.DrillStringUtils;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.compile.MergeAdapter.MergedClassResult;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.expr.CodeGenerator;
@@ -171,8 +172,8 @@ public class ClassTransformer {
public ClassNames(String className) {
dot = className;
- slash = className.replace('.', FileUtils.separatorChar);
- clazz = FileUtils.separatorChar + slash + ".class";
+ slash = className.replace('.', DrillFileUtils.SEPARATOR_CHAR);
+ clazz = DrillFileUtils.SEPARATOR_CHAR + slash + ".class";
}
@Override
@@ -310,7 +311,7 @@ public class ClassTransformer {
}
for (String s : result.innerClasses) {
- s = s.replace(FileUtils.separatorChar, '.');
+ s = s.replace(DrillFileUtils.SEPARATOR_CHAR, '.');
names.add(nextSet.getChild(s));
}
classLoader.injectByteCode(nextGenerated.dot, result.bytes);
@@ -319,7 +320,7 @@ public class ClassTransformer {
// adds byte code of the classes that have not been merged to make them accessible for outer class
for (Map.Entry<String, Pair<byte[], ClassNode>> clazz : classesToMerge.entrySet()) {
- classLoader.injectByteCode(clazz.getKey().replace(FileUtils.separatorChar, '.'), clazz.getValue().getKey());
+ classLoader.injectByteCode(clazz.getKey().replace(DrillFileUtils.SEPARATOR_CHAR, '.'), clazz.getValue().getKey());
}
Class<?> c = classLoader.findClass(set.generated.dot);
if (templateDefinition.getExternalInterface().isAssignableFrom(c)) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
index 247fda1..0db00f9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
@@ -22,7 +22,6 @@ import java.util.List;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.expr.CodeGenerator;
-import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionSet;
import com.google.common.annotations.VisibleForTesting;
@@ -110,6 +109,11 @@ public class CodeCompiler {
public static final String DISABLE_CACHE_CONFIG = COMPILE_BASE + ".disable_cache";
/**
+ * Enables saving generated code for debugging
+ */
+ public static final String ENABLE_SAVE_CODE_FOR_DEBUG_TOPN = COMPILE_BASE + ".codegen.debug.topn";
+
+ /**
* Prefer to generate code as plain Java when the code generator
* supports that mechanism.
*/
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
index 899bc4b..66f73b9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.drill.exec.compile.ClassBuilder;
import org.apache.drill.exec.compile.TemplateClassDefinition;
import org.apache.drill.exec.compile.sig.MappingSet;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.server.options.OptionSet;
import com.google.common.base.Preconditions;
@@ -90,12 +89,11 @@ public class CodeGenerator<T> {
private String generatedCode;
private String generifiedCode;
- CodeGenerator(TemplateClassDefinition<T> definition, FunctionImplementationRegistry funcRegistry, OptionSet optionManager) {
- this(ClassGenerator.getDefaultMapping(), definition, funcRegistry, optionManager);
+ CodeGenerator(TemplateClassDefinition<T> definition, OptionSet optionManager) {
+ this(ClassGenerator.getDefaultMapping(), definition, optionManager);
}
- CodeGenerator(MappingSet mappingSet, TemplateClassDefinition<T> definition,
- FunctionImplementationRegistry funcRegistry, OptionSet optionManager) {
+ CodeGenerator(MappingSet mappingSet, TemplateClassDefinition<T> definition, OptionSet optionManager) {
Preconditions.checkNotNull(definition.getSignature(),
"The signature for defintion %s was incorrectly initialized.", definition);
this.definition = definition;
@@ -104,8 +102,9 @@ public class CodeGenerator<T> {
try {
this.model = new JCodeModel();
JDefinedClass clazz = model._package(PACKAGE_NAME)._class(className);
- rootGenerator = new ClassGenerator<>(this, mappingSet, definition.getSignature(), new EvaluationVisitor(
- funcRegistry), clazz, model, optionManager);
+ rootGenerator = new ClassGenerator<>(this, mappingSet,
+ definition.getSignature(), new EvaluationVisitor(),
+ clazz, model, optionManager);
} catch (JClassAlreadyExistsException e) {
throw new IllegalStateException(e);
}
@@ -218,29 +217,24 @@ public class CodeGenerator<T> {
public String getClassName() { return className; }
- public static <T> CodeGenerator<T> get(TemplateClassDefinition<T> definition,
- FunctionImplementationRegistry funcRegistry) {
- return get(definition, funcRegistry, null);
+ public static <T> CodeGenerator<T> get(TemplateClassDefinition<T> definition) {
+ return get(definition, null);
}
- public static <T> CodeGenerator<T> get(TemplateClassDefinition<T> definition,
- FunctionImplementationRegistry funcRegistry, OptionSet optionManager) {
- return new CodeGenerator<T>(definition, funcRegistry, optionManager);
+ public static <T> CodeGenerator<T> get(TemplateClassDefinition<T> definition, OptionSet optionManager) {
+ return new CodeGenerator<T>(definition, optionManager);
}
- public static <T> ClassGenerator<T> getRoot(TemplateClassDefinition<T> definition,
- FunctionImplementationRegistry funcRegistry, OptionSet optionManager) {
- return get(definition, funcRegistry, optionManager).getRoot();
+ public static <T> ClassGenerator<T> getRoot(TemplateClassDefinition<T> definition, OptionSet optionManager) {
+ return get(definition, optionManager).getRoot();
}
- public static <T> ClassGenerator<T> getRoot(MappingSet mappingSet, TemplateClassDefinition<T> definition,
- FunctionImplementationRegistry funcRegistry, OptionSet optionManager) {
- return get(mappingSet, definition, funcRegistry, optionManager).getRoot();
+ public static <T> ClassGenerator<T> getRoot(MappingSet mappingSet, TemplateClassDefinition<T> definition, OptionSet optionManager) {
+ return get(mappingSet, definition, optionManager).getRoot();
}
- public static <T> CodeGenerator<T> get(MappingSet mappingSet, TemplateClassDefinition<T> definition,
- FunctionImplementationRegistry funcRegistry, OptionSet optionManager) {
- return new CodeGenerator<T>(mappingSet, definition, funcRegistry, optionManager);
+ public static <T> CodeGenerator<T> get(MappingSet mappingSet, TemplateClassDefinition<T> definition, OptionSet optionManager) {
+ return new CodeGenerator<T>(mappingSet, definition, optionManager);
}
@Override
[16/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java
index 38ab7fb..b76d308 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionDistinct.java
@@ -25,22 +25,26 @@ import org.apache.drill.categories.SqlTest;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.work.foreman.SqlUnsupportedException;
import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException;
-import org.junit.Ignore;
+import org.apache.drill.test.BaseTestQuery;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
import java.util.List;
@Category({SqlTest.class, OperatorTest.class})
public class TestUnionDistinct extends BaseTestQuery {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUnionDistinct.class);
-
private static final String sliceTargetSmall = "alter session set `planner.slice_target` = 1";
private static final String sliceTargetDefault = "alter session reset `planner.slice_target`";
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel"));
+ }
+
@Test // Simple Union over two scans
public void testUnionDistinct1() throws Exception {
String query = "(select n_regionkey from cp.`tpch/nation.parquet`) union (select r_regionkey from cp.`tpch/region.parquet`)";
@@ -183,13 +187,13 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test // Union-Distinct of * column from JSON files in different directories
public void testUnionDistinct9() throws Exception {
- String file0 = FileUtils.getResourceAsFile("/multilevel/json/1994/Q1/orders_94_q1.json").toURI().toString();
- String file1 = FileUtils.getResourceAsFile("/multilevel/json/1995/Q1/orders_95_q1.json").toURI().toString();
- String query = String.format("select o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, o_orderkey from dfs_test.`%s` union \n" +
- "select o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, o_orderkey from dfs_test.`%s`", file0, file1);
+ String file0 = "/multilevel/json/1994/Q1/orders_94_q1.json";
+ String file1 = "/multilevel/json/1995/Q1/orders_95_q1.json";
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, o_orderkey " +
+ "from cp.`%s` union select o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, o_orderkey from cp.`%s`",
+ file0, file1)
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q9.tsv")
.baselineTypes(TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.VARCHAR, TypeProtos.MinorType.FLOAT8, TypeProtos.MinorType.VARCHAR,
@@ -202,12 +206,9 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test // Union-Distinct constant literals
public void testUnionDistinct10() throws Exception {
- String query = "(select n_name, 'LEFT' as LiteralConstant, n_nationkey, '1' as NumberConstant from cp.`tpch/nation.parquet`) \n" +
- "union \n" +
- "(select 'RIGHT', r_name, '2', r_regionkey from cp.`tpch/region.parquet`)";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("(select n_name, 'LEFT' as LiteralConstant, n_nationkey, '1' as NumberConstant from cp.`tpch/nation.parquet`) " +
+ "union (select 'RIGHT', r_name, '2', r_regionkey from cp.`tpch/region.parquet`)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q10.tsv")
.baselineTypes(TypeProtos.MinorType.VARCHAR, TypeProtos.MinorType.VARCHAR, TypeProtos.MinorType.INT, TypeProtos.MinorType.INT)
@@ -218,17 +219,17 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctViewExpandableStar() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testunion as select n_name, n_nationkey from cp.`tpch/nation.parquet`;");
test("create view region_view_testunion as select r_name, r_regionkey from cp.`tpch/region.parquet`;");
- String query1 = "(select * from dfs_test.tmp.`nation_view_testunion`) \n" +
+ String query1 = "(select * from dfs.tmp.`nation_view_testunion`) \n" +
"union \n" +
- "(select * from dfs_test.tmp.`region_view_testunion`)";
+ "(select * from dfs.tmp.`region_view_testunion`)";
String query2 = "(select r_name, r_regionkey from cp.`tpch/region.parquet`) \n" +
"union \n" +
- "(select * from dfs_test.tmp.`nation_view_testunion`)";
+ "(select * from dfs.tmp.`nation_view_testunion`)";
try {
testBuilder()
@@ -256,11 +257,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test(expected = UnsupportedRelOperatorException.class)
public void testUnionDistinctViewUnExpandableStar() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testunion as select * from cp.`tpch/nation.parquet`;");
try {
- String query = "(select * from dfs_test.tmp.`nation_view_testunion`) \n" +
+ String query = "(select * from dfs.tmp.`nation_view_testunion`) \n" +
"union (select * from cp.`tpch/region.parquet`)";
test(query);
} catch(UserException ex) {
@@ -273,7 +274,7 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testDiffDataTypesAndModes() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testunion as select n_name, n_nationkey from cp.`tpch/nation.parquet`;");
test("create view region_view_testunion as select r_name, r_regionkey from cp.`tpch/region.parquet`;");
@@ -301,14 +302,10 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testDistinctOverUnionDistinctwithFullyQualifiedColumnNames() throws Exception {
- String query = "select distinct sq.x1, sq.x2 \n" +
- "from \n" +
- "((select n_regionkey as a1, n_name as b1 from cp.`tpch/nation.parquet`) \n" +
- "union \n" +
- "(select r_regionkey as a2, r_name as b2 from cp.`tpch/region.parquet`)) as sq(x1,x2)";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select distinct sq.x1, sq.x2 " +
+ "from ((select n_regionkey as a1, n_name as b1 from cp.`tpch/nation.parquet`) " +
+ "union (select r_regionkey as a2, r_name as b2 from cp.`tpch/region.parquet`)) as sq(x1,x2)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q14.tsv")
.baselineTypes(TypeProtos.MinorType.INT, TypeProtos.MinorType.VARCHAR)
@@ -319,12 +316,9 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctContainsColumnANumericConstant() throws Exception {
- String query = "(select n_nationkey, n_regionkey, n_name from cp.`tpch/nation.parquet` limit 5) \n" +
- "union \n" +
- "(select 1, n_regionkey, 'abc' from cp.`tpch/nation.parquet` limit 5)";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("(select n_nationkey, n_regionkey, n_name from cp.`tpch/nation.parquet` limit 5) " +
+ "union (select 1, n_regionkey, 'abc' from cp.`tpch/nation.parquet` limit 5)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q15.tsv")
.baselineTypes(TypeProtos.MinorType.INT, TypeProtos.MinorType.INT, TypeProtos.MinorType.VARCHAR)
@@ -334,16 +328,9 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctEmptySides() throws Exception {
- String query1 = "(select n_nationkey, n_regionkey, n_name from cp.`tpch/nation.parquet` limit 0) \n" +
- "union \n" +
- "(select 1, n_regionkey, 'abc' from cp.`tpch/nation.parquet` limit 5)";
-
- String query2 = "(select n_nationkey, n_regionkey, n_name from cp.`tpch/nation.parquet` limit 5) \n" +
- "union \n" +
- "(select 1, n_regionkey, 'abc' from cp.`tpch/nation.parquet` limit 0)";
-
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("(select n_nationkey, n_regionkey, n_name from cp.`tpch/nation.parquet` limit 0) " +
+ "union (select 1, n_regionkey, 'abc' from cp.`tpch/nation.parquet` limit 5)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q16.tsv")
.baselineTypes(TypeProtos.MinorType.INT, TypeProtos.MinorType.INT, TypeProtos.MinorType.VARCHAR)
@@ -352,7 +339,8 @@ public class TestUnionDistinct extends BaseTestQuery {
.run();
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("(select n_nationkey, n_regionkey, n_name from cp.`tpch/nation.parquet` limit 5) " +
+ "union (select 1, n_regionkey, 'abc' from cp.`tpch/nation.parquet` limit 0)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q17.tsv")
.baselineTypes(TypeProtos.MinorType.INT, TypeProtos.MinorType.INT, TypeProtos.MinorType.VARCHAR)
@@ -363,23 +351,13 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testAggregationOnUnionDistinctOperator() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
- String query1 = String.format(
- "(select calc1, max(b1) as `max`, min(b1) as `min`, count(c1) as `count` \n" +
- "from (select a1 + 10 as calc1, b1, c1 from dfs_test.`%s` \n" +
- "union \n" +
- "select a1 + 100 as diff1, b1 as diff2, c1 as diff3 from dfs_test.`%s`) \n" +
- "group by calc1 order by calc1)", root, root);
-
- String query2 = String.format(
- "(select calc1, min(b1) as `min`, max(b1) as `max`, count(c1) as `count` \n" +
- "from (select a1 + 10 as calc1, b1, c1 from dfs_test.`%s` \n" +
- "union \n" +
- "select a1 + 100 as diff1, b1 as diff2, c1 as diff3 from dfs_test.`%s`) \n" +
- "group by calc1 order by calc1)", root, root);
+ String root = "/store/text/data/t.json";
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("(select calc1, max(b1) as `max`, min(b1) as `min`, count(c1) as `count` " +
+ "from (select a1 + 10 as calc1, b1, c1 from cp.`%s` " +
+ "union select a1 + 100 as diff1, b1 as diff2, c1 as diff3 from cp.`%s`) " +
+ "group by calc1 order by calc1)", root, root)
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/testAggregationOnUnionDistinctOperator_1.tsv")
.baselineTypes(TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT)
@@ -388,7 +366,10 @@ public class TestUnionDistinct extends BaseTestQuery {
.run();
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("(select calc1, min(b1) as `min`, max(b1) as `max`, count(c1) as `count` " +
+ "from (select a1 + 10 as calc1, b1, c1 from cp.`%s` " +
+ "union select a1 + 100 as diff1, b1 as diff2, c1 as diff3 from cp.`%s`) " +
+ "group by calc1 order by calc1)", root, root)
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/testAggregationOnUnionDistinctOperator_2.tsv")
.baselineTypes(TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT)
@@ -399,39 +380,18 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test(expected = UserException.class)
public void testUnionDistinctImplicitCastingFailure() throws Exception {
- String rootInt = FileUtils.getResourceAsFile("/store/json/intData.json").toURI().toString();
- String rootBoolean = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- String query = String.format(
- "(select key from dfs_test.`%s` \n" +
- "union \n" +
- "select key from dfs_test.`%s` )", rootInt, rootBoolean);
-
- test(query);
+ test("(select key from cp.`store/json/intData.json` " +
+ "union select key from cp.`store/json/booleanData.json` )");
}
@Test
public void testDateAndTimestampJson() throws Exception {
- String rootDate = FileUtils.getResourceAsFile("/store/json/dateData.json").toURI().toString();
- String rootTimpStmp = FileUtils.getResourceAsFile("/store/json/timeStmpData.json").toURI().toString();
-
- String query1 = String.format(
- "(select max(key) as key from dfs_test.`%s` \n" +
- "union \n" +
- "select key from dfs_test.`%s`)", rootDate, rootTimpStmp);
-
- String query2 = String.format(
- "select key from dfs_test.`%s` \n" +
- "union \n" +
- "select max(key) as key from dfs_test.`%s`", rootDate, rootTimpStmp);
-
- String query3 = String.format(
- "select key from dfs_test.`%s` \n" +
- "union \n" +
- "select max(key) as key from dfs_test.`%s`", rootDate, rootTimpStmp);
+ String rootDate = "/store/json/dateData.json";
+ String rootTimpStmp = "/store/json/timeStmpData.json";
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("(select max(key) as key from cp.`%s` " +
+ "union select key from cp.`%s`)", rootDate, rootTimpStmp)
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q18_1.tsv")
.baselineTypes(TypeProtos.MinorType.VARCHAR)
@@ -440,7 +400,8 @@ public class TestUnionDistinct extends BaseTestQuery {
.run();
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select key from cp.`%s` " +
+ "union select max(key) as key from cp.`%s`", rootDate, rootTimpStmp)
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q18_2.tsv")
.baselineTypes(TypeProtos.MinorType.VARCHAR)
@@ -449,7 +410,8 @@ public class TestUnionDistinct extends BaseTestQuery {
.run();
testBuilder()
- .sqlQuery(query3)
+ .sqlQuery("select key from cp.`%s` " +
+ "union select max(key) as key from cp.`%s`", rootDate, rootTimpStmp)
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/q18_3.tsv")
.baselineTypes(TypeProtos.MinorType.VARCHAR)
@@ -460,21 +422,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctOneInputContainsAggFunction() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/csv/1994/Q1/orders_94_q1.csv").toURI().toString();
- String query1 = String.format("select * from ((select count(c1) as ct from (select columns[0] c1 from dfs.`%s`)) \n" +
- "union \n" +
- "(select columns[0] c2 from dfs.`%s`)) order by ct limit 3", root, root);
-
- String query2 = String.format("select * from ((select columns[0] ct from dfs.`%s`) \n" +
- "union \n" +
- "(select count(c1) as c2 from (select columns[0] c1 from dfs.`%s`))) order by ct limit 3", root, root);
-
- String query3 = String.format("select * from ((select count(c1) as ct from (select columns[0] c1 from dfs.`%s`) )\n" +
- "union \n" +
- "(select count(c1) as c2 from (select columns[0] c1 from dfs.`%s`))) order by ct", root, root);
+ String root = "/multilevel/csv/1994/Q1/orders_94_q1.csv";
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select * from ((select count(c1) as ct from (select columns[0] c1 from cp.`%s`)) " +
+ "union (select columns[0] c2 from dfs.`%s`)) order by ct limit 3", root, root)
.unOrdered()
.baselineColumns("ct")
.baselineValues((long) 10)
@@ -484,7 +436,8 @@ public class TestUnionDistinct extends BaseTestQuery {
.run();
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select * from ((select columns[0] ct from dfs.`%s`) " +
+ "union (select count(c1) as c2 from (select columns[0] c1 from dfs.`%s`))) order by ct limit 3", root, root)
.unOrdered()
.baselineColumns("ct")
.baselineValues((long) 10)
@@ -494,7 +447,8 @@ public class TestUnionDistinct extends BaseTestQuery {
.run();
testBuilder()
- .sqlQuery(query3)
+ .sqlQuery("select * from ((select count(c1) as ct from (select columns[0] c1 from dfs.`%s`))" +
+ "union (select count(c1) as c2 from (select columns[0] c1 from dfs.`%s`))) order by ct", root, root)
.unOrdered()
.baselineColumns("ct")
.baselineValues((long) 10)
@@ -504,11 +458,9 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctDiffTypesAtPlanning() throws Exception {
- String query = "select count(c1) as ct from (select cast(r_regionkey as int) c1 from cp.`tpch/region.parquet`) \n" +
- "union \n" +
- "(select cast(r_regionkey as int) c2 from cp.`tpch/region.parquet`)";
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select count(c1) as ct from (select cast(r_regionkey as int) c1 from cp.`tpch/region.parquet`) " +
+ "union (select cast(r_regionkey as int) c2 from cp.`tpch/region.parquet`)")
.unOrdered()
.baselineColumns("ct")
.baselineValues((long) 5)
@@ -523,16 +475,9 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctRightEmptyJson() throws Exception {
- String rootEmpty = FileUtils.getResourceAsFile("/project/pushdown/empty.json").toURI().toString();
- String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- String queryRightEmpty = String.format(
- "select key from dfs_test.`%s` \n" +
- "union \n" +
- "select key from dfs_test.`%s`", rootSimple, rootEmpty);
-
testBuilder()
- .sqlQuery(queryRightEmpty)
+ .sqlQuery("select key from cp.`project/pushdown/empty.json` " +
+ "union select key from cp.`store/json/booleanData.json`")
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -543,18 +488,9 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctLeftEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile("/project/pushdown/empty.json").toURI().toString();
- final String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- final String queryLeftEmpty = String.format(
- "select key from dfs_test.`%s` " +
- "union " +
- "select key from dfs_test.`%s`",
- rootEmpty,
- rootSimple);
-
testBuilder()
- .sqlQuery(queryLeftEmpty)
+ .sqlQuery("select key from cp.`project/pushdown/empty.json` " +
+ "union select key from cp.`store/json/booleanData.json`")
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -565,14 +501,7 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctBothEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile("/project/pushdown/empty.json").toURI().toString();
- final String query = String.format(
- "select key from dfs_test.`%s` " +
- "union " +
- "select key from dfs_test.`%s`",
- rootEmpty,
- rootEmpty);
-
+ final String rootEmpty = "project/pushdown/empty.json";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
final TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
.setMinorType(TypeProtos.MinorType.INT)
@@ -581,7 +510,8 @@ public class TestUnionDistinct extends BaseTestQuery {
expectedSchema.add(Pair.of(SchemaPath.getSimplePath("key"), majorType));
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select key from cp.`%s` union select key from cp.`%s`",
+ rootEmpty, rootEmpty)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -589,17 +519,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctRightEmptyDataBatch() throws Exception {
- String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- String queryRightEmptyBatch = String.format(
- "select key from dfs_test.`%s` " +
- "union " +
- "select key from dfs_test.`%s` where 1 = 0",
- rootSimple,
- rootSimple);
+ final String rootSimple = "store/json/booleanData.json";
testBuilder()
- .sqlQuery(queryRightEmptyBatch)
+ .sqlQuery("select key from cp.`%s` union select key from cp.`%s` where 1 = 0",
+ rootSimple, rootSimple)
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -609,17 +533,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctLeftEmptyDataBatch() throws Exception {
- String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- final String queryLeftBatch = String.format(
- "select key from dfs_test.`%s` where 1 = 0 " +
- "union " +
- "select key from dfs_test.`%s`",
- rootSimple,
- rootSimple);
+ final String rootSimple = "store/json/booleanData.json";
testBuilder()
- .sqlQuery(queryLeftBatch)
+ .sqlQuery("select key from cp.`%s` where 1 = 0 union select key from cp.`%s`",
+ rootSimple, rootSimple)
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -630,13 +548,7 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testUnionDistinctBothEmptyDataBatch() throws Exception {
- String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
- final String query = String.format(
- "select key from dfs_test.`%s` where 1 = 0 " +
- "union " +
- "select key from dfs_test.`%s` where 1 = 0",
- rootSimple,
- rootSimple);
+ final String rootSimple = "store/json/booleanData.json";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
final TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
@@ -646,7 +558,9 @@ public class TestUnionDistinct extends BaseTestQuery {
expectedSchema.add(Pair.of(SchemaPath.getSimplePath("key"), majorType));
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select key from cp.`%s` where 1 = 0 " +
+ "union select key from cp.`%s` where 1 = 0",
+ rootSimple, rootSimple)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -654,13 +568,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testFilterPushDownOverUnionDistinct() throws Exception {
- String query = "select n_regionkey from \n"
- + "(select n_regionkey from cp.`tpch/nation.parquet` union select r_regionkey from cp.`tpch/region.parquet`) \n"
- + "where n_regionkey > 0 and n_regionkey < 2 \n"
- + "order by n_regionkey";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select n_regionkey from "
+ + "(select n_regionkey from cp.`tpch/nation.parquet` union select r_regionkey from cp.`tpch/region.parquet`) "
+ + "where n_regionkey > 0 and n_regionkey < 2 "
+ + "order by n_regionkey")
.ordered()
.baselineColumns("n_regionkey")
.baselineValues(1)
@@ -670,14 +582,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testInListPushDownOverUnionDistinct() throws Exception {
- String query = "select n_nationkey \n" +
- "from (select n1.n_nationkey from cp.`tpch/nation.parquet` n1 inner join cp.`tpch/region.parquet` r1 on n1.n_regionkey = r1.r_regionkey \n" +
- "union \n" +
- "select n2.n_nationkey from cp.`tpch/nation.parquet` n2 inner join cp.`tpch/region.parquet` r2 on n2.n_regionkey = r2.r_regionkey) \n" +
- "where n_nationkey in (1, 2)";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select n_nationkey " +
+ "from (select n1.n_nationkey from cp.`tpch/nation.parquet` n1 inner join cp.`tpch/region.parquet` r1 on n1.n_regionkey = r1.r_regionkey " +
+ "union select n2.n_nationkey from cp.`tpch/nation.parquet` n2 inner join cp.`tpch/region.parquet` r2 on n2.n_regionkey = r2.r_regionkey) " +
+ "where n_nationkey in (1, 2)")
.unOrdered()
.baselineColumns("n_nationkey")
.baselineValues(1)
@@ -688,15 +597,13 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testFilterPushDownOverUnionDistinctCSV() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/csv/1994/Q1/orders_94_q1.csv").toURI().toString();
- String query = String.format("select ct \n" +
- "from ((select count(c1) as ct from (select columns[0] c1 from dfs.`%s`)) \n" +
- "union \n" +
- "(select columns[0] c2 from dfs.`%s`)) \n" +
- "where ct < 100", root, root);
+ final String root = "/multilevel/csv/1994/Q1/orders_94_q1.csv";
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select ct " +
+ "from ((select count(c1) as ct from (select columns[0] c1 from cp.`%s`)) " +
+ "union (select columns[0] c2 from cp.`%s`)) " +
+ "where ct < 100", root, root)
.ordered()
.baselineColumns("ct")
.baselineValues((long) 10)
@@ -708,12 +615,10 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testProjectPushDownOverUnionDistinctWithProject() throws Exception {
- String query = "select n_nationkey, n_name from \n" +
- "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` \n" +
- "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select n_nationkey, n_name from " +
+ "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` " +
+ "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/testProjectPushDownOverUnionDistinctWithProject.tsv")
.baselineTypes(TypeProtos.MinorType.INT, TypeProtos.MinorType.VARCHAR)
@@ -724,12 +629,10 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testProjectPushDownOverUnionDistinctWithoutProject() throws Exception {
- String query = "select n_nationkey from \n" +
- "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` \n" +
- "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select n_nationkey from " +
+ "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` " +
+ "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/testProjectPushDownOverUnionDistinctWithoutProject.tsv")
.baselineTypes(TypeProtos.MinorType.INT)
@@ -740,13 +643,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testProjectWithExpressionPushDownOverUnionDistinct() throws Exception {
- String query = "select 2 * n_nationkey as col from \n" +
- "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` \n" +
- "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)";
-
// Validate the result
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select 2 * n_nationkey as col from " +
+ "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` " +
+ "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/testProjectWithExpressionPushDownOverUnionDistinct.tsv")
.baselineTypes(TypeProtos.MinorType.INT)
@@ -757,14 +658,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testProjectDownOverUnionDistinctImplicitCasting() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/nations.csv").toURI().toString();
- String query = String.format("select 2 * n_nationkey as col from \n" +
- "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` \n" +
- "union select columns[0], columns[1], columns[2] from dfs.`%s`) \n" +
- "order by col limit 10", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select 2 * n_nationkey as col from " +
+ "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` " +
+ "union select columns[0], columns[1], columns[2] from cp.`store/text/data/nations.csv`) " +
+ "order by col limit 10")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/testProjectDownOverUnionDistinctImplicitCasting.tsv")
.baselineTypes(TypeProtos.MinorType.INT)
@@ -775,12 +673,10 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testProjectPushDownProjectColumnReorderingAndAlias() throws Exception {
- String query = "select n_comment as col1, n_nationkey as col2, n_name as col3 from \n" +
- "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` \n" +
- "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select n_comment as col1, n_nationkey as col2, n_name as col3 from " +
+ "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` " +
+ "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/testProjectPushDownProjectColumnReorderingAndAlias.tsv")
.baselineTypes(TypeProtos.MinorType.VARCHAR, TypeProtos.MinorType.INT, TypeProtos.MinorType.VARCHAR)
@@ -791,13 +687,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test
public void testProjectFiltertPushDownOverUnionDistinct() throws Exception {
- String query = "select n_nationkey from \n" +
- "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` \n" +
- "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`) \n" +
- "where n_nationkey > 0 and n_nationkey < 4";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select n_nationkey from " +
+ "(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` " +
+ "union select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`) " +
+ "where n_nationkey > 0 and n_nationkey < 4")
.unOrdered()
.csvBaselineFile("testframework/unionDistinct/testProjectFiltertPushDownOverUnionDistinct.tsv")
.baselineTypes(TypeProtos.MinorType.INT)
@@ -808,9 +702,9 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test // DRILL-3296
public void testGroupByUnionDistinct() throws Exception {
- String query = "select n_nationkey from \n" +
- "(select n_nationkey from cp.`tpch/nation.parquet` \n" +
- "union select n_nationkey from cp.`tpch/nation.parquet`) \n" +
+ String query = "select n_nationkey from " +
+ "(select n_nationkey from cp.`tpch/nation.parquet` " +
+ "union select n_nationkey from cp.`tpch/nation.parquet`) " +
"group by n_nationkey";
@@ -833,11 +727,11 @@ public class TestUnionDistinct extends BaseTestQuery {
@Test // DRILL-4147 // union-distinct base case
public void testDrill4147_1() throws Exception {
- final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
- final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+ final String l = "/multilevel/parquet/1994";
+ final String r = "/multilevel/parquet/1995";
- final String query = String.format("SELECT o_custkey FROM dfs_test.`%s` \n" +
- "Union distinct SELECT o_custkey FROM dfs_test.`%s`", l, r);
+ final String query = String.format("SELECT o_custkey FROM dfs.`%s` \n" +
+ "Union distinct SELECT o_custkey FROM dfs.`%s`", l, r);
// Validate the plan
final String[] expectedPlan = {"(?s)UnionExchange.*HashAgg.*HashToRandomExchange.*UnionAll.*"};
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestUtf8SupportInQueryString.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUtf8SupportInQueryString.java b/exec/java-exec/src/test/java/org/apache/drill/TestUtf8SupportInQueryString.java
index e8573f8..8dfddde 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUtf8SupportInQueryString.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUtf8SupportInQueryString.java
@@ -22,6 +22,7 @@ import mockit.MockUp;
import mockit.integration.junit4.JMockit;
import org.apache.calcite.util.SaffronProperties;
import org.apache.drill.common.exceptions.UserRemoteException;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
import org.junit.runner.RunWith;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java
index 41b5a73..c99f0a7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSeparatePlanningTest.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec;
import static org.junit.Assert.*;
import io.netty.buffer.DrillBuf;
+import java.nio.file.Paths;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -27,12 +28,11 @@ import java.util.Map.Entry;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.common.DrillAutoCloseables;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.client.PrintingResultsListener;
import org.apache.drill.exec.client.QuerySubmitter.Format;
@@ -55,6 +55,7 @@ import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.util.VectorUtil;
import org.apache.drill.exec.vector.ValueVector;
+import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -67,20 +68,11 @@ import org.junit.experimental.categories.Category;
*/
@Category({SlowTest.class, PlannerTest.class})
public class DrillSeparatePlanningTest extends BaseTestQuery {
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSeparatePlanningTest.class);
-
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
-
- //final String query = "SELECT sales_city, COUNT(*) cnt FROM cp.`region.json` GROUP BY sales_city";
- //final String query = "SELECT * FROM cp.`employee.json` where employee_id > 1 and employee_id < 1000";
- //final String query = "SELECT o_orderkey, o_custkey FROM dfs.tmp.`multilevel` where dir0 = 1995 and o_orderkey > 100 and o_orderkey < 1000 limit 5";
- //final String query = "SELECT sum(o_totalprice) FROM dfs.tmp.`multilevel` where dir0 = 1995 and o_orderkey > 100 and o_orderkey < 1000";
- //final String query = "SELECT o_orderkey FROM dfs.tmp.`multilevel` order by o_orderkey";
- //final String query = "SELECT dir1, sum(o_totalprice) FROM dfs.tmp.`multilevel` where dir0 = 1995 group by dir1 order by dir1";
- //final String query = String.format("SELECT dir0, sum(o_totalprice) FROM dfs_test.`%s/multilevel/json` group by dir0 order by dir0", TEST_RES_PATH);
-
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel", "json"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel", "csv"));
+ }
@Test(timeout=60000)
public void testSingleFragmentQuery() throws Exception {
@@ -98,7 +90,7 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
@Test(timeout=60000)
public void testMultiMinorFragmentSimpleQuery() throws Exception {
- final String query = String.format("SELECT o_orderkey FROM dfs_test.`%s/multilevel/json`", TEST_RES_PATH);
+ final String query = "SELECT o_orderkey FROM dfs.`multilevel/json`";
QueryPlanFragments planFragments = getFragmentsHelper(query);
@@ -115,7 +107,7 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
@Test(timeout=60000)
public void testMultiMinorFragmentComplexQuery() throws Exception {
- final String query = String.format("SELECT dir0, sum(o_totalprice) FROM dfs_test.`%s/multilevel/json` group by dir0 order by dir0", TEST_RES_PATH);
+ final String query = "SELECT dir0, sum(o_totalprice) FROM dfs.`multilevel/json` group by dir0 order by dir0";
QueryPlanFragments planFragments = getFragmentsHelper(query);
@@ -128,12 +120,11 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
}
getResultsHelper(planFragments);
-
}
@Test(timeout=60000)
public void testPlanningNoSplit() throws Exception {
- final String query = String.format("SELECT dir0, sum(o_totalprice) FROM dfs_test.`%s/multilevel/json` group by dir0 order by dir0", TEST_RES_PATH);
+ final String query = "SELECT dir0, sum(o_totalprice) FROM dfs.`multilevel/json` group by dir0 order by dir0";
updateTestCluster(2, config);
@@ -154,12 +145,11 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
assertFalse(rootFragment.getLeafFragment());
getCombinedResultsHelper(planFragments);
-
}
@Test(timeout=60000)
public void testPlanningNegative() throws Exception {
- final String query = String.format("SELECT dir0, sum(o_totalprice) FROM dfs_test.`%s/multilevel/json` group by dir0 order by dir0", TEST_RES_PATH);
+ final String query = "SELECT dir0, sum(o_totalprice) FROM dfs.`multilevel/json` group by dir0 order by dir0";
updateTestCluster(2, config);
// LOGICAL is not supported
@@ -177,7 +167,7 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
@Test(timeout=60000)
public void testPlanning() throws Exception {
- final String query = String.format("SELECT dir0, columns[3] FROM dfs_test.`%s/multilevel/csv` order by dir0", TEST_RES_PATH);
+ final String query = "SELECT dir0, columns[3] FROM dfs.`multilevel/csv` order by dir0";
updateTestCluster(2, config);
@@ -187,8 +177,6 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
}
AwaitableUserResultsListener listener =
new AwaitableUserResultsListener(new PrintingResultsListener(client.getConfig(), Format.TSV, VectorUtil.DEFAULT_COLUMN_WIDTH));
- //AwaitableUserResultsListener listener =
- // new AwaitableUserResultsListener(new SilentListener());
client.runQuery(QueryType.SQL, query, listener);
@SuppressWarnings("unused")
int rows = listener.await();
@@ -251,8 +239,6 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
List<PlanFragment> fragmentList = Lists.newArrayList();
fragmentList.add(fragment);
- //AwaitableUserResultsListener listener =
- // new AwaitableUserResultsListener(new PrintingResultsListener(client.getConfig(), Format.TSV, VectorUtil.DEFAULT_COLUMN_WIDTH));
AwaitableUserResultsListener listener =
new AwaitableUserResultsListener(new SilentListener());
fragmentClient.runQuery(QueryType.EXECUTION, fragmentList, listener);
@@ -267,9 +253,6 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
@SuppressWarnings("unused")
AwaitableUserResultsListener listenerBits =
new AwaitableUserResultsListener(myListener);
-
- //AwaitableUserResultsListener listener =
- // new AwaitableUserResultsListener(new PrintingResultsListener(client.getConfig(), Format.TSV, VectorUtil.DEFAULT_COLUMN_WIDTH));
AwaitableUserResultsListener listener =
new AwaitableUserResultsListener(new SilentListener());
client.runQuery(QueryType.EXECUTION, planFragments.getFragmentsList(), listener);
@@ -348,6 +331,5 @@ public class DrillSeparatePlanningTest extends BaseTestQuery {
@Override
public void queryCompleted(QueryState state) {
}
-
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
index 418e4a4..9ade940 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
@@ -18,14 +18,12 @@
package org.apache.drill.exec;
import com.codahale.metrics.MetricRegistry;
-import com.google.common.io.Files;
import mockit.Mock;
import mockit.MockUp;
import mockit.NonStrictExpectations;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
-import org.apache.commons.io.FileUtils;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.parser.ExprLexer;
@@ -47,7 +45,6 @@ import org.joda.time.DateTimeUtils;
import org.joda.time.DateTimeZone;
import org.junit.After;
import org.junit.BeforeClass;
-import java.io.File;
import java.io.IOException;
import java.text.DateFormatSymbols;
import java.util.Locale;
@@ -88,23 +85,6 @@ public class ExecTest extends DrillTest {
return FileSystem.get(configuration);
}
- /**
- * Create a temp directory to store the given <i>dirName</i>.
- * Directory will be deleted on exit.
- * @param dirName directory name
- * @return Full path including temp parent directory and given directory name.
- */
- public static String getTempDir(final String dirName) {
- final File dir = Files.createTempDir();
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- FileUtils.deleteQuietly(dir);
- }
- });
- return dir.getAbsolutePath() + File.separator + dirName;
- }
-
protected void mockDrillbitContext(final DrillbitContext bitContext) throws Exception {
new NonStrictExpectations() {{
bitContext.getMetrics(); result = new MetricRegistry();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/TestEmptyInputSql.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestEmptyInputSql.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestEmptyInputSql.java
index 0351250..b691d94 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestEmptyInputSql.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestEmptyInputSql.java
@@ -20,11 +20,10 @@ package org.apache.drill.exec;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.test.rowSet.SchemaBuilder;
import org.junit.Test;
@@ -47,16 +46,13 @@ public class TestEmptyInputSql extends BaseTestQuery {
*/
@Test
public void testQueryEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(SINGLE_EMPTY_JSON).toURI().toString();
- final String query = String.format("select key, key + 100 as key2 from dfs_test.`%s` ", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.addNullable("key", TypeProtos.MinorType.INT)
.addNullable("key2", TypeProtos.MinorType.INT)
.build();
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select key, key + 100 as key2 from cp.`%s`", SINGLE_EMPTY_JSON)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -69,22 +65,17 @@ public class TestEmptyInputSql extends BaseTestQuery {
*/
@Test
public void testQueryStarColEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(SINGLE_EMPTY_JSON).toURI().toString();
- final String query1 = String.format("select * from dfs_test.`%s` ", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.build();
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select * from cp.`%s` ", SINGLE_EMPTY_JSON)
.schemaBaseLine(expectedSchema)
.build()
.run();
- final String query2 = String.format("select *, * from dfs_test.`%s` ", rootEmpty);
-
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select *, * from cp.`%s` ", SINGLE_EMPTY_JSON)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -97,21 +88,16 @@ public class TestEmptyInputSql extends BaseTestQuery {
*/
@Test
public void testQueryQualifiedStarColEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(SINGLE_EMPTY_JSON).toURI().toString();
- final String query1 = String.format("select foo.* from dfs_test.`%s` as foo", rootEmpty);
-
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select foo.* from cp.`%s` as foo", SINGLE_EMPTY_JSON)
.schemaBaseLine(expectedSchema)
.build()
.run();
- final String query2 = String.format("select foo.*, foo.* from dfs_test.`%s` as foo", rootEmpty);
-
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select foo.*, foo.* from cp.`%s` as foo", SINGLE_EMPTY_JSON)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -120,9 +106,6 @@ public class TestEmptyInputSql extends BaseTestQuery {
@Test
public void testQueryMapArrayEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(SINGLE_EMPTY_JSON).toURI().toString();
- final String query = String.format("select foo.a.b as col1, foo.columns[2] as col2, foo.bar.columns[3] as col3 from dfs_test.`%s` as foo", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.addNullable("col1", TypeProtos.MinorType.INT)
.addNullable("col2", TypeProtos.MinorType.INT)
@@ -130,7 +113,7 @@ public class TestEmptyInputSql extends BaseTestQuery {
.build();
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select foo.a.b as col1, foo.columns[2] as col2, foo.bar.columns[3] as col3 from cp.`%s` as foo", SINGLE_EMPTY_JSON)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -144,12 +127,6 @@ public class TestEmptyInputSql extends BaseTestQuery {
*/
@Test
public void testQueryConstExprEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(SINGLE_EMPTY_JSON).toURI().toString();
- final String query = String.format("select 1.0 + 100.0 as key, "
- + " cast(100 as varchar(100)) as name, "
- + " cast(columns as varchar(100)) as name2 "
- + " from dfs_test.`%s` ", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.add("key", TypeProtos.MinorType.FLOAT8)
.add("name", TypeProtos.MinorType.VARCHAR, 100)
@@ -157,7 +134,10 @@ public class TestEmptyInputSql extends BaseTestQuery {
.build();
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select 1.0 + 100.0 as key, "
+ + " cast(100 as varchar(100)) as name, "
+ + " cast(columns as varchar(100)) as name2 "
+ + " from cp.`%s` ", SINGLE_EMPTY_JSON)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -169,14 +149,11 @@ public class TestEmptyInputSql extends BaseTestQuery {
*/
@Test
public void testQueryEmptyCsvH() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(SINGLE_EMPTY_CSVH).toURI().toString();
- final String query1 = String.format("select * from dfs_test.`%s` ", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.build();
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select * from cp.`%s`", SINGLE_EMPTY_CSVH)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -189,15 +166,12 @@ public class TestEmptyInputSql extends BaseTestQuery {
*/
@Test
public void testQueryEmptyCsv() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile(SINGLE_EMPTY_CSV).toURI().toString();
- final String query1 = String.format("select * from dfs_test.`%s` ", rootEmpty);
-
final BatchSchema expectedSchema = new SchemaBuilder()
.addArray("columns", TypeProtos.MinorType.VARCHAR)
.build();
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select * from cp.`%s`", SINGLE_EMPTY_CSV)
.schemaBaseLine(expectedSchema)
.build()
.run();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java
index 0a24073..44cdbc0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestQueriesOnLargeFile.java
@@ -24,12 +24,11 @@ import java.io.File;
import java.io.PrintWriter;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.vector.BigIntVector;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -37,25 +36,13 @@ import com.google.common.base.Charsets;
import com.google.common.io.Files;
public class TestQueriesOnLargeFile extends BaseTestQuery {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestQueriesOnLargeFile.class);
-
private static File dataFile = null;
private static int NUM_RECORDS = 15000;
@BeforeClass
public static void generateTestData() throws Exception {
// Generate a json file with NUM_RECORDS number of records
- while (true) {
- dataFile = File.createTempFile("drill-json", ".json");
- if (dataFile.exists()) {
- boolean success = dataFile.delete();
- if (success) {
- break;
- }
- }
- logger.trace("retry creating tmp file");
- }
-
+ dataFile = File.createTempFile("drill-json", ".json", dirTestWatcher.getRootDir());
PrintWriter printWriter = new PrintWriter(dataFile);
for (int i=1; i<=NUM_RECORDS; i++) {
@@ -73,7 +60,7 @@ public class TestQueriesOnLargeFile extends BaseTestQuery {
@Test
public void testRead() throws Exception {
List<QueryDataBatch> results = testSqlWithResults(
- String.format("SELECT count(*) FROM dfs_test.`default`.`%s`", dataFile.getPath()));
+ String.format("SELECT count(*) FROM dfs.`default`.`%s`", dataFile.getName()));
RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
@@ -94,7 +81,7 @@ public class TestQueriesOnLargeFile extends BaseTestQuery {
@Test
public void testMergingReceiver() throws Exception {
- String plan = Files.toString(FileUtils.getResourceAsFile("/largefiles/merging_receiver_large_data.json"),
+ String plan = Files.toString(DrillFileUtils.getResourceAsFile("/largefiles/merging_receiver_large_data.json"),
Charsets.UTF_8).replace("#{TEST_FILE}", escapeJsonString(dataFile.getPath()));
List<QueryDataBatch> results = testPhysicalWithResults(plan);
@@ -107,13 +94,4 @@ public class TestQueriesOnLargeFile extends BaseTestQuery {
assertTrue(String.format("Number of records in output is wrong: expected=%d, actual=%s",
NUM_RECORDS, recordsInOutput), NUM_RECORDS == recordsInOutput);
}
-
- @AfterClass
- public static void deleteTestData() throws Exception {
- if (dataFile != null) {
- if (dataFile.exists()) {
- org.apache.commons.io.FileUtils.forceDelete(dataFile);
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java
index 63c3be5..6c00d81 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java
@@ -18,49 +18,22 @@
package org.apache.drill.exec;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.categories.UnlikelyTest;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.BeforeClass;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
-import org.junit.experimental.categories.Category;
public class TestRepeatedReaders extends BaseTestQuery {
- static FileSystem fs;
-
- @BeforeClass
- public static void initFs() throws Exception {
- fs = getLocalFileSystem();
- }
-
- private static void deleteTableIfExists(String tableName) {
- try {
- Path path = new Path(getDfsTestTmpSchemaLocation(), tableName);
- if (fs.exists(path)) {
- fs.delete(path, true);
- }
- } catch (Exception e) {
- // ignore exceptions.
- }
- }
-
private void createAndQuery(String datafile) throws Exception {
String query = String.format("select * from cp.`parquet/%s`", datafile);
String tableName = "test_repeated_readers_"+datafile;
- try {
- test("create table dfs_test.tmp.`%s` as %s", tableName, query);
+ test("create table dfs.tmp.`%s` as %s", tableName, query);
- testBuilder()
- .sqlQuery("select * from dfs_test.tmp.`%s` d", tableName)
- .ordered()
- .jsonBaselineFile("parquet/" + datafile)
- .go();
- } finally {
- deleteTableIfExists(tableName);
- }
+ testBuilder()
+ .sqlQuery("select * from dfs.tmp.`%s` d", tableName)
+ .ordered()
+ .jsonBaselineFile("parquet/" + datafile)
+ .go();
}
@Test //DRILL-2292
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java
index bd616c7..7d7f4a5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java
@@ -17,29 +17,34 @@
*/
package org.apache.drill.exec;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.work.foreman.SqlUnsupportedException;
import org.apache.drill.exec.work.foreman.UnsupportedFunctionException;
import org.apache.drill.PlanTestBase;
import org.apache.drill.test.UserExceptionMatcher;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-public class TestWindowFunctions extends BaseTestQuery {
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
+import java.nio.file.Paths;
+public class TestWindowFunctions extends BaseTestQuery {
private static void throwAsUnsupportedException(UserException ex) throws Exception {
SqlUnsupportedException.errorClassNameToException(ex.getOrCreatePBError(false).getException().getExceptionClass());
throw ex;
}
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel/parquet"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("window"));
+ }
+
@Test // DRILL-3196
public void testSinglePartition() throws Exception {
final String query = "select sum(n_nationKey) over(partition by n_nationKey) as col1, count(*) over(partition by n_nationKey) as col2 \n" +
@@ -240,7 +245,7 @@ public class TestWindowFunctions extends BaseTestQuery {
"from testWindowGroupByOnView \n" +
"group by b";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test(createView);
test(query);
} finally {
@@ -406,7 +411,7 @@ public class TestWindowFunctions extends BaseTestQuery {
@Test // DRILL-3298
public void testCountEmptyPartitionByWithExchange() throws Exception {
- String query = String.format("select count(*) over (order by o_orderpriority) as cnt from dfs.`%s/multilevel/parquet` where o_custkey < 100", TEST_RES_PATH);
+ String query = "select count(*) over (order by o_orderpriority) as cnt from dfs.`multilevel/parquet` where o_custkey < 100";
try {
// Validate the plan
final String[] expectedPlan = {"Window.*partition \\{\\} order by \\[0\\].*COUNT\\(\\)",
@@ -508,9 +513,9 @@ public class TestWindowFunctions extends BaseTestQuery {
@Test
public void testCompoundIdentifierInWindowDefinition() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/csv/1994/Q1/orders_94_q1.csv").toURI().toString();
+ String root = DrillFileUtils.getResourceAsFile("/multilevel/csv/1994/Q1/orders_94_q1.csv").toURI().toString();
String query = String.format("SELECT count(*) OVER w as col1, count(*) OVER w as col2 \n" +
- "FROM dfs_test.`%s` \n" +
+ "FROM dfs.`%s` \n" +
"WINDOW w AS (PARTITION BY columns[1] ORDER BY columns[0] DESC)", root);
// Validate the plan
@@ -559,7 +564,7 @@ public class TestWindowFunctions extends BaseTestQuery {
@Test // DRILL-3404
public void testWindowSumAggIsNotNull() throws Exception {
- String query = String.format("select count(*) cnt from (select sum ( c1 ) over ( partition by c2 order by c1 asc nulls first ) w_sum from dfs.`%s/window/table_with_nulls.parquet` ) sub_query where w_sum is not null", TEST_RES_PATH);
+ String query = "select count(*) cnt from (select sum ( c1 ) over ( partition by c2 order by c1 asc nulls first ) w_sum from dfs.`window/table_with_nulls.parquet` ) sub_query where w_sum is not null";
// Validate the plan
final String[] expectedPlan = {"Window.*partition \\{1\\} order by \\[0 ASC-nulls-first\\].*SUM\\(\\$0\\)",
@@ -604,11 +609,11 @@ public class TestWindowFunctions extends BaseTestQuery {
@Test // DRILL-3567
public void testMultiplePartitions1() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
+ String root = DrillFileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
String query = String.format("select count(*) over(partition by b1 order by c1) as count1, \n" +
"sum(a1) over(partition by b1 order by c1) as sum1, \n" +
"count(*) over(partition by a1 order by c1) as count2 \n" +
- "from dfs_test.`%s`", root);
+ "from dfs.`%s`", root);
// Validate the plan
final String[] expectedPlan = {"Window.*partition \\{2\\} order by \\[1\\].*COUNT\\(\\)",
@@ -637,11 +642,11 @@ public class TestWindowFunctions extends BaseTestQuery {
@Test // DRILL-3567
public void testMultiplePartitions2() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
+ String root = DrillFileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
String query = String.format("select count(*) over(partition by b1 order by c1) as count1, \n" +
"count(*) over(partition by a1 order by c1) as count2, \n" +
"sum(a1) over(partition by b1 order by c1) as sum1 \n" +
- "from dfs_test.`%s`", root);
+ "from dfs.`%s`", root);
// Validate the plan
final String[] expectedPlan = {"Window.*partition \\{2\\} order by \\[1\\].*COUNT\\(\\)",
@@ -670,9 +675,9 @@ public class TestWindowFunctions extends BaseTestQuery {
@Test // see DRILL-3574
public void testWithAndWithoutPartitions() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
+ String root = DrillFileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
String query = String.format("select sum(a1) over(partition by b1, c1) as s1, sum(a1) over() as s2 \n" +
- "from dfs_test.`%s` \n" +
+ "from dfs.`%s` \n" +
"order by a1", root);
test("alter session set `planner.slice_target` = 1");
@@ -701,10 +706,10 @@ public class TestWindowFunctions extends BaseTestQuery {
@Test // see DRILL-3657
public void testConstantsInMultiplePartitions() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
+ String root = DrillFileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
String query = String.format(
"select sum(1) over(partition by b1 order by a1) as sum1, sum(1) over(partition by a1) as sum2, rank() over(order by b1) as rank1, rank() over(order by 1) as rank2 \n" +
- "from dfs_test.`%s` \n" +
+ "from dfs.`%s` \n" +
"order by 1, 2, 3, 4", root);
// Validate the plan
@@ -735,9 +740,9 @@ public class TestWindowFunctions extends BaseTestQuery {
@Test // DRILL-3580
public void testExpressionInWindowFunction() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
+ String root = DrillFileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
String query = String.format("select a1, b1, sum(b1) over (partition by a1) as c1, sum(a1 + b1) over (partition by a1) as c2\n" +
- "from dfs_test.`%s`", root);
+ "from dfs.`%s`", root);
// Validate the plan
final String[] expectedPlan = {"Window\\(window#0=\\[window\\(partition \\{0\\} order by \\[\\].*\\[SUM\\(\\$1\\), SUM\\(\\$2\\)\\]"};
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
index f94cf21..6e53d10 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
@@ -24,7 +24,7 @@ import com.google.common.io.Files;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.record.MaterializedField;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
index 3dc31ca..63957f4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.FileInputStream;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
@@ -64,7 +64,7 @@ public class DumpCatTest extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
index b941816..97dfd6b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.compile;
import java.io.IOException;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.compile.ClassTransformer.ClassSet;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
@@ -121,7 +121,7 @@ public class TestClassTransformation extends BaseTestQuery {
private <T, X extends T> CodeGenerator<T> newCodeGenerator(Class<T> iface, Class<X> impl) {
final TemplateClassDefinition<T> template = new TemplateClassDefinition<T>(iface, impl);
- CodeGenerator<T> cg = CodeGenerator.get(template, getDrillbitContext().getFunctionImplementationRegistry(), getDrillbitContext().getOptionManager());
+ CodeGenerator<T> cg = CodeGenerator.get(template, getDrillbitContext().getOptionManager());
cg.plainJavaCapable(true);
ClassGenerator<T> root = cg.getRoot();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
index 6b16b26..3c41c81 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java
@@ -45,8 +45,8 @@ public class TestEvaluationVisitor {
DrillConfig c = DrillConfig.create();
FunctionImplementationRegistry reg = new FunctionImplementationRegistry(c);
- EvaluationVisitor v = new EvaluationVisitor(reg);
- CodeGenerator<?> g = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, reg, null);
+ EvaluationVisitor v = new EvaluationVisitor();
+ CodeGenerator<?> g = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, null);
SchemaPath path = (SchemaPath) getExpr("a.b[4][2].c[6]");
TypedFieldId id = TypedFieldId.newBuilder() //
@@ -75,13 +75,6 @@ public class TestEvaluationVisitor {
private LogicalExpression getExpr(String expr) throws Exception{
ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
CommonTokenStream tokens = new CommonTokenStream(lexer);
-
-// tokens.fill();
-// for(Token t : (List<Token>) tokens.getTokens()){
-// System.out.println(t + "" + t.getType());
-// }
-// tokens.rewind();
-
ExprParser parser = new ExprParser(tokens);
parse_return ret = parser.parse();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
index 1903f35..bec6744 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
@@ -17,13 +17,16 @@
*/
package org.apache.drill.exec.compile;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.categories.SlowTest;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.rules.TestRule;
+@Category(SlowTest.class)
public class TestLargeFileCompilation extends BaseTestQuery {
@Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(150000); // 150secs
@@ -120,7 +123,7 @@ public class TestLargeFileCompilation extends BaseTestQuery {
@Test
public void testTEXT_WRITER() throws Exception {
testNoResult("alter session set `%s`='JDK'", ClassCompilerSelector.JAVA_COMPILER_OPTION);
- testNoResult("use dfs_test.tmp");
+ testNoResult("use dfs.tmp");
testNoResult("alter session set `%s`='csv'", ExecConstants.OUTPUT_FORMAT_OPTION);
testNoResult(LARGE_QUERY_WRITER, "wide_table_csv");
}
@@ -128,7 +131,7 @@ public class TestLargeFileCompilation extends BaseTestQuery {
@Test
public void testPARQUET_WRITER() throws Exception {
testNoResult("alter session set `%s`='JDK'", ClassCompilerSelector.JAVA_COMPILER_OPTION);
- testNoResult("use dfs_test.tmp");
+ testNoResult("use dfs.tmp");
testNoResult("alter session set `%s`='parquet'", ExecConstants.OUTPUT_FORMAT_OPTION);
testNoResult(ITERATION_COUNT, LARGE_QUERY_WRITER, "wide_table_parquet");
}
@@ -170,7 +173,7 @@ public class TestLargeFileCompilation extends BaseTestQuery {
testNoResult("alter session set `%s`='JDK'", ClassCompilerSelector.JAVA_COMPILER_OPTION);
testNoResult("alter session set `planner.enable_mergejoin` = false");
testNoResult("alter session set `planner.enable_nestedloopjoin` = false");
- testNoResult("use dfs_test.tmp");
+ testNoResult("use dfs.tmp");
testNoResult(LARGE_TABLE_WRITER, tableName);
testNoResult(QUERY_WITH_JOIN, tableName);
} finally {
@@ -188,7 +191,7 @@ public class TestLargeFileCompilation extends BaseTestQuery {
testNoResult("alter session set `%s`='JDK'", ClassCompilerSelector.JAVA_COMPILER_OPTION);
testNoResult("alter session set `planner.enable_hashjoin` = false");
testNoResult("alter session set `planner.enable_nestedloopjoin` = false");
- testNoResult("use dfs_test.tmp");
+ testNoResult("use dfs.tmp");
testNoResult(LARGE_TABLE_WRITER, tableName);
testNoResult(QUERY_WITH_JOIN, tableName);
} finally {
@@ -207,7 +210,7 @@ public class TestLargeFileCompilation extends BaseTestQuery {
testNoResult("alter session set `planner.enable_nljoin_for_scalar_only` = false");
testNoResult("alter session set `planner.enable_hashjoin` = false");
testNoResult("alter session set `planner.enable_mergejoin` = false");
- testNoResult("use dfs_test.tmp");
+ testNoResult("use dfs.tmp");
testNoResult(LARGE_TABLE_WRITER, tableName);
testNoResult(QUERY_WITH_JOIN, tableName);
} finally {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
index af2ee46..fbbb4c8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
@@ -23,9 +23,6 @@ import mockit.Injectable;
import mockit.NonStrict;
import mockit.NonStrictExpectations;
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.RecognitionException;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExpressionParsingException;
import org.apache.drill.common.expression.ErrorCollector;
@@ -33,9 +30,6 @@ import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
-import org.apache.drill.common.expression.parser.ExprParser.parse_return;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
@@ -130,7 +124,7 @@ public class ExpressionTest extends ExecTest {
}
FunctionImplementationRegistry funcReg = new FunctionImplementationRegistry(DrillConfig.create());
- final ClassGenerator<Projector> cg = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, funcReg, null).getRoot();
+ final ClassGenerator<Projector> cg = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, null).getRoot();
cg.addExpr(new ValueVectorWriteExpression(new TypedFieldId(materializedExpr.getMajorType(), -1), materializedExpr));
return cg.getCodeGenerator().generateAndGet();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java
index 3ddf776..bd465df 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestLogicalExprSerDe.java
@@ -18,7 +18,7 @@ package org.apache.drill.exec.expr;
* limitations under the License.
*/
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.exec.ExecConstants;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
index b7c6614..bae200c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
@@ -17,18 +17,26 @@
*/
package org.apache.drill.exec.expr;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.exec.proto.UserBitShared;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
@Category({SqlTest.class, PlannerTest.class})
public class TestSchemaPathMaterialization extends BaseTestQuery {
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("complex", "json", "multiple"));
+ }
+
@Test
public void testSingleProjectionFromMultiLevelRepeatedList() throws Exception {
final String query = "select t.odd[2][0][0] v1 " +
@@ -90,7 +98,7 @@ public class TestSchemaPathMaterialization extends BaseTestQuery {
@Test //DRILL-1962
@Category(UnlikelyTest.class)
public void testProjectionMultipleFiles() throws Exception {
- final String query="select t.oooa.oa.oab.oabc[1].rowValue1 rowValue from dfs.`${WORKING_PATH}/src/test/resources/complex/json/multiple/*.json` t";
+ final String query="select t.oooa.oa.oab.oabc[1].rowValue1 rowValue from dfs.`complex/json/multiple/*.json` t";
testBuilder()
.sqlQuery(query)
@@ -102,7 +110,7 @@ public class TestSchemaPathMaterialization extends BaseTestQuery {
@Test //DRILL-4264
@Category(UnlikelyTest.class)
public void testFieldNameWithDot() throws Exception {
- final String tableName = "dfs_test.tmp.table_with_dot_field";
+ final String tableName = "dfs.tmp.table_with_dot_field";
try {
test("create table %s as select o_custkey as `x.y.z` from cp.`tpch/orders.parquet`", tableName);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
index 3beef0d..c2d4bdb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
@@ -22,7 +22,7 @@ import mockit.Mock;
import mockit.MockUp;
import mockit.integration.junit4.JMockit;
import org.apache.drill.categories.SqlFunctionTest;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.util.JarUtil;
import org.codehaus.janino.Java;
import org.junit.BeforeClass;
@@ -30,9 +30,9 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
-import java.io.File;
import java.net.URL;
import java.net.URLClassLoader;
+import java.nio.file.Path;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -56,10 +56,12 @@ public class FunctionInitializerTest {
@BeforeClass
public static void init() throws Exception {
- File jars = new File(TestTools.getWorkingPath(), "/src/test/resources/jars");
+ Path jars = TestTools.WORKING_PATH
+ .resolve(TestTools.TEST_RESOURCES)
+ .resolve("jars");
String binaryName = "DrillUDF-1.0.jar";
String sourceName = JarUtil.getSourceName(binaryName);
- URL[] urls = {new File(jars, binaryName).toURI().toURL(), new File(jars, sourceName).toURI().toURL()};
+ URL[] urls = {jars.resolve(binaryName).toUri().toURL(), jars.resolve(sourceName).toUri().toURL()};
classLoader = new URLClassLoader(urls);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java
index a8a69e1..dfb74da 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/impl/TestStringFunctions.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue;
import mockit.integration.junit4.JMockit;
import org.apache.commons.io.FileUtils;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.exec.util.Text;
import org.junit.Ignore;
@@ -1408,22 +1408,13 @@ public class TestStringFunctions extends BaseTestQuery {
@Test // DRILL-5424
public void testReverseLongVarChars() throws Exception {
- File path = new File(BaseTestQuery.getTempDir("input"));
- try {
- path.mkdirs();
- String pathString = path.toPath().toString();
-
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(path, "table_with_long_varchars.json")))) {
- for (int i = 0; i < 10; i++) {
- writer.write("{ \"a\": \"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\"}");
- }
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "table_with_long_varchars.json")))) {
+ for (int i = 0; i < 10; i++) {
+ writer.write("{ \"a\": \"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\"}");
}
-
- test("select reverse(a) from dfs_test.`%s/table_with_long_varchars.json` t", pathString);
-
- } finally {
- FileUtils.deleteQuietly(path);
}
+
+ test("select reverse(a) from dfs.`table_with_long_varchars.json` t");
}
@Test
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java
index 3d7bac0..e7cdf44 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunction.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
@@ -52,7 +52,7 @@ public class TestAggregateFunction extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile(planPath), Charsets.UTF_8).replace("#{TEST_FILE}", dataPath));
+ Files.toString(DrillFileUtils.getResourceAsFile(planPath), Charsets.UTF_8).replace("#{TEST_FILE}", dataPath));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
[08/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java
index cca2bd0..194b78b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.store.dfs;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestTools;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
@@ -27,7 +27,7 @@ public class TestSchemaNotFoundException extends BaseTestQuery {
@Test(expected = Exception.class)
public void testSchemaNotFoundForWrongStoragePlgn() throws Exception {
- final String table = String.format("%s/empty", TestTools.getTestResourcesPath());
+ final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES));
final String query = String.format("select * from dfs1.`%s`", table);
try {
testNoResult(query);
@@ -41,7 +41,7 @@ public class TestSchemaNotFoundException extends BaseTestQuery {
@Test(expected = Exception.class)
public void testSchemaNotFoundForWrongWorkspace() throws Exception {
- final String table = String.format("%s/empty", TestTools.getTestResourcesPath());
+ final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES));
final String query = String.format("select * from dfs.tmp1.`%s`", table);
try {
testNoResult(query);
@@ -55,7 +55,7 @@ public class TestSchemaNotFoundException extends BaseTestQuery {
@Test(expected = Exception.class)
public void testSchemaNotFoundForWrongWorkspaceUsingDefaultWorkspace() throws Exception {
- final String table = String.format("%s/empty", TestTools.getTestResourcesPath());
+ final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES));
final String query = String.format("select * from tmp1.`%s`", table);
try {
testNoResult("use dfs");
@@ -70,7 +70,7 @@ public class TestSchemaNotFoundException extends BaseTestQuery {
@Test(expected = Exception.class)
public void testTableNotFoundException() throws Exception {
- final String table = String.format("%s/empty1", TestTools.getTestResourcesPath());
+ final String table = String.format("%s/empty1", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES));
final String query = String.format("select * from tmp.`%s`", table);
try {
testNoResult("use dfs");
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
index b7bc9fd..c792233 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
@@ -29,13 +29,16 @@ import java.io.PrintWriter;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.store.easy.text.TextFormatPlugin.TextFormatConfig;
+import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSetBuilder;
import org.apache.drill.test.rowSet.RowSetComparison;
import org.apache.drill.test.rowSet.SchemaBuilder;
+import org.apache.drill.test.DirTestWatcher;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
/**
@@ -49,20 +52,21 @@ public class TestCsv extends ClusterTest {
private static File testDir;
+ @ClassRule
+ public static final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
@BeforeClass
public static void setup() throws Exception {
- startCluster(ClusterFixture.builder()
- .maxParallelization(1)
- );
+ startCluster(ClusterFixture.builder(dirTestWatcher).maxParallelization(1));
// Set up CSV storage plugin using headers.
- testDir = cluster.makeTempDir("csv");
TextFormatConfig csvFormat = new TextFormatConfig();
csvFormat.fieldDelimiter = ',';
csvFormat.skipFirstLine = false;
csvFormat.extractHeader = true;
- cluster.defineWorkspace("dfs", "data", testDir.getAbsolutePath(), "csv", csvFormat);
+
+ testDir = cluster.makeDataDir("data", "csv", csvFormat);
}
String emptyHeaders[] = {
@@ -136,8 +140,8 @@ public class TestCsv extends ClusterTest {
public void testCsvHeadersCaseInsensitive() throws IOException {
String fileName = "case2.csv";
buildFile(fileName, validHeaders);
- String sql = "SELECT A, b, C FROM `dfs.data`.`" + fileName + "`";
- RowSet actual = client.queryBuilder().sql(sql).rowSet();
+ String sql = "SELECT A, b, C FROM `dfs.data`.`%s`";
+ RowSet actual = client.queryBuilder().sql(sql, fileName).rowSet();
BatchSchema expectedSchema = new SchemaBuilder()
.add("A", MinorType.VARCHAR)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java
index 6322224..5061346 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/ischema/Drill2283InfoSchemaVarchar1BugTest.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.store.ischema;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
index 7f787b8..7eb7259 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
@@ -17,35 +17,35 @@
*/
package org.apache.drill.exec.store.json;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.ExecConstants;
-import org.junit.Ignore;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.Assert;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class TestJsonRecordReader extends BaseTestQuery {
- // private static final org.slf4j.Logger logger =
- // org.slf4j.LoggerFactory.getLogger(TestJsonRecordReader.class);
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("jsoninput/drill_3353"));
+ }
@Test
public void testComplexJsonInput() throws Exception {
- // test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10");
test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` limit 10 ");
- // test("select x from cp.`jsoninput/input2.json`");
-
- // test("select z[0] from cp.`jsoninput/input2.json` limit 10");
}
@Test
public void testContainingArray() throws Exception {
- test("select * from dfs.`${WORKING_PATH}/src/test/resources/store/json/listdoc.json`");
+ test("select * from cp.`store/json/listdoc.json`");
}
@Test
@@ -110,7 +110,7 @@ public class TestJsonRecordReader extends BaseTestQuery {
@Category(UnlikelyTest.class)
// DRILL-1832
public void testJsonWithNulls2() throws Exception {
- final String query = "select SUM(1) as `sum_Number_of_Records_ok` from cp.`/jsoninput/twitter_43.json` having (COUNT(1) > 0)";
+ final String query = "select SUM(1) as `sum_Number_of_Records_ok` from cp.`jsoninput/twitter_43.json` having (COUNT(1) > 0)";
testBuilder().sqlQuery(query).unOrdered()
.jsonBaselineFile("jsoninput/drill-1832-2-result.json").go();
}
@@ -160,8 +160,8 @@ public class TestJsonRecordReader extends BaseTestQuery {
public void drill_3353() throws Exception {
try {
testNoResult("alter session set `store.json.all_text_mode` = true");
- test("create table dfs_test.tmp.drill_3353 as select a from dfs.`${WORKING_PATH}/src/test/resources/jsoninput/drill_3353` where e = true");
- String query = "select t.a.d cnt from dfs_test.tmp.drill_3353 t where t.a.d is not null";
+ test("create table dfs.tmp.drill_3353 as select a from dfs.`jsoninput/drill_3353` where e = true");
+ String query = "select t.a.d cnt from dfs.tmp.drill_3353 t where t.a.d is not null";
test(query);
testBuilder().sqlQuery(query).unOrdered().baselineColumns("cnt")
.baselineValues("1").go();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java
index 0f7c213..5a5207b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java
@@ -17,7 +17,8 @@
*/
package org.apache.drill.exec.store.parquet;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.BaseDirTestWatcher;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
@@ -25,6 +26,7 @@ import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.ClusterFixtureBuilder;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import java.util.HashMap;
@@ -32,9 +34,12 @@ import java.util.Map;
public class ParquetInternalsTest extends ClusterTest {
+ @ClassRule
+ public static BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
@BeforeClass
public static void setup( ) throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
// Set options, etc.
;
startCluster(builder);
@@ -44,7 +49,6 @@ public class ParquetInternalsTest extends ClusterTest {
public void testFixedWidth() throws Exception {
String sql = "SELECT l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity\n" +
"FROM `cp`.`tpch/lineitem.parquet` LIMIT 20";
-// client.queryBuilder().sql(sql).printCsv();
Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
typeMap.put(TestBuilder.parsePath("l_orderkey"), Types.required(TypeProtos.MinorType.INT));
@@ -66,7 +70,6 @@ public class ParquetInternalsTest extends ClusterTest {
public void testVariableWidth() throws Exception {
String sql = "SELECT s_name, s_address, s_phone, s_comment\n" +
"FROM `cp`.`tpch/supplier.parquet` LIMIT 20";
-// client.queryBuilder().sql(sql).printCsv();
Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
typeMap.put(TestBuilder.parsePath("s_name"), Types.required(TypeProtos.MinorType.VARCHAR));
@@ -87,7 +90,6 @@ public class ParquetInternalsTest extends ClusterTest {
public void testMixedWidth() throws Exception {
String sql = "SELECT s_suppkey, s_name, s_address, s_phone, s_acctbal\n" +
"FROM `cp`.`tpch/supplier.parquet` LIMIT 20";
-// client.queryBuilder().sql(sql).printCsv();
Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
typeMap.put(TestBuilder.parsePath("s_suppkey"), Types.required(TypeProtos.MinorType.INT));
@@ -109,7 +111,6 @@ public class ParquetInternalsTest extends ClusterTest {
public void testStar() throws Exception {
String sql = "SELECT *\n" +
"FROM `cp`.`tpch/supplier.parquet` LIMIT 20";
-// client.queryBuilder().sql(sql).printCsv();
Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
typeMap.put(TestBuilder.parsePath("s_suppkey"), Types.required(TypeProtos.MinorType.INT));
@@ -140,19 +141,17 @@ public class ParquetInternalsTest extends ClusterTest {
// TODO: Once the "row set" fixture is available, use that to verify
// that all rows are null.
-// client.queryBuilder().sql(sql).printCsv();
-
// Can't handle nulls this way...
-// Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
-// typeMap.put(TestBuilder.parsePath("s_suppkey"), Types.required(TypeProtos.MinorType.INT));
-// typeMap.put(TestBuilder.parsePath("bogus"), Types.optional(TypeProtos.MinorType.INT));
-// client.testBuilder()
-// .sqlQuery(sql)
-// .unOrdered()
-// .csvBaselineFile("parquet/expected/bogus.csv")
-// .baselineColumns("s_suppkey", "bogus")
-// .baselineTypes(typeMap)
-// .build()
-// .run();
+ // Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
+ // typeMap.put(TestBuilder.parsePath("s_suppkey"), Types.required(TypeProtos.MinorType.INT));
+ // typeMap.put(TestBuilder.parsePath("bogus"), Types.optional(TypeProtos.MinorType.INT));
+ // client.testBuilder()
+ // .sqlQuery(sql)
+ // .unOrdered()
+ // .csvBaselineFile("parquet/expected/bogus.csv")
+ // .baselineColumns("s_suppkey", "bogus")
+ // .baselineTypes(typeMap)
+ // .build()
+ // .run();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index 375ab75..397c512 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -32,12 +32,12 @@ import java.util.concurrent.TimeUnit;
import mockit.Injectable;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
@@ -90,7 +90,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
private static final int numberRowGroups = 1;
private static final int recordsPerRowGroup = 300;
private static int DEFAULT_BYTES_PER_PAGE = 1024 * 1024 * 1;
- private static final String fileName = "/tmp/parquet_test_file_many_types";
+ private static final String fileName = "tmp/parquet_test_file_many_types";
@BeforeClass
public static void generateFile() throws Exception {
@@ -103,15 +103,14 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
}
}
-
@Test
public void testMultipleRowGroupsAndReads3() throws Exception {
- final String planName = "/parquet/parquet_scan_screen.json";
+ final String planName = "parquet/parquet_scan_screen.json";
testParquetFullEngineLocalPath(planName, fileName, 2, numberRowGroups, recordsPerRowGroup);
}
public String getPlanForFile(String pathFileName, String parquetFileName) throws IOException {
- return Files.toString(FileUtils.getResourceAsFile(pathFileName), Charsets.UTF_8)
+ return Files.toString(DrillFileUtils.getResourceAsFile(pathFileName), Charsets.UTF_8)
.replaceFirst("&REPLACED_IN_PARQUET_TEST&", parquetFileName);
}
@@ -129,22 +128,21 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
}
}
- final String planText = Files.toString(FileUtils.getResourceAsFile(
- "/parquet/parquet_scan_screen_read_entry_replace.json"), Charsets.UTF_8).replaceFirst(
+ final String planText = Files.toString(DrillFileUtils.getResourceAsFile(
+ "parquet/parquet_scan_screen_read_entry_replace.json"), Charsets.UTF_8).replaceFirst(
"&REPLACED_IN_PARQUET_TEST&", readEntries.toString());
testParquetFullEngineLocalText(planText, fileName, i, numberRowGroups, recordsPerRowGroup, true);
}
@Test
-
public void testDictionaryError() throws Exception {
- testFull(QueryType.SQL, "select L_RECEIPTDATE from dfs.`/tmp/lineitem_null_dict.parquet`", "", 1, 1, 100000, false);
+ testFull(QueryType.SQL, "select L_RECEIPTDATE from dfs.`tmp/lineitem_null_dict.parquet`", "", 1, 1, 100000, false);
}
@Test
public void testNullableAgg() throws Exception {
final List<QueryDataBatch> result = testSqlWithResults(
- "select sum(a) as total_sum from dfs.`/tmp/parquet_with_nulls_should_sum_100000_nulls_first.parquet`");
+ "select sum(a) as total_sum from dfs.`tmp/parquet_with_nulls_should_sum_100000_nulls_first.parquet`");
assertEquals("Only expected one batch with data, and then the empty finishing batch.", 2, result.size());
final RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator());
@@ -163,7 +161,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
@Test
public void testNullableFilter() throws Exception {
final List<QueryDataBatch> result = testSqlWithResults(
- "select count(wr_return_quantity) as row_count from dfs.`/tmp/web_returns` where wr_return_quantity = 1");
+ "select count(wr_return_quantity) as row_count from dfs.`tmp/web_returns` where wr_return_quantity = 1");
assertEquals("Only expected one batch with data, and then the empty finishing batch.", 2, result.size());
final RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator());
@@ -182,9 +180,9 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
@Test
public void testFixedBinary() throws Exception {
- final String readEntries = "\"/tmp/drilltest/fixed_binary.parquet\"";
- final String planText = Files.toString(FileUtils.getResourceAsFile(
- "/parquet/parquet_scan_screen_read_entry_replace.json"), Charsets.UTF_8)
+ final String readEntries = "\"tmp/drilltest/fixed_binary.parquet\"";
+ final String planText = Files.toString(DrillFileUtils.getResourceAsFile(
+ "parquet/parquet_scan_screen_read_entry_replace.json"), Charsets.UTF_8)
.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries);
testParquetFullEngineLocalText(planText, fileName, 1, 1, 1000000, false);
}
@@ -192,35 +190,35 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
@Test
public void testNonNullableDictionaries() throws Exception {
testFull(QueryType.SQL,
- "select * from dfs.`/tmp/drilltest/non_nullable_dictionary.parquet`", "", 1, 1, 30000000, false);
+ "select * from dfs.`tmp/drilltest/non_nullable_dictionary.parquet`", "", 1, 1, 30000000, false);
}
@Test
public void testNullableVarCharMemory() throws Exception {
testFull(QueryType.SQL,
- "select s_comment,s_suppkey from dfs.`/tmp/sf100_supplier.parquet`", "", 1, 1, 1000, false);
+ "select s_comment,s_suppkey from dfs.`tmp/sf100_supplier.parquet`", "", 1, 1, 1000, false);
}
@Test
public void testReadVoter() throws Exception {
- testFull(QueryType.SQL, "select * from dfs.`/tmp/voter.parquet`", "", 1, 1, 1000, false);
+ testFull(QueryType.SQL, "select * from dfs.`tmp/voter.parquet`", "", 1, 1, 1000, false);
}
@Test
public void testDrill_1314() throws Exception {
testFull(QueryType.SQL, "select l_partkey " +
- "from dfs.`/tmp/drill_1314.parquet`", "", 1, 1, 10000, false);
+ "from dfs.`tmp/drill_1314.parquet`", "", 1, 1, 10000, false);
}
@Test
public void testDrill_1314_all_columns() throws Exception {
- testFull(QueryType.SQL, "select * from dfs.`/tmp/drill_1314.parquet`", "", 1, 1, 10000, false);
+ testFull(QueryType.SQL, "select * from dfs.`tmp/drill_1314.parquet`", "", 1, 1, 10000, false);
}
@Test
public void testDictionaryError_419() throws Exception {
testFull(QueryType.SQL,
- "select c_address from dfs.`/tmp/customer_snappyimpala_drill_419.parquet`", "", 1, 1, 150000, false);
+ "select c_address from dfs.`tmp/customer_snappyimpala_drill_419.parquet`", "", 1, 1, 150000, false);
}
@Test
@@ -234,14 +232,14 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
public void testNonExistentColumnLargeFile() throws Exception {
testFull(QueryType.SQL,
- "select non_existent_column, non_existent_col_2 from dfs.`/tmp/customer.dict.parquet`", "", 1, 1, 150000, false);
+ "select non_existent_column, non_existent_col_2 from dfs.`tmp/customer.dict.parquet`", "", 1, 1, 150000, false);
}
@Test
public void testNonExistentColumnsSomePresentColumnsLargeFile() throws Exception {
testFull(QueryType.SQL,
- "select cust_key, address, non_existent_column, non_existent_col_2 from dfs.`/tmp/customer.dict.parquet`",
+ "select cust_key, address, non_existent_column, non_existent_col_2 from dfs.`tmp/customer.dict.parquet`",
"", 1, 1, 150000, false);
}
@@ -249,19 +247,19 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
@Test
public void testTPCHPerformace_SF1() throws Exception {
testFull(QueryType.SQL,
- "select * from dfs.`/tmp/orders_part-m-00001.parquet`", "", 1, 1, 150000, false);
+ "select * from dfs.`tmp/orders_part-m-00001.parquet`", "", 1, 1, 150000, false);
}
@Test
public void testLocalDistributed() throws Exception {
- final String planName = "/parquet/parquet_scan_union_screen_physical.json";
+ final String planName = "parquet/parquet_scan_union_screen_physical.json";
testParquetFullEngineLocalTextDistributed(planName, fileName, 1, numberRowGroups, recordsPerRowGroup);
}
@Test
@Ignore
public void testRemoteDistributed() throws Exception {
- final String planName = "/parquet/parquet_scan_union_screen_physical.json";
+ final String planName = "parquet/parquet_scan_union_screen_physical.json";
testParquetFullEngineRemote(planName, fileName, 1, numberRowGroups, recordsPerRowGroup);
}
@@ -269,7 +267,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
public void testParquetFullEngineLocalPath(String planFileName, String filename,
int numberOfTimesRead /* specified in json plan */,
int numberOfRowGroups, int recordsPerRowGroup) throws Exception {
- testParquetFullEngineLocalText(Files.toString(FileUtils.getResourceAsFile(planFileName), Charsets.UTF_8), filename,
+ testParquetFullEngineLocalText(Files.toString(DrillFileUtils.getResourceAsFile(planFileName), Charsets.UTF_8), filename,
numberOfTimesRead, numberOfRowGroups, recordsPerRowGroup, true);
}
@@ -302,7 +300,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
public void testParquetFullEngineLocalTextDistributed(String planName, String filename,
int numberOfTimesRead /* specified in json plan */,
int numberOfRowGroups, int recordsPerRowGroup) throws Exception {
- String planText = Files.toString(FileUtils.getResourceAsFile(planName), Charsets.UTF_8);
+ String planText = Files.toString(DrillFileUtils.getResourceAsFile(planName), Charsets.UTF_8);
testFull(QueryType.PHYSICAL, planText, filename, numberOfTimesRead, numberOfRowGroups, recordsPerRowGroup, true);
}
@@ -330,7 +328,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
TestFileGenerator.populateFieldInfoMap(props);
final ParquetResultListener resultListener =
new ParquetResultListener(getAllocator(), props, numberOfTimesRead, true);
- testWithListener(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8), resultListener);
+ testWithListener(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile(plan), Charsets.UTF_8), resultListener);
resultListener.getResults();
}
@@ -399,7 +397,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props = new ParquetTestProperties(2, 300, DEFAULT_BYTES_PER_PAGE, fields);
populateFieldInfoMap(props);
- testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+ testParquetFullEngineEventBased(true, "parquet/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
}
// TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was
@@ -411,7 +409,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
ParquetTestProperties props = new ParquetTestProperties(1, 1500000, DEFAULT_BYTES_PER_PAGE, fields);
Object[] boolVals = {true, null, null};
props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
- testParquetFullEngineEventBased(false, "/parquet/parquet_nullable.json", "/tmp/nullable_test.parquet", 1, props);
+ testParquetFullEngineEventBased(false, "parquet/parquet_nullable.json", "/tmp/nullable_test.parquet", 1, props);
}
@@ -429,14 +427,14 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
byte[] val4 = { 'l','o','n','g','e','r',' ','s','t','r','i','n','g'};
Object[] byteArrayVals = { val, val2, val4};
props.fields.put("a", new FieldInfo("boolean", "a", 1, byteArrayVals, TypeProtos.MinorType.BIT, props));
- testParquetFullEngineEventBased(false, "/parquet/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props);
+ testParquetFullEngineEventBased(false, "parquet/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props);
HashMap<String, FieldInfo> fields2 = new HashMap<>();
// pass strings instead of byte arrays
Object[] textVals = { new org.apache.drill.exec.util.Text("b"), new org.apache.drill.exec.util.Text("b2"),
new org.apache.drill.exec.util.Text("b3") };
ParquetTestProperties props2 = new ParquetTestProperties(1, 30000, DEFAULT_BYTES_PER_PAGE, fields2);
props2.fields.put("a", new FieldInfo("boolean", "a", 1, textVals, TypeProtos.MinorType.BIT, props2));
- testParquetFullEngineEventBased(false, "/parquet/parquet_scan_screen_read_entry_replace.json",
+ testParquetFullEngineEventBased(false, "parquet/parquet_scan_screen_read_entry_replace.json",
"\"/tmp/varLen.parquet/a\"", "unused", 1, props2);
}
@@ -449,7 +447,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
// actually include null values
Object[] valuesWithNull = {new Text(""), new Text("longer string"), null};
props3.fields.put("a", new FieldInfo("boolean", "a", 1, valuesWithNull, TypeProtos.MinorType.BIT, props3));
- testParquetFullEngineEventBased(false, "/parquet/parquet_scan_screen_read_entry_replace.json",
+ testParquetFullEngineEventBased(false, "parquet/parquet_scan_screen_read_entry_replace.json",
"\"/tmp/nullable_with_nulls.parquet\"", "unused", 1, props3);
}
@@ -464,7 +462,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
props.fields.put("n_nationkey", null);
props.fields.put("n_regionkey", null);
props.fields.put("n_comment", null);
- testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json",
+ testParquetFullEngineEventBased(false, false, "parquet/parquet_scan_screen_read_entry_replace.json",
"\"/tmp/nation_dictionary_fail.parquet\"", "unused", 1, props, QueryType.LOGICAL);
fields = new HashMap<>();
@@ -477,7 +475,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
props.fields.put("gender_male", null);
props.fields.put("height", null);
props.fields.put("hair_thickness", null);
- testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json",
+ testParquetFullEngineEventBased(false, false, "parquet/parquet_scan_screen_read_entry_replace.json",
"\"/tmp/employees_5_16_14.parquet\"", "unused", 1, props, QueryType.LOGICAL);
}
@@ -495,17 +493,17 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
readEntries += ",";
}
}
- testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
+ testParquetFullEngineEventBased(true, "parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
"/tmp/test.parquet", i, props);
}
@Test
public void testReadError_Drill_901() throws Exception {
- // select cast( L_COMMENT as varchar) from dfs_test.`/tmp/drilltest/employee_parquet`
+ // select cast( L_COMMENT as varchar) from dfs.`/tmp/drilltest/employee_parquet`
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props = new ParquetTestProperties(1, 60175, DEFAULT_BYTES_PER_PAGE, fields);
- testParquetFullEngineEventBased(false, false, "/parquet/par_writer_test.json", null,
+ testParquetFullEngineEventBased(false, false, "parquet/par_writer_test.json", null,
"unused, no file is generated", 1, props, QueryType.PHYSICAL);
}
@@ -516,7 +514,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props = new ParquetTestProperties(1, 150000, DEFAULT_BYTES_PER_PAGE, fields);
String readEntries = "\"/tmp/customer_nonull.parquet\"";
- testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
+ testParquetFullEngineEventBased(false, false, "parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
"unused, no file is generated", 1, props, QueryType.LOGICAL);
}
@@ -527,7 +525,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
ParquetTestProperties props = new ParquetTestProperties(1, 150000, DEFAULT_BYTES_PER_PAGE, fields);
TestFileGenerator.populateDrill_418_fields(props);
String readEntries = "\"/tmp/customer.plain.parquet\"";
- testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
+ testParquetFullEngineEventBased(false, false, "parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
"unused, no file is generated", 1, props, QueryType.LOGICAL);
}
@@ -539,21 +537,21 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
ParquetTestProperties props = new ParquetTestProperties(1, 1500000, DEFAULT_BYTES_PER_PAGE, fields);
TestFileGenerator.populatePigTPCHCustomerFields(props);
String readEntries = "\"/tmp/tpc-h/customer\"";
- testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
+ testParquetFullEngineEventBased(false, false, "parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
"unused, no file is generated", 1, props, QueryType.LOGICAL);
fields = new HashMap<>();
props = new ParquetTestProperties(1, 100000, DEFAULT_BYTES_PER_PAGE, fields);
TestFileGenerator.populatePigTPCHSupplierFields(props);
- readEntries = "\"/tmp/tpc-h/supplier\"";
- testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
+ readEntries = "\"tmp/tpc-h/supplier\"";
+ testParquetFullEngineEventBased(false, false, "parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
"unused, no file is generated", 1, props, QueryType.LOGICAL);
}
@Test
public void test958_sql() throws Exception {
// testFull(QueryType.SQL, "select ss_ext_sales_price from dfs.`/tmp/store_sales`", "", 1, 1, 30000000, false);
- testFull(QueryType.SQL, "select * from dfs.`/tmp/store_sales`", "", 1, 1, 30000000, false);
+ testFull(QueryType.SQL, "select * from dfs.`tmp/store_sales`", "", 1, 1, 30000000, false);
}
@Test
@@ -561,8 +559,8 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props = new ParquetTestProperties(1, 2880404, DEFAULT_BYTES_PER_PAGE, fields);
TestFileGenerator.populatePigTPCHCustomerFields(props);
- String readEntries = "\"/tmp/store_sales\"";
- testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
+ String readEntries = "\"tmp/store_sales\"";
+ testParquetFullEngineEventBased(false, false, "parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
"unused, no file is generated", 1, props, QueryType.LOGICAL);
}
@@ -571,7 +569,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props = new ParquetTestProperties(2, 300, DEFAULT_BYTES_PER_PAGE, fields);
populateFieldInfoMap(props);
- testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+ testParquetFullEngineEventBased(true, "parquet/parquet_scan_screen.json", "tmp/test.parquet", 1, props);
}
@@ -681,7 +679,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
final ParquetResultListener resultListener = new ParquetResultListener(getAllocator(), props, numberOfTimesRead, testValues);
final long C = System.nanoTime();
- String planText = Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8);
+ String planText = Files.toString(DrillFileUtils.getResourceAsFile(plan), Charsets.UTF_8);
// substitute in the string for the read entries, allows reuse of the plan file for several tests
if (readEntries != null) {
planText = planText.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries);
@@ -694,7 +692,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
@Test
public void testLimit() throws Exception {
- List<QueryDataBatch> results = testSqlWithResults("SELECT * FROM cp.`/parquet/tpch/nation/01.parquet` LIMIT 1");
+ List<QueryDataBatch> results = testSqlWithResults("SELECT * FROM cp.`parquet/tpch/nation/01.parquet` LIMIT 1");
int recordsInOutput = 0;
for (QueryDataBatch batch : results) {
@@ -707,7 +705,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
@Test
public void testLimitBeyondRowCount() throws Exception {
- List<QueryDataBatch> results = testSqlWithResults("SELECT * FROM cp.`/parquet/tpch/nation/01.parquet` LIMIT 100");
+ List<QueryDataBatch> results = testSqlWithResults("SELECT * FROM cp.`parquet/tpch/nation/01.parquet` LIMIT 100");
int recordsInOutput = 0;
for (QueryDataBatch batch : results) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFixedlenDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFixedlenDecimal.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFixedlenDecimal.java
index 124a8c8..7635551 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFixedlenDecimal.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFixedlenDecimal.java
@@ -17,9 +17,8 @@
*/
package org.apache.drill.exec.store.parquet;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.apache.drill.exec.proto.UserBitShared;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -34,9 +33,8 @@ public class TestFixedlenDecimal extends BaseTestQuery {
@Test
public void testNullCount() throws Exception {
- String query = String.format("select count(*) as c from %s where department_id is null", DATAFILE);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select count(*) as c from %s where department_id is null", DATAFILE)
.unOrdered()
.baselineColumns("c")
.baselineValues(1L)
@@ -46,9 +44,8 @@ public class TestFixedlenDecimal extends BaseTestQuery {
@Test
public void testNotNullCount() throws Exception {
- String query = String.format("select count(*) as c from %s where department_id is not null", DATAFILE);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select count(*) as c from %s where department_id is not null", DATAFILE)
.unOrdered()
.baselineColumns("c")
.baselineValues(106L)
@@ -58,9 +55,8 @@ public class TestFixedlenDecimal extends BaseTestQuery {
@Test
public void testSimpleQueryWithCast() throws Exception {
- String query = String.format("select cast(department_id as bigint) as c from %s where cast(employee_id as decimal) = 170", DATAFILE);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select cast(department_id as bigint) as c from %s where cast(employee_id as decimal) = 170", DATAFILE)
.unOrdered()
.baselineColumns("c")
.baselineValues(80L)
@@ -70,9 +66,8 @@ public class TestFixedlenDecimal extends BaseTestQuery {
@Test
public void testSimpleQueryDrill4704Fix() throws Exception {
- String query = String.format("select cast(department_id as bigint) as c from %s where employee_id = 170", DATAFILE);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select cast(department_id as bigint) as c from %s where employee_id = 170", DATAFILE)
.unOrdered()
.baselineColumns("c")
.baselineValues(80L)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
index 301374f..7c148cb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
public class TestParquetComplex extends BaseTestQuery {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index cb5000f..85501d1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -15,60 +15,87 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.drill.exec.store.parquet;
+import org.apache.commons.io.FileUtils;
import org.apache.drill.PlanTestBase;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.proto.BitControl;
-import org.apache.drill.test.OperatorFixture;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
-import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import static org.junit.Assert.assertEquals;
public class TestParquetFilterPushDown extends PlanTestBase {
-
- private static final String WORKING_PATH = TestTools.getWorkingPath();
- private static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
+ private static final String CTAS_TABLE = "order_ctas";
private static FragmentContext fragContext;
- private static Configuration conf;
private static FileSystem fs;
@BeforeClass
public static void initFSAndCreateFragContext() throws Exception {
+ fs = getLocalFileSystem();
fragContext = new FragmentContext(bits[0].getContext(),
BitControl.PlanFragment.getDefaultInstance(), null, bits[0].getContext().getFunctionImplementationRegistry());
- fs = getLocalFileSystem();
- conf = fs.getConf();
+ dirTestWatcher.copyResourceToRoot(Paths.get("parquetFilterPush"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("parquet", "multirowgroup.parquet"));
}
@AfterClass
- public static void close() throws Exception {
- fragContext.close();
+ public static void teardown() throws IOException {
fs.close();
}
+ @Rule
+ public final TestWatcher ctasWatcher = new TestWatcher() {
+ @Override
+ protected void failed(Throwable e, Description description) {
+ deleteCtasTable();
+ }
+
+ @Override
+ protected void starting(Description description) {
+ deleteCtasTable();
+ }
+
+ @Override
+ protected void finished(Description description) {
+ deleteCtasTable();
+ }
+
+ private void deleteCtasTable() {
+ FileUtils.deleteQuietly(new File(dirTestWatcher.getDfsTestTmpDir(), CTAS_TABLE));
+ }
+ };
+
@Test
// Test filter evaluation directly without go through SQL queries.
public void testIntPredicateWithEval() throws Exception {
// intTbl.parquet has only one int column
// intCol : [0, 100].
- final String filePath = String.format("%s/parquetFilterPush/intTbl/intTbl.parquet", TEST_RES_PATH);
- ParquetMetadata footer = getParquetMetaData(filePath);
+ final File file = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(Paths.get("parquetFilterPush", "intTbl", "intTbl.parquet"))
+ .toFile();
+ ParquetMetadata footer = getParquetMetaData(file);
testParquetRowGroupFilterEval(footer, "intCol = 100", false);
testParquetRowGroupFilterEval(footer, "intCol = 0", false);
@@ -143,8 +170,11 @@ public class TestParquetFilterPushDown extends PlanTestBase {
public void testIntPredicateAgainstAllNullColWithEval() throws Exception {
// intAllNull.parquet has only one int column with all values being NULL.
// column values statistics: num_nulls: 25, min/max is not defined
- final String filePath = String.format("%s/parquetFilterPush/intTbl/intAllNull.parquet", TEST_RES_PATH);
- ParquetMetadata footer = getParquetMetaData(filePath);
+ final File file = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(Paths.get("parquetFilterPush", "intTbl", "intAllNull.parquet"))
+ .toFile();
+ ParquetMetadata footer = getParquetMetaData(file);
testParquetRowGroupFilterEval(footer, "intCol = 100", true);
testParquetRowGroupFilterEval(footer, "intCol = 0", true);
@@ -162,8 +192,11 @@ public class TestParquetFilterPushDown extends PlanTestBase {
// The parquet file is created on drill 1.8.0 with DRILL CTAS:
// create table dfs.tmp.`dateTblCorrupted/t1` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and date '1992-01-03';
- final String filePath = String.format("%s/parquetFilterPush/dateTblCorrupted/t1/0_0_0.parquet", TEST_RES_PATH);
- ParquetMetadata footer = getParquetMetaData(filePath);
+ final File file = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(Paths.get("parquetFilterPush", "dateTblCorrupted", "t1", "0_0_0.parquet"))
+ .toFile();
+ ParquetMetadata footer = getParquetMetaData(file);
testDatePredicateAgainstDrillCTASHelper(footer);
}
@@ -173,13 +206,15 @@ public class TestParquetFilterPushDown extends PlanTestBase {
// The parquet file is created on drill 1.9.0-SNAPSHOT (commit id:03e8f9f3e01c56a9411bb4333e4851c92db6e410) with DRILL CTAS:
// create table dfs.tmp.`dateTbl1_9/t1` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and date '1992-01-03';
- final String filePath = String.format("%s/parquetFilterPush/dateTbl1_9/t1/0_0_0.parquet", TEST_RES_PATH);
- ParquetMetadata footer = getParquetMetaData(filePath);
+ final File file = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(Paths.get("parquetFilterPush", "dateTbl1_9", "t1", "0_0_0.parquet"))
+ .toFile();
+ ParquetMetadata footer = getParquetMetaData(file);
testDatePredicateAgainstDrillCTASHelper(footer);
}
-
private void testDatePredicateAgainstDrillCTASHelper(ParquetMetadata footer) throws Exception{
testParquetRowGroupFilterEval(footer, "o_orderdate = cast('1992-01-01' as date)", false);
testParquetRowGroupFilterEval(footer, "o_orderdate = cast('1991-12-31' as date)", true);
@@ -202,8 +237,11 @@ public class TestParquetFilterPushDown extends PlanTestBase {
public void testTimeStampPredicateWithEval() throws Exception {
// Table dateTblCorrupted is created by CTAS in drill 1.8.0.
// create table dfs.tmp.`tsTbl/t1` as select DATE_ADD(cast(o_orderdate as date), INTERVAL '0 10:20:30' DAY TO SECOND) as o_ordertimestamp from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and date '1992-01-03';
- final String filePath = String.format("%s/parquetFilterPush/tsTbl/t1/0_0_0.parquet", TEST_RES_PATH);
- ParquetMetadata footer = getParquetMetaData(filePath);
+ final File file = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(Paths.get("parquetFilterPush", "tsTbl", "t1", "0_0_0.parquet"))
+ .toFile();
+ ParquetMetadata footer = getParquetMetaData(file);
testParquetRowGroupFilterEval(footer, "o_ordertimestamp = cast('1992-01-01 10:20:30' as timestamp)", false);
testParquetRowGroupFilterEval(footer, "o_ordertimestamp = cast('1992-01-01 10:20:29' as timestamp)", true);
@@ -226,97 +264,82 @@ public class TestParquetFilterPushDown extends PlanTestBase {
@Test
// Test against parquet files from Drill CTAS post 1.8.0 release.
public void testDatePredicateAgaistDrillCTASPost1_8() throws Exception {
- String tableName = "order_ctas";
-
- try {
- deleteTableIfExists(tableName);
-
- test("use dfs_test.tmp");
- test(String.format("create table `%s/t1` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and date '1992-01-03'", tableName));
- test(String.format("create table `%s/t2` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-04' and date '1992-01-06'", tableName));
- test(String.format("create table `%s/t3` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-07' and date '1992-01-09'", tableName));
-
- final String query1 = "select o_orderdate from dfs_test.tmp.order_ctas where o_orderdate = date '1992-01-01'";
- testParquetFilterPD(query1, 9, 1, false);
-
- final String query2 = "select o_orderdate from dfs_test.tmp.order_ctas where o_orderdate < date '1992-01-01'";
- testParquetFilterPD(query2, 0, 1, false);
-
- final String query3 = "select o_orderdate from dfs_test.tmp.order_ctas where o_orderdate between date '1992-01-01' and date '1992-01-03'";
- testParquetFilterPD(query3, 22, 1, false);
-
- final String query4 = "select o_orderdate from dfs_test.tmp.order_ctas where o_orderdate between date '1992-01-01' and date '1992-01-04'";
- testParquetFilterPD(query4, 33, 2, false);
-
- final String query5 = "select o_orderdate from dfs_test.tmp.order_ctas where o_orderdate between date '1992-01-01' and date '1992-01-06'";
- testParquetFilterPD(query5, 49, 2, false);
-
- final String query6 = "select o_orderdate from dfs_test.tmp.order_ctas where o_orderdate > date '1992-01-10'";
- testParquetFilterPD(query6, 0, 1, false);
-
- // Test parquet files with metadata cache files available.
- // Now, create parquet metadata cache files, and run the above queries again. Flag "usedMetadataFile" should be true.
- test(String.format("refresh table metadata %s", tableName));
+ test("use dfs.tmp");
+ test("create table `%s/t1` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and " +
+ "date '1992-01-03'", CTAS_TABLE);
+ test("create table `%s/t2` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-04' and " +
+ "date '1992-01-06'", CTAS_TABLE);
+ test("create table `%s/t3` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-07' and " +
+ "date '1992-01-09'", CTAS_TABLE);
+
+ final String query1 = "select o_orderdate from dfs.tmp.order_ctas where o_orderdate = date '1992-01-01'";
+ testParquetFilterPD(query1, 9, 1, false);
- testParquetFilterPD(query1, 9, 1, true);
+ final String query2 = "select o_orderdate from dfs.tmp.order_ctas where o_orderdate < date '1992-01-01'";
+ testParquetFilterPD(query2, 0, 1, false);
- testParquetFilterPD(query2, 0, 1, true);
+ final String query3 = "select o_orderdate from dfs.tmp.order_ctas where o_orderdate between date '1992-01-01' and date '1992-01-03'";
+ testParquetFilterPD(query3, 22, 1, false);
- testParquetFilterPD(query3, 22, 1, true);
+ final String query4 = "select o_orderdate from dfs.tmp.order_ctas where o_orderdate between date '1992-01-01' and date '1992-01-04'";
+ testParquetFilterPD(query4, 33, 2, false);
- testParquetFilterPD(query4, 33, 2, true);
+ final String query5 = "select o_orderdate from dfs.tmp.order_ctas where o_orderdate between date '1992-01-01' and date '1992-01-06'";
+ testParquetFilterPD(query5, 49, 2, false);
- testParquetFilterPD(query5, 49, 2, true);
+ final String query6 = "select o_orderdate from dfs.tmp.order_ctas where o_orderdate > date '1992-01-10'";
+ testParquetFilterPD(query6, 0, 1, false);
- testParquetFilterPD(query6, 0, 1, true);
- } finally {
- deleteTableIfExists(tableName);
- }
+ // Test parquet files with metadata cache files available.
+ // Now, create parquet metadata cache files, and run the above queries again. Flag "usedMetadataFile" should be true.
+ test(String.format("refresh table metadata %s", CTAS_TABLE));
+ testParquetFilterPD(query1, 9, 1, true);
+ testParquetFilterPD(query2, 0, 1, true);
+ testParquetFilterPD(query3, 22, 1, true);
+ testParquetFilterPD(query4, 33, 2, true);
+ testParquetFilterPD(query5, 49, 2, true);
+ testParquetFilterPD(query6, 0, 1, true);
}
@Test
public void testParquetFilterPDOptionsDisabled() throws Exception {
- final String tableName = "order_ctas";
-
try {
- deleteTableIfExists(tableName);
-
- test("alter session set `" + PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY + "` = false");
+ test("alter session set `%s` = false", PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY);
- test("use dfs_test.tmp");
- test(String.format("create table `%s/t1` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and date '1992-01-03'", tableName));
- test(String.format("create table `%s/t2` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-04' and date '1992-01-06'", tableName));
- test(String.format("create table `%s/t3` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-07' and date '1992-01-09'", tableName));
+ test("use dfs.tmp");
+ test("create table `%s/t1` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and " +
+ "date '1992-01-03'", CTAS_TABLE);
+ test("create table `%s/t2` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-04' and " +
+ "date '1992-01-06'", CTAS_TABLE);
+ test("create table `%s/t3` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-07' and " +
+ "date '1992-01-09'", CTAS_TABLE);
- final String query1 = "select o_orderdate from dfs_test.tmp.order_ctas where o_orderdate = date '1992-01-01'";
+ final String query1 = "select o_orderdate from dfs.tmp.order_ctas where o_orderdate = date '1992-01-01'";
testParquetFilterPD(query1, 9, 3, false);
} finally {
resetSessionOption(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY);
- deleteTableIfExists(tableName);
}
}
@Test
public void testParquetFilterPDOptionsThreshold() throws Exception {
- final String tableName = "order_ctas";
-
try {
- deleteTableIfExists(tableName);
-
test("alter session set `" + PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY + "` = 2 ");
- test("use dfs_test.tmp");
- test(String.format("create table `%s/t1` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and date '1992-01-03'", tableName));
- test(String.format("create table `%s/t2` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-04' and date '1992-01-06'", tableName));
- test(String.format("create table `%s/t3` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-07' and date '1992-01-09'", tableName));
+ test("use dfs.tmp");
+ test("create table `%s/t1` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-01' and " +
+ "date '1992-01-03'", CTAS_TABLE);
+ test("create table `%s/t2` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-04' and " +
+ "date '1992-01-06'", CTAS_TABLE);
+ test("create table `%s/t3` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-07' and " +
+ "date '1992-01-09'", CTAS_TABLE);
- final String query1 = "select o_orderdate from dfs_test.tmp.order_ctas where o_orderdate = date '1992-01-01'";
+ final String query1 = "select o_orderdate from dfs.tmp.order_ctas where o_orderdate = date '1992-01-01'";
testParquetFilterPD(query1, 9, 3, false);
} finally {
resetSessionOption(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY);
- deleteTableIfExists(tableName);
}
}
@@ -328,22 +351,22 @@ public class TestParquetFilterPushDown extends PlanTestBase {
// create table dfs.tmp.`dateTblCorrupted/t2` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-04' and date '1992-01-06';
// create table dfs.tmp.`dateTblCorrupted/t3` as select cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-07' and date '1992-01-09';
- final String query1 = String.format("select o_orderdate from dfs_test.`%s/parquetFilterPush/dateTblCorrupted` where o_orderdate = date '1992-01-01'", TEST_RES_PATH);
+ final String query1 = "select o_orderdate from dfs.`parquetFilterPush/dateTblCorrupted` where o_orderdate = date '1992-01-01'";
testParquetFilterPD(query1, 9, 1, false);
- final String query2 = String.format("select o_orderdate from dfs_test.`%s/parquetFilterPush/dateTblCorrupted` where o_orderdate < date '1992-01-01'", TEST_RES_PATH);
+ final String query2 = "select o_orderdate from dfs.`parquetFilterPush/dateTblCorrupted` where o_orderdate < date '1992-01-01'";
testParquetFilterPD(query2, 0, 1, false);
- final String query3 = String.format("select o_orderdate from dfs_test.`%s/parquetFilterPush/dateTblCorrupted` where o_orderdate between date '1992-01-01' and date '1992-01-03'", TEST_RES_PATH);
+ final String query3 = "select o_orderdate from dfs.`parquetFilterPush/dateTblCorrupted` where o_orderdate between date '1992-01-01' and date '1992-01-03'";
testParquetFilterPD(query3, 22, 1, false);
- final String query4 = String.format("select o_orderdate from dfs_test.`%s/parquetFilterPush/dateTblCorrupted` where o_orderdate between date '1992-01-01' and date '1992-01-04'", TEST_RES_PATH);
+ final String query4 = "select o_orderdate from dfs.`parquetFilterPush/dateTblCorrupted` where o_orderdate between date '1992-01-01' and date '1992-01-04'";
testParquetFilterPD(query4, 33, 2, false);
- final String query5 = String.format("select o_orderdate from dfs_test.`%s/parquetFilterPush/dateTblCorrupted` where o_orderdate between date '1992-01-01' and date '1992-01-06'", TEST_RES_PATH);
+ final String query5 = "select o_orderdate from dfs.`parquetFilterPush/dateTblCorrupted` where o_orderdate between date '1992-01-01' and date '1992-01-06'";
testParquetFilterPD(query5, 49, 2, false);
- final String query6 = String.format("select o_orderdate from dfs_test.`%s/parquetFilterPush/dateTblCorrupted` where o_orderdate > date '1992-01-10'", TEST_RES_PATH);
+ final String query6 = "select o_orderdate from dfs.`parquetFilterPush/dateTblCorrupted` where o_orderdate > date '1992-01-10'";
testParquetFilterPD(query6, 0, 1, false);
}
@@ -355,18 +378,18 @@ public class TestParquetFilterPushDown extends PlanTestBase {
// create table dfs.tmp.`tsTbl/t2` as select DATE_ADD(cast(o_orderdate as date), INTERVAL '0 10:20:30' DAY TO SECOND) as o_ordertimestamp from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-04' and date '1992-01-06';
// create table dfs.tmp.`tsTbl/t3` as select DATE_ADD(cast(o_orderdate as date), INTERVAL '0 10:20:30' DAY TO SECOND) as o_ordertimestamp from cp.`tpch/orders.parquet` where o_orderdate between date '1992-01-07' and date '1992-01-09';
- final String query1 = String.format("select o_ordertimestamp from dfs_test.`%s/parquetFilterPush/tsTbl` where o_ordertimestamp = timestamp '1992-01-01 10:20:30'", TEST_RES_PATH);
+ final String query1 = "select o_ordertimestamp from dfs.`parquetFilterPush/tsTbl` where o_ordertimestamp = timestamp '1992-01-01 10:20:30'";
testParquetFilterPD(query1, 9, 1, false);
- final String query2 = String.format("select o_ordertimestamp from dfs_test.`%s/parquetFilterPush/tsTbl` where o_ordertimestamp < timestamp '1992-01-01 10:20:30'", TEST_RES_PATH);
+ final String query2 = "select o_ordertimestamp from dfs.`parquetFilterPush/tsTbl` where o_ordertimestamp < timestamp '1992-01-01 10:20:30'";
testParquetFilterPD(query2, 0, 1, false);
- final String query3 = String.format("select o_ordertimestamp from dfs_test.`%s/parquetFilterPush/tsTbl` where o_ordertimestamp between timestamp '1992-01-01 00:00:00' and timestamp '1992-01-06 10:20:30'", TEST_RES_PATH);
+ final String query3 = "select o_ordertimestamp from dfs.`parquetFilterPush/tsTbl` where o_ordertimestamp between timestamp '1992-01-01 00:00:00' and timestamp '1992-01-06 10:20:30'";
testParquetFilterPD(query3, 49, 2, false);
}
@Test // DRILL-5359
- public void testFilterWithItemFlatten() throws Exception {
+ public void testFilterWithItemFlatten() throws Exception {
final String sql = "select n_regionkey\n"
+ "from (select n_regionkey, \n"
+ " flatten(nation.cities) as cities \n"
@@ -384,7 +407,7 @@ public class TestParquetFilterPushDown extends PlanTestBase {
public void testMultiRowGroup() throws Exception {
// multirowgroup is a parquet file with 2 rowgroups inside. One with a = 1 and the other with a = 2;
// FilterPushDown should be able to remove the rowgroup with a = 1 from the scan operator.
- final String sql = String.format("select * from dfs_test.`%s/parquet/multirowgroup.parquet` where a > 1", TEST_RES_PATH);
+ final String sql = String.format("select * from dfs.`parquet/multirowgroup.parquet` where a > 1");
final String[] expectedPlan = {"numRowGroups=1"};
final String[] excludedPlan = {};
PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
@@ -416,19 +439,7 @@ public class TestParquetFilterPushDown extends PlanTestBase {
Assert.assertEquals(canDropExpected, canDrop);
}
- private ParquetMetadata getParquetMetaData(String filePathStr) throws IOException{
- return ParquetFileReader.readFooter(new Configuration(conf), new Path(filePathStr));
+ private ParquetMetadata getParquetMetaData(File file) throws IOException{
+ return ParquetFileReader.readFooter(new Configuration(fs.getConf()), new Path(file.toURI()));
}
-
- private static void deleteTableIfExists(String tableName) {
- try {
- Path path = new Path(getDfsTestTmpSchemaLocation(), tableName);
- if (fs.exists(path)) {
- fs.delete(path, true);
- }
- } catch (Exception e) {
- // ignore exceptions.
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetGroupScan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetGroupScan.java
index 21d6285..d7ccea4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetGroupScan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetGroupScan.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.junit.Test;
@@ -28,27 +28,27 @@ public class TestParquetGroupScan extends BaseTestQuery {
private void prepareTables(final String tableName, boolean refreshMetadata) throws Exception {
// first create some parquet subfolders
- testNoResult("CREATE TABLE dfs_test.tmp.`%s` AS SELECT employee_id FROM cp.`employee.json` LIMIT 1", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/501` AS SELECT employee_id FROM cp.`employee.json` LIMIT 2", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/502` AS SELECT employee_id FROM cp.`employee.json` LIMIT 4", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/503` AS SELECT employee_id FROM cp.`employee.json` LIMIT 8", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/504` AS SELECT employee_id FROM cp.`employee.json` LIMIT 16", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/505` AS SELECT employee_id FROM cp.`employee.json` LIMIT 32", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/60` AS SELECT employee_id FROM cp.`employee.json` LIMIT 64", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/602` AS SELECT employee_id FROM cp.`employee.json` LIMIT 128", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/6031` AS SELECT employee_id FROM cp.`employee.json` LIMIT 256", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/6032` AS SELECT employee_id FROM cp.`employee.json` LIMIT 512", tableName);
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/6033` AS SELECT employee_id FROM cp.`employee.json` LIMIT 1024", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s` AS SELECT employee_id FROM cp.`employee.json` LIMIT 1", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/501` AS SELECT employee_id FROM cp.`employee.json` LIMIT 2", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/502` AS SELECT employee_id FROM cp.`employee.json` LIMIT 4", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/503` AS SELECT employee_id FROM cp.`employee.json` LIMIT 8", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/504` AS SELECT employee_id FROM cp.`employee.json` LIMIT 16", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/505` AS SELECT employee_id FROM cp.`employee.json` LIMIT 32", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/60` AS SELECT employee_id FROM cp.`employee.json` LIMIT 64", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/602` AS SELECT employee_id FROM cp.`employee.json` LIMIT 128", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/6031` AS SELECT employee_id FROM cp.`employee.json` LIMIT 256", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/6032` AS SELECT employee_id FROM cp.`employee.json` LIMIT 512", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/6033` AS SELECT employee_id FROM cp.`employee.json` LIMIT 1024", tableName);
// we need an empty subfolder `4376/20160401`
// to do this we first create a table inside that subfolder
- testNoResult("CREATE TABLE dfs_test.tmp.`%s/6041/a` AS SELECT * FROM cp.`employee.json` LIMIT 1", tableName);
+ testNoResult("CREATE TABLE dfs.tmp.`%s/6041/a` AS SELECT * FROM cp.`employee.json` LIMIT 1", tableName);
// then we delete the table, leaving the parent subfolder empty
- testNoResult("DROP TABLE dfs_test.tmp.`%s/6041/a`", tableName);
+ testNoResult("DROP TABLE dfs.tmp.`%s/6041/a`", tableName);
if (refreshMetadata) {
// build the metadata cache file
- testNoResult("REFRESH TABLE METADATA dfs_test.tmp.`%s`", tableName);
+ testNoResult("REFRESH TABLE METADATA dfs.tmp.`%s`", tableName);
}
}
@@ -57,7 +57,7 @@ public class TestParquetGroupScan extends BaseTestQuery {
prepareTables("4376_1", true);
testBuilder()
- .sqlQuery("SELECT COUNT(*) AS `count` FROM dfs_test.tmp.`4376_1/60*`")
+ .sqlQuery("SELECT COUNT(*) AS `count` FROM dfs.tmp.`4376_1/60*`")
.ordered()
.baselineColumns("count").baselineValues(1984L)
.go();
@@ -68,7 +68,7 @@ public class TestParquetGroupScan extends BaseTestQuery {
prepareTables("4376_2", true);
try {
- runSQL("SELECT COUNT(*) AS `count` FROM dfs_test.tmp.`4376_2/604*`");
+ runSQL("SELECT COUNT(*) AS `count` FROM dfs.tmp.`4376_2/604*`");
fail("Query should've failed!");
} catch (UserRemoteException uex) {
final String expectedMsg = "The table you tried to query is empty";
@@ -82,10 +82,10 @@ public class TestParquetGroupScan extends BaseTestQuery {
prepareTables("4376_3", false);
try {
- runSQL("SELECT COUNT(*) AS `count` FROM dfs_test.tmp.`4376_3/604*`");
+ runSQL("SELECT COUNT(*) AS `count` FROM dfs.tmp.`4376_3/604*`");
fail("Query should've failed!");
} catch (UserRemoteException uex) {
- final String expectedMsg = "Table 'dfs_test.tmp.4376_3/604*' not found";
+ final String expectedMsg = "Table 'dfs.tmp.4376_3/604*' not found";
assertTrue(String.format("Error message should contain \"%s\" but was instead \"%s\"", expectedMsg,
uex.getMessage()), uex.getMessage().contains(expectedMsg));
}
@@ -96,7 +96,7 @@ public class TestParquetGroupScan extends BaseTestQuery {
prepareTables("4376_4", true);
try {
- runSQL("SELECT COUNT(*) AS `count` FROM dfs_test.tmp.`4376_4/6041`");
+ runSQL("SELECT COUNT(*) AS `count` FROM dfs.tmp.`4376_4/6041`");
fail("Query should've failed!");
} catch (UserRemoteException uex) {
final String expectedMsg = "The table you tried to query is empty";
@@ -109,10 +109,10 @@ public class TestParquetGroupScan extends BaseTestQuery {
public void testSelectEmptyNoCache() throws Exception {
prepareTables("4376_5", false);
try {
- runSQL("SELECT COUNT(*) AS `count` FROM dfs_test.tmp.`4376_5/6041`");
+ runSQL("SELECT COUNT(*) AS `count` FROM dfs.tmp.`4376_5/6041`");
fail("Query should've failed!");
} catch (UserRemoteException uex) {
- final String expectedMsg = "Table 'dfs_test.tmp.4376_5/6041' not found";
+ final String expectedMsg = "Table 'dfs.tmp.4376_5/6041' not found";
assertTrue(String.format("Error message should contain \"%s\" but was instead \"%s\"", expectedMsg,
uex.getMessage()), uex.getMessage().contains(expectedMsg));
}
[14/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java
index 3ef5651..38a01d5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestBroadcastExchange.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.util.List;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
@@ -50,9 +50,9 @@ public class TestBroadcastExchange extends PopUnitTestBase {
client.connect();
String physicalPlan = Files.toString(
- FileUtils.getResourceAsFile("/sender/broadcast_exchange.json"), Charsets.UTF_8)
- .replace("#{LEFT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString())
- .replace("#{RIGHT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString());
+ DrillFileUtils.getResourceAsFile("/sender/broadcast_exchange.json"), Charsets.UTF_8)
+ .replace("#{LEFT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString())
+ .replace("#{RIGHT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString());
List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, physicalPlan);
int count = 0;
for (QueryDataBatch b : results) {
@@ -78,7 +78,7 @@ public class TestBroadcastExchange extends PopUnitTestBase {
client.connect();
String physicalPlan = Files.toString(
- FileUtils.getResourceAsFile("/sender/broadcast_exchange_long_run.json"), Charsets.UTF_8);
+ DrillFileUtils.getResourceAsFile("/sender/broadcast_exchange_long_run.json"), Charsets.UTF_8);
List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, physicalPlan);
int count = 0;
for (QueryDataBatch b : results) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
index 3cd293e..11f739d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers;
@@ -62,9 +62,7 @@ import com.google.common.io.Files;
import mockit.Injectable;
-public class TestCastFunctions extends PopUnitTestBase{
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class);
-
+public class TestCastFunctions extends PopUnitTestBase {
@Test
// cast to bigint.
public void testCastBigInt(@Injectable final DrillbitContext bitContext,
@@ -73,7 +71,7 @@ public class TestCastFunctions extends PopUnitTestBase{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastBigInt.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastBigInt.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -111,7 +109,7 @@ public class TestCastFunctions extends PopUnitTestBase{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastInt.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastInt.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -147,7 +145,7 @@ public class TestCastFunctions extends PopUnitTestBase{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastFloat4.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastFloat4.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -184,7 +182,7 @@ public class TestCastFunctions extends PopUnitTestBase{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastFloat8.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastFloat8.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -221,7 +219,7 @@ public class TestCastFunctions extends PopUnitTestBase{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarChar.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastVarChar.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -257,7 +255,7 @@ public class TestCastFunctions extends PopUnitTestBase{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarBinary.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastVarBinary.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -294,7 +292,7 @@ public class TestCastFunctions extends PopUnitTestBase{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastNested.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastNested.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -332,7 +330,7 @@ public class TestCastFunctions extends PopUnitTestBase{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastNumException.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastNumException.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(CONFIG);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -360,7 +358,7 @@ public class TestCastFunctions extends PopUnitTestBase{
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"), Charsets.UTF_8).replace("#{TEST_FILE}", "/jsoninput/input1.json"));
+ Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"), Charsets.UTF_8).replace("#{TEST_FILE}", "/jsoninput/input1.json"));
final QueryDataBatch batch = results.get(0);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
index 8d904eb..5bf5711 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastVarCharToBigInt.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -50,7 +50,7 @@ public class TestCastVarCharToBigInt extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/functions/cast/test_cast_varchar_to_bigint.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/test_cast_varchar_to_bigint.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/scan_json_test_cast.json")
);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
index 55dc130..c1951a3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestConvertFunctions.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.physical.impl;
-import static org.apache.drill.TestBuilder.listOf;
-import static org.apache.drill.TestBuilder.mapOf;
+import static org.apache.drill.test.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -28,8 +28,8 @@ import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.QueryTestUtil;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.QueryTestUtil;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.ClassTransformer.ScalarReplacementOption;
@@ -60,8 +60,6 @@ import org.junit.experimental.categories.Category;
@Category(UnlikelyTest.class)
public class TestConvertFunctions extends BaseTestQuery {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestConvertFunctions.class);
-
private static final String CONVERSION_TEST_LOGICAL_PLAN = "functions/conv/conversionTestWithLogicalPlan.json";
private static final String CONVERSION_TEST_PHYSICAL_PLAN = "functions/conv/conversionTestWithPhysicalPlan.json";
@@ -98,19 +96,15 @@ public class TestConvertFunctions extends BaseTestQuery {
final OptionValue srOption = QueryTestUtil.setupScalarReplacementOption(bits[0], ScalarReplacementOption.OFF);
try {
final String newTblName = "testConvertFromConvertToInt_tbl";
- final String ctasQuery = String.format("CREATE TABLE %s.%s as \n" +
- "SELECT convert_to(r_regionkey, 'INT') as ct \n" +
- "FROM cp.`tpch/region.parquet`",
- TEMP_SCHEMA, newTblName);
- final String query = String.format("SELECT convert_from(ct, 'INT') as cf \n" +
- "FROM %s.%s \n" +
- "ORDER BY ct",
- TEMP_SCHEMA, newTblName);
test("alter session set `planner.slice_target` = 1");
- test(ctasQuery);
+ test("CREATE TABLE dfs.%s as \n" +
+ "SELECT convert_to(r_regionkey, 'INT') as ct \n" +
+ "FROM cp.`tpch/region.parquet`", newTblName);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT convert_from(ct, 'INT') as cf \n" +
+ "FROM dfs.%s \n" +
+ "ORDER BY ct", newTblName)
.ordered()
.baselineColumns("cf")
.baselineValues(0)
@@ -132,7 +126,7 @@ public class TestConvertFunctions extends BaseTestQuery {
String listStr = "[ 4, 6 ]";
testBuilder()
- .sqlQuery("select cast(convert_to(rl[1], 'JSON') as varchar(100)) as json_str from cp.`/store/json/input2.json`")
+ .sqlQuery("select cast(convert_to(rl[1], 'JSON') as varchar(100)) as json_str from cp.`store/json/input2.json`")
.unOrdered()
.baselineColumns("json_str")
.baselineValues(listStr)
@@ -143,7 +137,7 @@ public class TestConvertFunctions extends BaseTestQuery {
Object listVal = listOf(4l, 6l);
testBuilder()
- .sqlQuery("select convert_from(convert_to(rl[1], 'JSON'), 'JSON') list_col from cp.`/store/json/input2.json`")
+ .sqlQuery("select convert_from(convert_to(rl[1], 'JSON'), 'JSON') list_col from cp.`store/json/input2.json`")
.unOrdered()
.baselineColumns("list_col")
.baselineValues(listVal)
@@ -155,7 +149,7 @@ public class TestConvertFunctions extends BaseTestQuery {
Object mapVal1 = mapOf("f1", 4l, "f2", 6l);
Object mapVal2 = mapOf("f1", 11l);
testBuilder()
- .sqlQuery("select convert_from(convert_to(rl[1], 'JSON'), 'JSON') as map_col from cp.`/store/json/json_project_null_object_from_list.json`")
+ .sqlQuery("select convert_from(convert_to(rl[1], 'JSON'), 'JSON') as map_col from cp.`store/json/json_project_null_object_from_list.json`")
.unOrdered()
.baselineColumns("map_col")
.baselineValues(mapVal1)
@@ -171,10 +165,10 @@ public class TestConvertFunctions extends BaseTestQuery {
Object mapVal2 = mapOf("y", "bill", "z", "peter");
// right side of union-all produces 0 rows due to FALSE filter, column t.x is a map
- String query1 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t "
+ String query1 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t "
+ " where t.`integer` = 2010 "
+ " union all "
- + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t"
+ + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t"
+ " where 1 = 0");
testBuilder()
@@ -185,10 +179,10 @@ public class TestConvertFunctions extends BaseTestQuery {
.go();
// left side of union-all produces 0 rows due to FALSE filter, column t.x is a map
- String query2 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t "
+ String query2 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t "
+ " where 1 = 0 "
+ " union all "
- + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t "
+ + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t "
+ " where t.`integer` = 2010");
testBuilder()
@@ -199,10 +193,10 @@ public class TestConvertFunctions extends BaseTestQuery {
.go();
// sanity test where neither side produces 0 rows
- String query3 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t "
+ String query3 = String.format("select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t "
+ " where t.`integer` = 2010 "
+ " union all "
- + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t "
+ + " select 'abc' as col1, convert_from(convert_to(t.x, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t "
+ " where t.`integer` = 2001");
testBuilder()
@@ -217,9 +211,9 @@ public class TestConvertFunctions extends BaseTestQuery {
Object listVal1 = listOf(listOf(2l, 1l), listOf(4l, 6l));
Object listVal2 = listOf(); // empty
- String query4 = String.format("select 'abc' as col1, convert_from(convert_to(t.rl, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t "
+ String query4 = String.format("select 'abc' as col1, convert_from(convert_to(t.rl, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t "
+ " union all "
- + " select 'abc' as col1, convert_from(convert_to(t.rl, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`/store/json/input2.json` t"
+ + " select 'abc' as col1, convert_from(convert_to(t.rl, 'JSON'), 'JSON') as col2, 'xyz' as col3 from cp.`store/json/input2.json` t"
+ " where 1 = 0");
testBuilder()
@@ -238,12 +232,10 @@ public class TestConvertFunctions extends BaseTestQuery {
public void testConvertFromJson_drill4693() throws Exception {
Object mapVal1 = mapOf("x", "y");
- String query = String.format("select 'abc' as col1, convert_from('{\"x\" : \"y\"}', 'json') as col2, 'xyz' as col3 "
- + " from cp.`/store/json/input2.json` t"
- + " where t.`integer` = 2001");
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select 'abc' as col1, convert_from('{\"x\" : \"y\"}', 'json') as col2, 'xyz' as col3 "
+ + " from cp.`store/json/input2.json` t"
+ + " where t.`integer` = 2001")
.unOrdered()
.baselineColumns("col1", "col2", "col3")
.baselineValues("abc", mapVal1, "xyz")
@@ -263,7 +255,7 @@ public class TestConvertFunctions extends BaseTestQuery {
String result2 = "[ ]";
testBuilder()
- .sqlQuery("select cast(convert_to(rl[1], 'EXTENDEDJSON') as varchar(100)) as json_str from cp.`/store/json/input2.json`")
+ .sqlQuery("select cast(convert_to(rl[1], 'EXTENDEDJSON') as varchar(100)) as json_str from cp.`store/json/input2.json`")
.unOrdered()
.baselineColumns("json_str")
.baselineValues(result1)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java
index a24e511..8ba7e57 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDecimal.java
@@ -24,7 +24,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -58,7 +58,7 @@ public class TestDecimal extends PopUnitTestBase{
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/decimal/cast_simple_decimal.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/decimal/cast_simple_decimal.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/input_simple_decimal.json")
);
@@ -103,7 +103,7 @@ public class TestDecimal extends PopUnitTestBase{
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/decimal/cast_float_decimal.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/decimal/cast_float_decimal.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/input_simple_decimal.json")
);
@@ -148,7 +148,7 @@ public class TestDecimal extends PopUnitTestBase{
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/decimal/simple_decimal_arithmetic.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/decimal/simple_decimal_arithmetic.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/input_simple_decimal.json")
);
@@ -199,7 +199,7 @@ public class TestDecimal extends PopUnitTestBase{
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/decimal/test_decimal_complex.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/decimal/test_decimal_complex.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/input_complex_decimal.json")
);
@@ -242,7 +242,7 @@ public class TestDecimal extends PopUnitTestBase{
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/decimal/test_decimal_sort_complex.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/decimal/test_decimal_sort_complex.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/input_sort_complex_decimal.json")
);
@@ -291,7 +291,7 @@ public class TestDecimal extends PopUnitTestBase{
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/decimal/simple_decimal_math.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/decimal/simple_decimal_math.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/input_simple_decimal.json")
);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java
index 0349352..4ae4f40 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestDistributedFragmentRun.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.util.List;
import org.apache.drill.categories.SlowTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
@@ -46,7 +46,7 @@ public class TestDistributedFragmentRun extends PopUnitTestBase{
try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet); DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());){
bit1.run();
client.connect();
- List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_single_exchange.json"), Charsets.UTF_8));
+ List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile("/physical_single_exchange.json"), Charsets.UTF_8));
int count = 0;
for(QueryDataBatch b : results){
count += b.getHeader().getRowCount();
@@ -66,7 +66,7 @@ public class TestDistributedFragmentRun extends PopUnitTestBase{
try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet); DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());){
bit1.run();
client.connect();
- List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_single_exchange_double_entry.json"), Charsets.UTF_8));
+ List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile("/physical_single_exchange_double_entry.json"), Charsets.UTF_8));
int count = 0;
for(QueryDataBatch b : results){
count += b.getHeader().getRowCount();
@@ -85,7 +85,7 @@ public class TestDistributedFragmentRun extends PopUnitTestBase{
try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet); DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());){
bit1.run();
client.connect();
- List<QueryDataBatch> results = client.runQuery(QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile("/scan_screen_logical.json"), Charsets.UTF_8));
+ List<QueryDataBatch> results = client.runQuery(QueryType.LOGICAL, Files.toString(DrillFileUtils.getResourceAsFile("/scan_screen_logical.json"), Charsets.UTF_8));
int count = 0;
for(QueryDataBatch b : results){
count += b.getHeader().getRowCount();
@@ -105,7 +105,7 @@ public class TestDistributedFragmentRun extends PopUnitTestBase{
bit1.run();
bit2.run();
client.connect();
- List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_single_exchange_double_entry.json"), Charsets.UTF_8));
+ List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile("/physical_single_exchange_double_entry.json"), Charsets.UTF_8));
int count = 0;
for(QueryDataBatch b : results){
count += b.getHeader().getRowCount();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java
index 937faf8..87bc182 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestExtractFunctions.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
import org.apache.drill.categories.SlowTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -114,7 +114,7 @@ public class TestExtractFunctions extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/functions/extractFrom.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/functions/extractFrom.json"), Charsets.UTF_8)
.replace("#{TEST_TYPE}", fromType)
.replace("#{TEST_FILE}", testDataFile)
.replace("#{COLUMN_NAME}", columnName));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java
index c9954de..50e0b54 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.util.List;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
@@ -50,7 +50,7 @@ public class TestHashToRandomExchange extends PopUnitTestBase {
bit2.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/sender/hash_exchange.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/sender/hash_exchange.json"),
Charsets.UTF_8));
int count = 0;
for (QueryDataBatch b : results) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java
index f712930..3c10884 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java
@@ -23,7 +23,7 @@ import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.drill.PlanTestBase;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.exec.physical.base.Exchange;
import org.apache.drill.exec.physical.config.UnorderedDeMuxExchange;
import org.apache.drill.exec.physical.config.HashToRandomExchange;
@@ -54,6 +54,7 @@ import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
+import java.nio.file.Paths;
import java.util.List;
import static org.apache.drill.exec.planner.physical.HashPrelUtil.HASH_EXPR_NAME;
@@ -71,16 +72,15 @@ import static org.junit.Assert.assertTrue;
* at most one partition sender per Drillbit.
*/
public class TestLocalExchange extends PlanTestBase {
-
- public static TemporaryFolder testTempFolder = new TemporaryFolder();
-
- private final static int CLUSTER_SIZE = 3;
- private final static String MUX_EXCHANGE = "\"unordered-mux-exchange\"";
- private final static String DEMUX_EXCHANGE = "\"unordered-demux-exchange\"";
- private final static String MUX_EXCHANGE_CONST = "unordered-mux-exchange";
- private final static String DEMUX_EXCHANGE_CONST = "unordered-demux-exchange";
+ private static final int CLUSTER_SIZE = 3;
+ private static final String MUX_EXCHANGE = "\"unordered-mux-exchange\"";
+ private static final String DEMUX_EXCHANGE = "\"unordered-demux-exchange\"";
+ private static final String MUX_EXCHANGE_CONST = "unordered-mux-exchange";
+ private static final String DEMUX_EXCHANGE_CONST = "unordered-demux-exchange";
private static final String HASH_EXCHANGE = "hash-to-random-exchange";
- private final static UserSession USER_SESSION = UserSession.Builder.newBuilder()
+ private static final String EMPT_TABLE = "empTable";
+ private static final String DEPT_TABLE = "deptTable";
+ private static final UserSession USER_SESSION = UserSession.Builder.newBuilder()
.withCredentials(UserBitShared.UserCredentials.newBuilder().setUserName("foo").build())
.build();
@@ -95,9 +95,6 @@ public class TestLocalExchange extends PlanTestBase {
private final static int NUM_MNGRS = 1;
private final static int NUM_IDS = 1;
- private static String empTableLocation;
- private static String deptTableLocation;
-
private static String groupByQuery;
private static String joinQuery;
@@ -112,23 +109,18 @@ public class TestLocalExchange extends PlanTestBase {
updateTestCluster(CLUSTER_SIZE, null);
}
- @BeforeClass
- public static void setupTempFolder() throws IOException {
- testTempFolder.create();
- }
-
/**
* Generate data for two tables. Each table consists of several JSON files.
*/
@BeforeClass
public static void generateTestDataAndQueries() throws Exception {
// Table 1 consists of two columns "emp_id", "emp_name" and "dept_id"
- empTableLocation = testTempFolder.newFolder().getAbsolutePath();
+ final File empTableLocation = dirTestWatcher.makeRootSubDir(Paths.get(EMPT_TABLE));
// Write 100 records for each new file
final int empNumRecsPerFile = 100;
for(int fileIndex=0; fileIndex<NUM_EMPLOYEES/empNumRecsPerFile; fileIndex++) {
- File file = new File(empTableLocation + File.separator + fileIndex + ".json");
+ File file = new File(empTableLocation, fileIndex + ".json");
PrintWriter printWriter = new PrintWriter(file);
for (int recordIndex = fileIndex*empNumRecsPerFile; recordIndex < (fileIndex+1)*empNumRecsPerFile; recordIndex++) {
String record = String.format("{ \"emp_id\" : %d, \"emp_name\" : \"Employee %d\", \"dept_id\" : %d, \"mng_id\" : %d, \"some_id\" : %d }",
@@ -139,12 +131,12 @@ public class TestLocalExchange extends PlanTestBase {
}
// Table 2 consists of two columns "dept_id" and "dept_name"
- deptTableLocation = testTempFolder.newFolder().getAbsolutePath();
+ final File deptTableLocation = dirTestWatcher.makeRootSubDir(Paths.get(DEPT_TABLE));
// Write 4 records for each new file
final int deptNumRecsPerFile = 4;
for(int fileIndex=0; fileIndex<NUM_DEPTS/deptNumRecsPerFile; fileIndex++) {
- File file = new File(deptTableLocation + File.separator + fileIndex + ".json");
+ File file = new File(deptTableLocation, fileIndex + ".json");
PrintWriter printWriter = new PrintWriter(file);
for (int recordIndex = fileIndex*deptNumRecsPerFile; recordIndex < (fileIndex+1)*deptNumRecsPerFile; recordIndex++) {
String record = String.format("{ \"dept_id\" : %d, \"dept_name\" : \"Department %d\" }",
@@ -155,9 +147,9 @@ public class TestLocalExchange extends PlanTestBase {
}
// Initialize test queries
- groupByQuery = String.format("SELECT dept_id, count(*) as numEmployees FROM dfs.`%s` GROUP BY dept_id", empTableLocation);
+ groupByQuery = String.format("SELECT dept_id, count(*) as numEmployees FROM dfs.`%s` GROUP BY dept_id", EMPT_TABLE);
joinQuery = String.format("SELECT e.emp_name, d.dept_name FROM dfs.`%s` e JOIN dfs.`%s` d ON e.dept_id = d.dept_id",
- empTableLocation, deptTableLocation);
+ EMPT_TABLE, DEPT_TABLE);
// Generate and store output data for test queries. Used when verifying the output of queries ran using different
// configurations.
@@ -200,7 +192,7 @@ public class TestLocalExchange extends PlanTestBase {
test("ALTER SESSION SET `planner.enable_mux_exchange`=" + true);
test("ALTER SESSION SET `planner.enable_demux_exchange`=" + false);
- final String groupByMultipleQuery = String.format("SELECT dept_id, mng_id, some_id, count(*) as numEmployees FROM dfs.`%s` e GROUP BY dept_id, mng_id, some_id", empTableLocation);
+ final String groupByMultipleQuery = String.format("SELECT dept_id, mng_id, some_id, count(*) as numEmployees FROM dfs.`%s` e GROUP BY dept_id, mng_id, some_id", EMPT_TABLE);
final String[] groupByMultipleQueryBaselineColumns = new String[] { "dept_id", "mng_id", "some_id", "numEmployees" };
final int numOccurrances = NUM_EMPLOYEES/NUM_DEPTS;
@@ -478,9 +470,4 @@ public class TestLocalExchange extends PlanTestBase {
assertTrue("Some endpoints have more than one fragment that has ParitionSender", ImmutableSet.copyOf(assignments).size() == assignments.size());
}
}
-
- @AfterClass
- public static void cleanupTempFolder() throws IOException {
- testTempFolder.delete();
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index f09e569..b6bc497 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -25,7 +25,7 @@ import mockit.NonStrictExpectations;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.coord.ClusterCoordinator;
@@ -119,7 +119,7 @@ public class TestOptiqPlans extends ExecTest {
final QueryContext qc = new QueryContext(UserSession.Builder.newBuilder().setSupportComplexTypes(true).build(),
bitContext, QueryId.getDefaultInstance());
final PhysicalPlanReader reader = bitContext.getPlanReader();
- final LogicalPlan plan = reader.readLogicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
+ final LogicalPlan plan = reader.readLogicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(file), Charsets.UTF_8));
final PhysicalPlan pp = new BasicOptimizer(qc, connection).optimize(new BasicOptimizer.BasicOptimizationContext(qc), plan);
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config);
@@ -312,7 +312,7 @@ public class TestOptiqPlans extends ExecTest {
final StoragePluginRegistry reg = new StoragePluginRegistryImpl(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(config, reg);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(file), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java
index ae8302d..84d68f5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestReverseImplicitCast.java
@@ -25,7 +25,7 @@ import java.util.List;
import mockit.Injectable;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -56,7 +56,7 @@ public class TestReverseImplicitCast extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/functions/cast/two_way_implicit_cast.json"), Charsets.UTF_8));
+ Files.toString(DrillFileUtils.getResourceAsFile("/functions/cast/two_way_implicit_cast.json"), Charsets.UTF_8));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index 3b8ab3f..6dd181b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.memory.RootAllocatorFactory;
import org.apache.drill.exec.pop.PopUnitTestBase;
@@ -53,7 +53,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
client.connect();
final String path = "/physical_test2.json";
// String path = "/filter/test1.json";
- final List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile(path), Charsets.UTF_8));
+ final List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile(path), Charsets.UTF_8));
// look at records
final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
@@ -113,8 +113,8 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
bit.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8)
- .replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString())
+ Files.toString(DrillFileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8)
+ .replace("#{TEST_FILE}", DrillFileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString())
);
// look at records
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index c21facb..1f83fc4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -33,7 +33,7 @@ import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
@@ -148,7 +148,7 @@ public class TestSimpleFunctions extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testSubstring.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/testSubstring.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -181,7 +181,7 @@ public class TestSimpleFunctions extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testSubstringNegative.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/testSubstringNegative.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -215,7 +215,7 @@ public class TestSimpleFunctions extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testByteSubstring.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/testByteSubstring.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
index b87a085..dfcbc71 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java
@@ -41,7 +41,6 @@ import com.google.common.io.Resources;
import mockit.Injectable;
public class TestStringFunctions extends ExecTest {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestStringFunctions.class);
private final DrillConfig c = DrillConfig.create();
private PhysicalPlanReader reader;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java
index dc653c2..74b3a7e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestUnionExchange.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.util.List;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
@@ -50,7 +50,7 @@ public class TestUnionExchange extends PopUnitTestBase {
bit2.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/sender/union_exchange.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/sender/union_exchange.json"),
Charsets.UTF_8));
int count = 0;
for (QueryDataBatch b : results) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestSimpleTopN.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestSimpleTopN.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestSimpleTopN.java
deleted file mode 100644
index 43c0af0..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestSimpleTopN.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.TopN;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-
-import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.pop.PopUnitTestBase;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.rpc.user.QueryDataBatch;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.vector.BigIntVector;
-import org.junit.Test;
-
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import org.junit.experimental.categories.Category;
-
-@Category(OperatorTest.class)
-public class TestSimpleTopN extends PopUnitTestBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleTopN.class);
- DrillConfig c = DrillConfig.create();
-
- @Test
- public void sortOneKeyAscending() throws Throwable{
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-
- try (Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
- Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
-
- bit1.run();
- bit2.run();
- client.connect();
- List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/topN/one_key_sort.json"),
- Charsets.UTF_8));
- int count = 0;
- for (QueryDataBatch b : results) {
- if (b.getHeader().getRowCount() != 0) {
- count += b.getHeader().getRowCount();
- }
- }
- assertEquals(100, count);
-
- long previousBigInt = Long.MIN_VALUE;
-
- int recordCount = 0;
- int batchCount = 0;
-
- for (QueryDataBatch b : results) {
- if (b.getHeader().getRowCount() == 0) {
- continue;
- }
- batchCount++;
- RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
- loader.load(b.getHeader().getDef(),b.getData());
- BigIntVector c1 = (BigIntVector) loader.getValueAccessorById(BigIntVector.class, loader.getValueVectorId(new SchemaPath("blue", ExpressionPosition.UNKNOWN)).getFieldIds()).getValueVector();
-
-
- BigIntVector.Accessor a1 = c1.getAccessor();
-// IntVector.Accessor a2 = c2.getAccessor();
-
- for (int i =0; i < c1.getAccessor().getValueCount(); i++) {
- recordCount++;
- assertTrue(previousBigInt <= a1.get(i));
- previousBigInt = a1.get(i);
- }
- loader.clear();
- b.release();
- }
- System.out.println(String.format("Sorted %,d records in %d batches.", recordCount, batchCount));
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java
index 066d966..031041f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TestTopNSchemaChanges.java
@@ -17,39 +17,57 @@
*/
package org.apache.drill.exec.physical.impl.TopN;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.TestBuilder;
+import org.apache.drill.test.SubDirTestWatcher;
+import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
+import java.nio.file.Path;
+import java.nio.file.Paths;
@Category(OperatorTest.class)
public class TestTopNSchemaChanges extends BaseTestQuery {
+ private static final Path TABLE = Paths.get("table");
+
+ private static File tableDir;
+
+ @BeforeClass
+ public static void setupTestFiles() {
+ tableDir = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(TABLE)
+ .toFile();
+ }
+
+ @Rule
+ public final SubDirTestWatcher localDirTestWatcher =
+ new SubDirTestWatcher.Builder(dirTestWatcher.getRootDir())
+ .addSubDir(TABLE)
+ .build();
@Test
public void testNumericTypes() throws Exception {
- final File data_dir = new File(BaseTestQuery.getTempDir("topn-schemachanges"));
- data_dir.mkdirs();
-
// left side int and strings
- BufferedWriter writer = new BufferedWriter(new FileWriter(new File(data_dir, "d1.json")));
+ BufferedWriter writer = new BufferedWriter(new FileWriter(new File(tableDir, "d1.json")));
for (int i = 0; i < 10000; i+=2) {
writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(data_dir, "d2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(tableDir, "d2.json")));
for (int i = 1; i < 10000; i+=2) {
writer.write(String.format("{ \"kl\" : %f , \"vl\": %f }\n", (float)i, (float)i));
}
writer.close();
- String query = String.format("select * from dfs_test.`%s` order by kl limit 12", data_dir.toPath().toString());
TestBuilder builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from dfs.`%s` order by kl limit 12", TABLE)
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.ordered()
.baselineColumns("kl", "vl");
@@ -66,24 +84,20 @@ public class TestTopNSchemaChanges extends BaseTestQuery {
@Test
public void testNumericAndStringTypes() throws Exception {
- final File data_dir = new File(BaseTestQuery.getTempDir("topn-schemachanges"));
- data_dir.mkdirs();
-
// left side int and strings
- BufferedWriter writer = new BufferedWriter(new FileWriter(new File(data_dir, "d1.json")));
+ BufferedWriter writer = new BufferedWriter(new FileWriter(new File(tableDir, "d1.json")));
for (int i = 0; i < 1000; i+=2) {
writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(data_dir, "d2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(tableDir, "d2.json")));
for (int i = 1; i < 1000; i+=2) {
writer.write(String.format("{ \"kl\" : \"%s\" , \"vl\": \"%s\" }\n", i, i));
}
writer.close();
- String query = String.format("select * from dfs_test.`%s` order by kl limit 12", data_dir.toPath().toString());
TestBuilder builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from dfs.`%s` order by kl limit 12", TABLE)
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.ordered()
.baselineColumns("kl", "vl");
@@ -92,9 +106,8 @@ public class TestTopNSchemaChanges extends BaseTestQuery {
builder.baselineValues(i, i);
}
- query = String.format("select * from dfs_test.`%s` order by kl desc limit 12", data_dir.toPath().toString());
builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from dfs.`%s` order by kl desc limit 12", TABLE)
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.ordered()
.baselineColumns("kl", "vl")
@@ -115,11 +128,8 @@ public class TestTopNSchemaChanges extends BaseTestQuery {
@Test
public void testUnionTypes() throws Exception {
- final File data_dir = new File(BaseTestQuery.getTempDir("topn-schemachanges"));
- data_dir.mkdirs();
-
// union of int and float and string.
- BufferedWriter writer = new BufferedWriter(new FileWriter(new File(data_dir, "d1.json")));
+ BufferedWriter writer = new BufferedWriter(new FileWriter(new File(tableDir, "d1.json")));
for (int i = 0; i <= 9; ++i) {
switch (i%3) {
case 0: // 0, 3, 6, 9
@@ -134,10 +144,9 @@ public class TestTopNSchemaChanges extends BaseTestQuery {
}
}
writer.close();
- String query = String.format("select * from dfs_test.`%s` order by kl limit 8", data_dir.toPath().toString());
TestBuilder builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from dfs.`%s` order by kl limit 8", TABLE)
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.ordered()
.baselineColumns("kl", "vl");
@@ -155,29 +164,25 @@ public class TestTopNSchemaChanges extends BaseTestQuery {
@Test
public void testMissingColumn() throws Exception {
- final File data_dir = new File(BaseTestQuery.getTempDir("topn-schemachanges"));
- data_dir.mkdirs();
-
- BufferedWriter writer = new BufferedWriter(new FileWriter(new File(data_dir, "d1.json")));
+ BufferedWriter writer = new BufferedWriter(new FileWriter(new File(tableDir, "d1.json")));
for (int i = 0; i < 100; i++) {
writer.write(String.format("{ \"kl1\" : %d , \"vl1\": %d }\n", i, i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(data_dir, "d2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(tableDir, "d2.json")));
for (int i = 100; i < 200; i++) {
writer.write(String.format("{ \"kl\" : %f , \"vl\": %f }\n", (float)i, (float)i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(data_dir, "d3.json")));
+ writer = new BufferedWriter(new FileWriter(new File(tableDir, "d3.json")));
for (int i = 200; i < 300; i++) {
writer.write(String.format("{ \"kl2\" : \"%s\" , \"vl2\": \"%s\" }\n", i, i));
}
writer.close();
- String query = String.format("select kl, vl, kl1, vl1, kl2, vl2 from dfs_test.`%s` order by kl limit 3", data_dir.toPath().toString());
TestBuilder builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select kl, vl, kl1, vl1, kl2, vl2 from dfs.`%s` order by kl limit 3", TABLE)
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.ordered()
.baselineColumns("kl", "vl", "kl1", "vl1", "kl2", "vl2")
@@ -186,9 +191,8 @@ public class TestTopNSchemaChanges extends BaseTestQuery {
.baselineValues(102.0d, 102.0d, null, null, null, null);
builder.go();
- query = String.format("select kl, vl, kl1, vl1, kl2, vl2 from dfs_test.`%s` order by kl1 limit 3", data_dir.toPath().toString());
builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select kl, vl, kl1, vl1, kl2, vl2 from dfs.`%s` order by kl1 limit 3", TABLE)
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.ordered()
.baselineColumns("kl", "vl", "kl1", "vl1", "kl2", "vl2")
@@ -197,9 +201,8 @@ public class TestTopNSchemaChanges extends BaseTestQuery {
.baselineValues(null, null, 2l, 2l, null, null);
builder.go();
- query = String.format("select kl, vl, kl1, vl1, kl2, vl2 from dfs_test.`%s` order by kl2 limit 3", data_dir.toPath().toString());
builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select kl, vl, kl1, vl1, kl2, vl2 from dfs.`%s` order by kl2 limit 3", TABLE)
.optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
.ordered()
.baselineColumns("kl", "vl", "kl1", "vl1", "kl2", "vl2")
@@ -210,6 +213,6 @@ public class TestTopNSchemaChanges extends BaseTestQuery {
// Since client can't handle new columns which are not in first batch, we won't test output of query.
// Query should run w/o any errors.
- test(String.format("select * from dfs_test.`%s` order by kl limit 3", data_dir.toPath().toString()));
+ test("select * from dfs.`%s` order by kl limit 3", TABLE);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java
new file mode 100644
index 0000000..fa6e318
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.TopN;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import com.google.common.collect.Lists;
+import org.apache.drill.test.BaseDirTestWatcher;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.TestBuilder;
+import org.apache.drill.categories.OperatorTest;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.compile.ClassBuilder;
+import org.apache.drill.exec.compile.CodeCompiler;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.memory.RootAllocator;
+import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
+import org.apache.drill.exec.pop.PopUnitTestBase;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.ExpandableHyperContainer;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.test.ClientFixture;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.rowSet.HyperRowSetImpl;
+import org.apache.drill.test.rowSet.RowSet;
+import org.apache.drill.test.rowSet.RowSetBuilder;
+import org.apache.drill.test.rowSet.RowSetComparison;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(OperatorTest.class)
+public class TopNBatchTest extends PopUnitTestBase {
+ @Rule
+ public BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
+ /**
+ * Priority queue unit test.
+ * @throws Exception
+ */
+ @Test
+ public void priorityQueueOrderingTest() throws Exception {
+ Properties properties = new Properties();
+ properties.setProperty(ClassBuilder.CODE_DIR_OPTION, dirTestWatcher.getDir().getAbsolutePath());
+
+ DrillConfig drillConfig = DrillConfig.create(properties);
+
+ FieldReference expr = FieldReference.getWithQuotedRef("colA");
+ Order.Ordering ordering = new Order.Ordering(Order.Ordering.ORDER_DESC, expr, Order.Ordering.NULLS_FIRST);
+ List<Order.Ordering> orderings = Lists.newArrayList(ordering);
+
+ MaterializedField colA = MaterializedField.create("colA", Types.required(TypeProtos.MinorType.INT));
+ MaterializedField colB = MaterializedField.create("colB", Types.required(TypeProtos.MinorType.INT));
+
+ List<MaterializedField> cols = Lists.newArrayList(colA, colB);
+ BatchSchema batchSchema = new BatchSchema(BatchSchema.SelectionVectorMode.NONE, cols);
+ RowSet expectedRowSet;
+
+ try (RootAllocator allocator = new RootAllocator(100_000_000)) {
+ expectedRowSet = new RowSetBuilder(allocator, batchSchema)
+ .add(110, 10)
+ .add(109, 9)
+ .add(108, 8)
+ .add(107, 7)
+ .add(106, 6)
+ .add(105, 5)
+ .add(104, 4)
+ .add(103, 3)
+ .add(102, 2)
+ .add(101, 1)
+ .build();
+
+ PriorityQueue queue;
+ ExpandableHyperContainer hyperContainer;
+
+ {
+ VectorContainer container = new RowSetBuilder(allocator, batchSchema)
+ .build()
+ .container();
+ hyperContainer = new ExpandableHyperContainer(container);
+
+ queue = TopNBatch.createNewPriorityQueue(
+ TopNBatch.createMainMappingSet(), TopNBatch.createLeftMappingSet(),
+ TopNBatch.createRightMappingSet(), optionManager,
+ new FunctionImplementationRegistry(drillConfig),
+ new CodeCompiler(drillConfig, optionManager),
+ orderings, hyperContainer, false,
+ true, 10, allocator,
+ batchSchema.getSelectionVectorMode());
+ }
+
+ List<RecordBatchData> testBatches = Lists.newArrayList();
+
+ try {
+ final Random random = new Random();
+ final int bound = 100;
+ final int numBatches = 11;
+ final int numRecordsPerBatch = 100;
+
+ for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) {
+ RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, batchSchema);
+ rowSetBuilder.add((batchCounter + bound), batchCounter);
+
+ for (int recordCounter = 0; recordCounter < numRecordsPerBatch; recordCounter++) {
+ rowSetBuilder.add(random.nextInt(bound), random.nextInt(bound));
+ }
+
+ VectorContainer vectorContainer = rowSetBuilder.build().container();
+ queue.add(new RecordBatchData(vectorContainer, allocator));
+ }
+
+ queue.generate();
+ VectorContainer resultContainer = queue.getHyperBatch();
+ resultContainer.buildSchema(BatchSchema.SelectionVectorMode.NONE);
+
+ RowSet.HyperRowSet actualHyperSet = new HyperRowSetImpl(allocator, resultContainer, queue.getFinalSv4());
+ new RowSetComparison(expectedRowSet).verify(actualHyperSet);
+ } finally {
+ if (expectedRowSet != null) {
+ expectedRowSet.clear();
+ }
+
+ queue.cleanup();
+ hyperContainer.clear();
+
+ for (RecordBatchData testBatch: testBatches) {
+ testBatch.clear();
+ }
+ }
+ }
+ }
+
+ /**
+ * End to end test of the TopN operator.
+ * @throws Throwable
+ */
+ @Test
+ public void sortOneKeyAscending() throws Throwable {
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .configProperty(ClassBuilder.CODE_DIR_OPTION, dirTestWatcher.getDir().getAbsolutePath())
+ .configProperty(CodeCompiler.ENABLE_SAVE_CODE_FOR_DEBUG_TOPN, true);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ TestBuilder testBuilder = new TestBuilder(new ClusterFixture.FixtureTestServices(client));
+ testBuilder.ordered()
+ .physicalPlanFromFile("topN/one_key_sort.json")
+ .baselineColumns("blue")
+ .go();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
index 8dd2da0..f07352c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -57,7 +57,7 @@ public class TestAgg extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(file), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java
index b7c9e92..3de5519 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.physical.impl.agg;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
import org.junit.Ignore;
import org.junit.Test;
[13/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java
index 2e81acd..7be6195 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java
@@ -19,13 +19,13 @@
package org.apache.drill.exec.physical.impl.agg;
import ch.qos.logback.classic.Level;
-import org.apache.drill.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.physical.impl.aggregate.HashAggTemplate;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterFixtureBuilder;
@@ -33,6 +33,7 @@ import org.apache.drill.test.LogFixture;
import org.apache.drill.test.ProfileParser;
import org.apache.drill.test.QueryBuilder;
import org.apache.drill.categories.SlowTest;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -43,43 +44,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
- * Test spilling for the Hash Aggr operator (using the mock reader)
+ * Test spilling for the Hash Aggr operator (using the mock reader)
*/
@Category({SlowTest.class, OperatorTest.class})
public class TestHashAggrSpill {
- private void runAndDump(ClientFixture client, String sql, long expectedRows, long spillCycle, long fromSpilledPartitions, long toSpilledPartitions) throws Exception {
- String plan = client.queryBuilder().sql(sql).explainJson();
-
- QueryBuilder.QuerySummary summary = client.queryBuilder().sql(sql).run();
- if ( expectedRows > 0 ) {
- assertEquals(expectedRows, summary.recordCount());
- }
- // System.out.println(String.format("======== \n Results: %,d records, %d batches, %,d ms\n ========", summary.recordCount(), summary.batchCount(), summary.runTimeMs() ) );
-
- //System.out.println("Query ID: " + summary.queryIdString());
- ProfileParser profile = client.parseProfile(summary.queryIdString());
- //profile.print();
- List<ProfileParser.OperatorProfile> ops = profile.getOpsOfType(UserBitShared.CoreOperatorType.HASH_AGGREGATE_VALUE);
-
- assertTrue( ! ops.isEmpty() );
- // check for the first op only
- ProfileParser.OperatorProfile hag0 = ops.get(0);
- long opCycle = hag0.getMetric(HashAggTemplate.Metric.SPILL_CYCLE.ordinal());
- assertEquals(spillCycle, opCycle);
- long op_spilled_partitions = hag0.getMetric(HashAggTemplate.Metric.SPILLED_PARTITIONS.ordinal());
- assertTrue( op_spilled_partitions >= fromSpilledPartitions && op_spilled_partitions <= toSpilledPartitions );
- /* assertEquals(3, ops.size());
- for ( int i = 0; i < ops.size(); i++ ) {
- ProfileParser.OperatorProfile hag = ops.get(i);
- long cycle = hag.getMetric(HashAggTemplate.Metric.SPILL_CYCLE.ordinal());
- long num_partitions = hag.getMetric(HashAggTemplate.Metric.NUM_PARTITIONS.ordinal());
- long spilled_partitions = hag.getMetric(HashAggTemplate.Metric.SPILLED_PARTITIONS.ordinal());
- long mb_spilled = hag.getMetric(HashAggTemplate.Metric.SPILL_MB.ordinal());
- System.out.println(String.format("(%d) Spill cycle: %d, num partitions: %d, spilled partitions: %d, MB spilled: %d", i,cycle, num_partitions, spilled_partitions,
- mb_spilled));
- } */
- }
+ @Rule
+ public final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
/**
* A template for Hash Aggr spilling tests
@@ -90,24 +61,18 @@ public class TestHashAggrSpill {
String sql, long expectedRows, int cycle, int fromPart, int toPart) throws Exception {
LogFixture.LogFixtureBuilder logBuilder = LogFixture.builder()
.toConsole()
- //.logger("org.apache.drill.exec.physical.impl.aggregate", Level.INFO)
- .logger("org.apache.drill", Level.WARN)
- ;
+ .logger("org.apache.drill", Level.WARN);
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
.sessionOption(ExecConstants.HASHAGG_MAX_MEMORY_KEY,maxMem)
.sessionOption(ExecConstants.HASHAGG_NUM_PARTITIONS_KEY,numPartitions)
.sessionOption(ExecConstants.HASHAGG_MIN_BATCHES_PER_PARTITION_KEY,minBatches)
.configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, false)
.sessionOption(PlannerSettings.FORCE_2PHASE_AGGR_KEY,true)
- // .sessionOption(PlannerSettings.EXCHANGE.getOptionName(), true)
.sessionOption(ExecConstants.HASHAGG_FALLBACK_ENABLED_KEY, fallback)
.sessionOption(ExecConstants.HASHAGG_USE_MEMORY_PREDICTION_KEY,predict)
-
.maxParallelization(maxParallel)
- .saveProfiles()
- //.keepLocalFiles()
- ;
+ .saveProfiles();
String sqlStr = sql != null ? sql : // if null then use this default query
"SELECT empid_s17, dept_i, branch_i, AVG(salary_i) FROM `mock`.`employee_1200K` GROUP BY empid_s17, dept_i, branch_i";
@@ -137,52 +102,68 @@ public class TestHashAggrSpill {
*/
@Test
public void testNoPredictHashAggrSpill() throws Exception {
- testSpill(58_000_000, 16, 2, 2, false,false /* no prediction */,
- null,1_200_000, 1,1, 1
- );
+ testSpill(58_000_000, 16, 2, 2, false, false /* no prediction */, null,
+ 1_200_000, 1, 1, 1);
}
- /**
- * Test Secondary and Tertiary spill cycles - Happens when some of the spilled partitions cause more spilling as they are read back
- *
- * @throws Exception
- */
- @Test
- public void testHashAggrSecondaryTertiarySpill() throws Exception {
- testSpill(58_000_000, 16, 3, 1, false,true,
- "SELECT empid_s44, dept_i, branch_i, AVG(salary_i) FROM `mock`.`employee_1100K` GROUP BY empid_s44, dept_i, branch_i",
- 1_100_000, 3,2, 2
- );
+ private void runAndDump(ClientFixture client, String sql, long expectedRows, long spillCycle, long fromSpilledPartitions, long toSpilledPartitions) throws Exception {
+ QueryBuilder.QuerySummary summary = client.queryBuilder().sql(sql).run();
+ if (expectedRows > 0) {
+ assertEquals(expectedRows, summary.recordCount());
}
- /**
- * Test with the "fallback" option disabled: When not enough memory available to allow spilling, then fail (Resource error) !!
- *
- * @throws Exception
- */
- @Test
- public void testHashAggrFailWithFallbackDisabed() throws Exception {
-
- try {
- testSpill(34_000_000, 4, 5, 2, false /* no fallback */, true, null,
- 1_200_000, 0 /* no spill due to fallback to pre-1.11 */, 0, 0);
- fail(); // in case the above test did not throw
- } catch (Exception ex) {
- assertTrue(ex instanceof UserRemoteException);
- assertTrue(((UserRemoteException)ex).getErrorType() == UserBitShared.DrillPBError.ErrorType.RESOURCE);
- // must get here for the test to succeed ...
- }
- }
- /**
- * Test with the "fallback" option ON: When not enough memory is available to allow spilling (internally need enough memory to
- * create multiple partitions), then behave like the pre-1.11 Hash Aggregate: Allocate unlimited memory, no spill.
- *
- * @throws Exception
- */
- @Test
- public void testHashAggrSuccessWithFallbackEnabled() throws Exception {
- testSpill(34_000_000, 4, 5, 2, true /* do fallback */,true, null,
- 1_200_000, 0 /* no spill due to fallback to pre-1.11 */,0, 0
- );
+ ProfileParser profile = client.parseProfile(summary.queryIdString());
+ List<ProfileParser.OperatorProfile> ops = profile.getOpsOfType(UserBitShared.CoreOperatorType.HASH_AGGREGATE_VALUE);
+
+ assertTrue(!ops.isEmpty());
+ // check for the first op only
+ ProfileParser.OperatorProfile hag0 = ops.get(0);
+ long opCycle = hag0.getMetric(HashAggTemplate.Metric.SPILL_CYCLE.ordinal());
+ assertEquals(spillCycle, opCycle);
+ long op_spilled_partitions = hag0.getMetric(HashAggTemplate.Metric.SPILLED_PARTITIONS.ordinal());
+ assertTrue(op_spilled_partitions >= fromSpilledPartitions && op_spilled_partitions <= toSpilledPartitions);
+ }
+
+ /**
+ * Test Secondary and Tertiary spill cycles - Happens when some of the spilled partitions cause more spilling as they are read back
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testHashAggrSecondaryTertiarySpill() throws Exception {
+
+ testSpill(58_000_000, 16, 3, 1, false, true, "SELECT empid_s44, dept_i, branch_i, AVG(salary_i) FROM `mock`.`employee_1100K` GROUP BY empid_s44, dept_i, branch_i",
+ 1_100_000, 3, 2, 2);
+ }
+
+ /**
+ * Test with the "fallback" option disabled: When not enough memory available to allow spilling, then fail (Resource error) !!
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testHashAggrFailWithFallbackDisabed() throws Exception {
+
+ try {
+ testSpill(34_000_000, 4, 5, 2, false /* no fallback */, true, null,
+ 1_200_000, 0 /* no spill due to fallback to pre-1.11 */, 0, 0);
+ fail(); // in case the above test did not throw
+ } catch (Exception ex) {
+ assertTrue(ex instanceof UserRemoteException);
+ assertTrue(((UserRemoteException) ex).getErrorType() == UserBitShared.DrillPBError.ErrorType.RESOURCE);
+ // must get here for the test to succeed ...
}
+ }
+
+ /**
+ * Test with the "fallback" option ON: When not enough memory is available to allow spilling (internally need enough memory to
+ * create multiple partitions), then behave like the pre-1.11 Hash Aggregate: Allocate unlimited memory, no spill.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testHashAggrSuccessWithFallbackEnabled() throws Exception {
+ testSpill(34_000_000, 4, 5, 2, true /* do fallback */, true, null,
+ 1_200_000, 0 /* no spill due to fallback to pre-1.11 */, 0, 0);
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java
index 5b869e1..970f4a4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/broadcastsender/TestBroadcast.java
@@ -17,35 +17,40 @@
*/
package org.apache.drill.exec.physical.impl.broadcastsender;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
@Category(OperatorTest.class)
public class TestBroadcast extends BaseTestQuery {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBroadcast.class);
+ public static final String BROAD_CAST_QUERY = "select * from "
+ + "dfs.`broadcast/sales` s "
+ + "INNER JOIN "
+ + "dfs.`broadcast/customer` c "
+ + "ON s.id = c.id";
- String broadcastQuery = "select * from "
- + "dfs.`${WORKING_PATH}/src/test/resources/broadcast/sales` s "
- + "INNER JOIN "
- + "dfs.`${WORKING_PATH}/src/test/resources/broadcast/customer` c "
- + "ON s.id = c.id";
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("broadcast"));
+ }
@Test
public void plansWithBroadcast() throws Exception {
//TODO: actually verify that this plan has a broadcast exchange in it once plan tools are enabled.
setup();
- test("explain plan for " + broadcastQuery);
+ test("explain plan for %s", BROAD_CAST_QUERY);
}
@Test
public void broadcastExecuteWorks() throws Exception {
setup();
- test(broadcastQuery);
+ test(BROAD_CAST_QUERY);
}
-
private void setup() throws Exception{
testNoResult("alter session set `planner.slice_target` = 1");
testNoResult("alter session set `planner.enable_broadcast_join` = true");
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java
index 353c201..a68dffc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.physical.impl.filter;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.categories.UnlikelyTest;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
index 499ac5b..750276c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertTrue;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -55,7 +55,7 @@ public class TestSimpleFilter extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/filter/test1.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/filter/test1.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -77,7 +77,7 @@ public class TestSimpleFilter extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/filter/test_sv4.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/filter/test_sv4.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
index fc954be..1dc07df 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -17,35 +17,38 @@
******************************************************************************/
package org.apache.drill.exec.physical.impl.flatten;
-import static org.apache.commons.io.FileUtils.deleteQuietly;
-import static org.apache.drill.TestBuilder.listOf;
-import static org.apache.drill.TestBuilder.mapOf;
+import static org.apache.drill.test.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
import static org.junit.Assert.assertEquals;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.categories.UnlikelyTest;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.fn.interp.TestConstantFolding;
import org.apache.drill.exec.store.easy.json.JSONRecordReader;
import org.apache.drill.exec.util.JsonStringHashMap;
+import org.apache.drill.test.SubDirTestWatcher;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TemporaryFolder;
import com.google.common.collect.Lists;
@Category(OperatorTest.class)
public class TestFlatten extends BaseTestQuery {
+ private static final Path TEST_DIR = Paths.get("test");
+
/**
* enable this if you have the following files:
* - /tmp/yelp_academic_dataset_business.json
@@ -55,23 +58,33 @@ public class TestFlatten extends BaseTestQuery {
*/
public static boolean RUN_ADVANCED_TESTS = false;
+ private static File pathDir;
+
+ @BeforeClass
+ public static void initFile() {
+ pathDir = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(TEST_DIR)
+ .toFile();
+ }
+
@Rule
- public TemporaryFolder folder = new TemporaryFolder();
+ public final SubDirTestWatcher subDirTestWatcher =
+ new SubDirTestWatcher.Builder(dirTestWatcher.getRootDir())
+ .addSubDir(TEST_DIR)
+ .build();
@Test
public void testFlattenFailure() throws Exception {
- test("select flatten(complex), rownum from cp.`/store/json/test_flatten_mappify2.json`");
-// test("select complex, rownum from cp.`/store/json/test_flatten_mappify2.json`");
+ test("select flatten(complex), rownum from cp.`store/json/test_flatten_mappify2.json`");
}
@Test
@Category(UnlikelyTest.class)
public void testFlatten_Drill2162_complex() throws Exception {
- String path = folder.getRoot().toPath().toString();
-
String jsonRecords = BaseTestQuery.getFile("flatten/complex_transaction_example_data.json");
int numCopies = 700;
- new TestConstantFolding.SmallFileCreator(folder)
+ new TestConstantFolding.SmallFileCreator(pathDir)
.setRecord(jsonRecords)
.createFiles(1, numCopies, "json");
@@ -97,7 +110,7 @@ public class TestFlatten extends BaseTestQuery {
TestBuilder builder = testBuilder()
.sqlQuery("select uid, flatten(d.lst_lst[1]) lst1, flatten(d.lst_lst[0]) lst0, flatten(d.lst_lst) lst from " +
- "dfs.`" + path + "/bigfile/bigfile.json` d")
+ "dfs.`%s/bigfile/bigfile.json` d", TEST_DIR)
.unOrdered()
.baselineColumns("uid", "lst1", "lst0", "lst");
for (int i = 0; i < numCopies; i++) {
@@ -168,8 +181,6 @@ public class TestFlatten extends BaseTestQuery {
@Test
public void testFlatten_Drill2162_simple() throws Exception {
- String path = folder.getRoot().toPath().toString();
-
List<Long> inputList = Lists.newArrayList();
String jsonRecord = "{ \"int_list\" : [";
final int listSize = 30;
@@ -180,7 +191,7 @@ public class TestFlatten extends BaseTestQuery {
jsonRecord += listSize + "] }";
inputList.add((long) listSize);
int numRecords = 3000;
- new TestConstantFolding.SmallFileCreator(folder)
+ new TestConstantFolding.SmallFileCreator(pathDir)
.setRecord(jsonRecord)
.createFiles(1, numRecords, "json");
@@ -192,7 +203,7 @@ public class TestFlatten extends BaseTestQuery {
List<JsonStringHashMap<String, Object>> result = flatten(data, "int_list");
TestBuilder builder = testBuilder()
- .sqlQuery("select flatten(int_list) as int_list from dfs.`" + path + "/bigfile/bigfile.json`")
+ .sqlQuery("select flatten(int_list) as int_list from dfs.`%s/bigfile/bigfile.json`", TEST_DIR)
.unOrdered()
.baselineColumns("int_list");
@@ -207,7 +218,9 @@ public class TestFlatten extends BaseTestQuery {
@Test
@Category(UnlikelyTest.class)
public void drill1671() throws Exception{
- int rowCount = testSql("select * from (select count(*) as cnt from (select id, flatten(evnts1), flatten(evnts2), flatten(evnts3), flatten(evnts4), flatten(evnts5), flatten(evnts6), flatten(evnts7), flatten(evnts8), flatten(evnts9), flatten(evnts10), flatten(evnts11) from cp.`/flatten/many-arrays-50.json`)x )y where cnt = 2048");
+ int rowCount = testSql("select * from (select count(*) as cnt from (select id, flatten(evnts1), flatten(evnts2), flatten(evnts3), flatten(evnts4), " +
+ "flatten(evnts5), flatten(evnts6), flatten(evnts7), flatten(evnts8), flatten(evnts9), flatten(evnts10), flatten(evnts11) " +
+ "from cp.`flatten/many-arrays-50.json`)x )y where cnt = 2048");
assertEquals(rowCount, 1);
}
@@ -215,7 +228,7 @@ public class TestFlatten extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void drill3370() throws Exception {
testBuilder()
- .sqlQuery("select a from (select flatten(arr) as a from cp.`/flatten/drill-3370.json`) where a > 100")
+ .sqlQuery("select a from (select flatten(arr) as a from cp.`flatten/drill-3370.json`) where a > 100")
.unOrdered()
.baselineColumns("a")
.baselineValues(131l)
@@ -226,24 +239,22 @@ public class TestFlatten extends BaseTestQuery {
@Test
@Ignore("not yet fixed")
public void drill1660() throws Exception {
- test("select * from cp.`/flatten/empty-rm.json`");
+ test("select * from cp.`flatten/empty-rm.json`");
}
@Test // repeated list within a repeated map
@Category(UnlikelyTest.class)
public void drill1673() throws Exception {
- String path = folder.getRoot().toPath().toString();
-
String jsonRecords = BaseTestQuery.getFile("store/json/1673.json");
int numCopies = 25000;
- new TestConstantFolding.SmallFileCreator(folder)
+ new TestConstantFolding.SmallFileCreator(pathDir)
.setRecord(jsonRecords)
.createFiles(1, numCopies, "json");
TestBuilder builder = testBuilder()
.sqlQuery("select t.fixed_column as fixed_column, " +
"flatten(t.list_column) as list_col " +
- "from dfs.`" + path + "/bigfile/bigfile.json` as t")
+ "from dfs.`%s/bigfile/bigfile.json` as t", TEST_DIR)
.baselineColumns("fixed_column", "list_col")
.unOrdered();
Object map1 = mapOf("id1", "1",
@@ -263,7 +274,7 @@ public class TestFlatten extends BaseTestQuery {
@Test
@Category(UnlikelyTest.class)
public void drill1653() throws Exception{
- int rowCount = testSql("select * from (select sum(t.flat.`value`) as sm from (select id, flatten(kvgen(m)) as flat from cp.`/flatten/missing-map.json`)t) where sm = 10 ");
+ int rowCount = testSql("select * from (select sum(t.flat.`value`) as sm from (select id, flatten(kvgen(m)) as flat from cp.`flatten/missing-map.json`)t) where sm = 10 ");
assertEquals(1, rowCount);
}
@@ -271,7 +282,7 @@ public class TestFlatten extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void drill1652() throws Exception {
if(RUN_ADVANCED_TESTS){
- test("select uid, flatten(transactions) from dfs.`/tmp/bigfile.json`");
+ test("select uid, flatten(transactions) from dfs.`tmp/bigfile.json`");
}
}
@@ -282,7 +293,7 @@ public class TestFlatten extends BaseTestQuery {
"from (\n" +
" select userinfo.transaction.trans_id trans_id, max(userinfo.event.event_time) max_event_time\n" +
" from (\n" +
- " select uid, flatten(events) event, flatten(transactions) transaction from cp.`/flatten/single-user-transactions.json`\n" +
+ " select uid, flatten(events) event, flatten(transactions) transaction from cp.`flatten/single-user-transactions.json`\n" +
" ) userinfo\n" +
" where userinfo.transaction.trans_time >= userinfo.event.event_time\n" +
" group by userinfo.transaction.trans_id\n" +
@@ -290,7 +301,7 @@ public class TestFlatten extends BaseTestQuery {
"inner join\n" +
"(\n" +
" select uid, flatten(events) event\n" +
- " from cp.`/flatten/single-user-transactions.json`\n" +
+ " from cp.`flatten/single-user-transactions.json`\n" +
") event_info\n" +
"on transaction_info.max_event_time = event_info.event.event_time;");
}
@@ -299,25 +310,25 @@ public class TestFlatten extends BaseTestQuery {
public void testKVGenFlatten1() throws Exception {
// works - TODO and verify results
test("select flatten(kvgen(f1)) as monkey, x " +
- "from cp.`/store/json/test_flatten_mapify.json`");
+ "from cp.`store/json/test_flatten_mapify.json`");
}
@Test
public void testTwoFlattens() throws Exception {
// second re-write rule has been added to test the fixes together, this now runs
- test("select `integer`, `float`, x, flatten(z), flatten(l) from cp.`/jsoninput/input2_modified.json`");
+ test("select `integer`, `float`, x, flatten(z), flatten(l) from cp.`jsoninput/input2_modified.json`");
}
@Test
public void testFlattenRepeatedMap() throws Exception {
- test("select `integer`, `float`, x, flatten(z) from cp.`/jsoninput/input2.json`");
+ test("select `integer`, `float`, x, flatten(z) from cp.`jsoninput/input2.json`");
}
@Test
public void testFlattenKVGenFlatten() throws Exception {
// currently does not fail, but produces incorrect results, requires second re-write rule to split up expressions
// with complex outputs
- test("select `integer`, `float`, x, flatten(kvgen(flatten(z))) from cp.`/jsoninput/input2.json`");
+ test("select `integer`, `float`, x, flatten(kvgen(flatten(z))) from cp.`jsoninput/input2.json`");
}
@Test
@@ -325,7 +336,7 @@ public class TestFlatten extends BaseTestQuery {
// currently runs
// TODO - re-verify results by hand
if(RUN_ADVANCED_TESTS){
- test("select flatten(kvgen(visited_cellid_counts)) as mytb from dfs.`/tmp/mapkv.json`") ;
+ test("select flatten(kvgen(visited_cellid_counts)) as mytb from dfs.`tmp/mapkv.json`") ;
}
}
@@ -334,7 +345,7 @@ public class TestFlatten extends BaseTestQuery {
// WORKS!!
// TODO - hand verify results
test("select t2.key from (select t.monkey.`value` as val, t.monkey.key as key from (select flatten(kvgen(f1)) as monkey, x " +
- "from cp.`/store/json/test_flatten_mapify.json`) as t) as t2 where t2.val > 1");
+ "from cp.`store/json/test_flatten_mapify.json`) as t) as t2 where t2.val > 1");
}
@Test
@@ -347,7 +358,7 @@ public class TestFlatten extends BaseTestQuery {
// these types of problems are being solved more generally as we develp better support for chaning schema
if(RUN_ADVANCED_TESTS){
test("select celltbl.catl from (\n" +
- " select flatten(categories) catl from dfs.`/tmp/yelp_academic_dataset_business.json` b limit 100\n" +
+ " select flatten(categories) catl from dfs.`tmp/yelp_academic_dataset_business.json` b limit 100\n" +
" ) celltbl where celltbl.catl = 'Doctors'");
}
}
@@ -356,23 +367,22 @@ public class TestFlatten extends BaseTestQuery {
public void countAggFlattened() throws Exception {
if(RUN_ADVANCED_TESTS){
test("select celltbl.catl, count(celltbl.catl) from ( " +
- "select business_id, flatten(categories) catl from dfs.`/tmp/yelp_academic_dataset_business.json` b limit 100 " +
+ "select business_id, flatten(categories) catl from dfs.`tmp/yelp_academic_dataset_business.json` b limit 100 " +
") celltbl group by celltbl.catl limit 10 ");
}
}
-
@Test
public void flattenAndAdditionalColumn() throws Exception {
if(RUN_ADVANCED_TESTS){
- test("select business_id, flatten(categories) from dfs.`/tmp/yelp_academic_dataset_business.json` b");
+ test("select business_id, flatten(categories) from dfs.`tmp/yelp_academic_dataset_business.json` b");
}
}
@Test
public void testFailingFlattenAlone() throws Exception {
if(RUN_ADVANCED_TESTS){
- test("select flatten(categories) from dfs.`/tmp/yelp_academic_dataset_business.json` b ");
+ test("select flatten(categories) from dfs.`tmp/yelp_academic_dataset_business.json` b ");
}
}
@@ -380,7 +390,7 @@ public class TestFlatten extends BaseTestQuery {
public void testDistinctAggrFlattened() throws Exception {
if(RUN_ADVANCED_TESTS){
test(" select distinct(celltbl.catl) from (\n" +
- " select flatten(categories) catl from dfs.`/tmp/yelp_academic_dataset_business.json` b\n" +
+ " select flatten(categories) catl from dfs.`tmp/yelp_academic_dataset_business.json` b\n" +
" ) celltbl");
}
@@ -390,24 +400,24 @@ public class TestFlatten extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void testDrill1665() throws Exception {
if(RUN_ADVANCED_TESTS){
- test("select id, flatten(evnts) as rpt from dfs.`/tmp/drill1665.json`");
+ test("select id, flatten(evnts) as rpt from dfs.`tmp/drill1665.json`");
}
}
@Test
public void testFlattenComplexRepeatedMap() throws Exception {
- test("select a, flatten(r_map_1), flatten(r_map_2) from cp.`/store/json/complex_repeated_map.json`");
+ test("select a, flatten(r_map_1), flatten(r_map_2) from cp.`store/json/complex_repeated_map.json`");
}
@Test
public void testFlatten2_levelRepeatedMap() throws Exception {
- test("select flatten(rm) from cp.`/store/json/2_level_repeated_map.json`");
+ test("select flatten(rm) from cp.`store/json/2_level_repeated_map.json`");
}
@Test
public void testDrill_1770() throws Exception {
- test("select flatten(sub.fk.`value`) from (select flatten(kvgen(map)) fk from cp.`/store/json/nested_repeated_map.json`) sub");
+ test("select flatten(sub.fk.`value`) from (select flatten(kvgen(map)) fk from cp.`store/json/nested_repeated_map.json`) sub");
}
@@ -455,7 +465,7 @@ public class TestFlatten extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void testDrill_2013() throws Exception {
testBuilder()
- .sqlQuery("select flatten(complex), rownum from cp.`/store/json/test_flatten_mappify2.json` where rownum > 5")
+ .sqlQuery("select flatten(complex), rownum from cp.`store/json/test_flatten_mappify2.json` where rownum > 5")
.expectsEmptyResultSet()
.build().run();
}
@@ -480,41 +490,25 @@ public class TestFlatten extends BaseTestQuery {
@Test // see DRILL-2146
@Category(UnlikelyTest.class)
public void testFalttenWithStar() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/sample.json").toURI().toString();
- String q1 = String.format("select *, flatten(j.topping) tt, flatten(j.batters.batter) bb, j.id " +
- "from dfs_test.`%s` j " +
- "where j.type = 'donut'", root);
- String q2 = String.format("select *, flatten(j.topping) tt, flatten(j.batters.batter) bb, j.id, j.type " +
- "from dfs_test.`%s` j " +
- "where j.type = 'donut'", root);
-
- test(q1);
- test(q2);
+ test("select *, flatten(j.topping) tt, flatten(j.batters.batter) bb, j.id " +
+ "from cp.`store/text/sample.json` j where j.type = 'donut'");
+ test("select *, flatten(j.topping) tt, flatten(j.batters.batter) bb, j.id, j.type " +
+ "from cp.`store/text/sample.json` j where j.type = 'donut'");
}
@Test // see DRILL-2012
@Category(UnlikelyTest.class)
public void testMultipleFalttenWithWhereClause() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/sample.json").toURI().toString();
- String q1 = String.format("select flatten(j.topping) tt " +
- "from dfs_test.`%s` j " +
- "where j.type = 'donut'", root);
- String q2 = String.format("select j.type, flatten(j.topping) tt " +
- "from dfs_test.`%s` j " +
- "where j.type = 'donut'", root);
-
- test(q1);
- test(q2);
+ test("select flatten(j.topping) tt from cp.`store/text/sample.json` j where j.type = 'donut'");
+ test("select j.type, flatten(j.topping) tt from cp.`store/text/sample.json` j where j.type = 'donut'");
}
@Test //DRILL-2099
@Category(UnlikelyTest.class)
public void testFlattenAfterSort() throws Exception {
- String query = "select flatten(s1.rms.rptd) rptds from " +
- "(select d.uid uid, flatten(d.map.rm) rms from cp.`jsoninput/flatten_post_sort.json` d order by d.uid) s1";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select flatten(s1.rms.rptd) rptds from " +
+ "(select d.uid uid, flatten(d.map.rm) rms from cp.`jsoninput/flatten_post_sort.json` d order by d.uid) s1")
.unOrdered()
.jsonBaselineFile("flatten/drill-2099-result.json")
.go();
@@ -523,12 +517,10 @@ public class TestFlatten extends BaseTestQuery {
@Test //DRILL-2268
@Category(UnlikelyTest.class)
public void testFlattenAfterJoin1() throws Exception {
- String query = "select flatten(sub1.events) flat_events from "+
- "(select t1.events events from cp.`complex/json/flatten_join.json` t1 "+
- "inner join cp.`complex/json/flatten_join.json` t2 on t1.id=t2.id) sub1";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select flatten(sub1.events) flat_events from "+
+ "(select t1.events events from cp.`complex/json/flatten_join.json` t1 "+
+ "inner join cp.`complex/json/flatten_join.json` t2 on t1.id=t2.id) sub1")
.unOrdered()
.jsonBaselineFile("complex/drill-2268-1-result.json")
.go();
@@ -537,11 +529,9 @@ public class TestFlatten extends BaseTestQuery {
@Test //DRILL-2268
@Category(UnlikelyTest.class)
public void testFlattenAfterJoin2() throws Exception {
- String query = "select flatten(t1.events) flat_events from cp.`complex/json/flatten_join.json` t1 " +
- "inner join cp.`complex/json/flatten_join.json` t2 on t1.id=t2.id";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select flatten(t1.events) flat_events from cp.`complex/json/flatten_join.json` t1 " +
+ "inner join cp.`complex/json/flatten_join.json` t2 on t1.id=t2.id")
.unOrdered()
.jsonBaselineFile("complex/drill-2268-2-result.json")
.go();
@@ -550,12 +540,10 @@ public class TestFlatten extends BaseTestQuery {
@Test //DRILL-2268
@Category(UnlikelyTest.class)
public void testFlattenAfterJoin3() throws Exception {
- String query = "select flatten(sub1.lst_lst) flat_lst_lst from "+
- "(select t1.lst_lst lst_lst from cp.`complex/json/flatten_join.json` t1 "+
- "inner join cp.`complex/json/flatten_join.json` t2 on t1.id=t2.id) sub1";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select flatten(sub1.lst_lst) flat_lst_lst from "+
+ "(select t1.lst_lst lst_lst from cp.`complex/json/flatten_join.json` t1 "+
+ "inner join cp.`complex/json/flatten_join.json` t2 on t1.id=t2.id) sub1")
.unOrdered()
.jsonBaselineFile("complex/drill-2268-3-result.json")
.go();
@@ -563,51 +551,40 @@ public class TestFlatten extends BaseTestQuery {
@Test
public void testFlattenWithScalarFunc() throws Exception {
- final String query = "select flatten(t.l) + 1 as c1 from cp.`/jsoninput/input2.json` t";
-
testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .baselineColumns("c1")
- .baselineValues(5L)
- .baselineValues(3L)
- .baselineValues(5L)
- .baselineValues(3L)
- .baselineValues(5L)
- .baselineValues(3L)
- .go();
-
+ .sqlQuery("select flatten(t.l) + 1 as c1 from cp.`jsoninput/input2.json` t")
+ .unOrdered()
+ .baselineColumns("c1")
+ .baselineValues(5L)
+ .baselineValues(3L)
+ .baselineValues(5L)
+ .baselineValues(3L)
+ .baselineValues(5L)
+ .baselineValues(3L)
+ .go();
}
@Test
public void testFlattenOnEmptyArrayAndNestedMap() throws Exception {
- File path = new File(BaseTestQuery.getTempDir("json/input"));
- try {
- path.mkdirs();
- String pathString = path.toPath().toString();
-
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(path, "empty_arrays.json")))) {
- writer.write("{\"a\" : {\"a1\" : \"a1\"}, \"b\" : [1]}\n");
- for (int i = 0; i < JSONRecordReader.DEFAULT_ROWS_PER_BATCH; i++) {
- writer.write("{\"a\" : {\"a1\" : \"a1\"}, \"b\" : [], \"c\" : 1}\n");
- }
- writer.write("{\"a\" : {\"a1\" : \"a1\"}, \"b\" : [1], \"c\" : 1}");
+ final Path path = Paths.get("json", "input");
+ final File dir = dirTestWatcher.makeRootSubDir(path);
+
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dir, "empty_arrays.json")))) {
+ writer.write("{\"a\" : {\"a1\" : \"a1\"}, \"b\" : [1]}\n");
+ for (int i = 0; i < JSONRecordReader.DEFAULT_ROWS_PER_BATCH; i++) {
+ writer.write("{\"a\" : {\"a1\" : \"a1\"}, \"b\" : [], \"c\" : 1}\n");
}
+ writer.write("{\"a\" : {\"a1\" : \"a1\"}, \"b\" : [1], \"c\" : 1}");
+ }
- String query = "select typeof(t1.a.a1) as col from " +
- "(select t.*, flatten(t.b) as b from dfs_test.`%s/empty_arrays.json` t where t.c is not null) t1";
+ String query = "select typeof(t1.a.a1) as col from " +
+ "(select t.*, flatten(t.b) as b from dfs.`%s/empty_arrays.json` t where t.c is not null) t1";
- testBuilder()
- .sqlQuery(query, pathString)
- .unOrdered()
- .baselineColumns("col")
- .baselineValues("VARCHAR")
- .go();
-
- } finally {
- deleteQuietly(path);
- }
+ testBuilder()
+ .sqlQuery(query, path)
+ .unOrdered()
+ .baselineColumns("col")
+ .baselineValues("VARCHAR")
+ .go();
}
-
}
-
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlattenPlanning.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlattenPlanning.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlattenPlanning.java
index 75537af..1a5117f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlattenPlanning.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlattenPlanning.java
@@ -30,7 +30,7 @@ public class TestFlattenPlanning extends PlanTestBase {
public void testFlattenPlanningAvoidUnnecessaryProject() throws Exception {
// Because of Java7 vs Java8 map ordering differences, we check for both cases
// See DRILL-4331 for details
- testPlanMatchingPatterns("select flatten(complex), rownum from cp.`/store/json/test_flatten_mappify2.json`",
+ testPlanMatchingPatterns("select flatten(complex), rownum from cp.`store/json/test_flatten_mappify2.json`",
new String[]{"\\QProject(EXPR$0=[$1], rownum=[$0])\\E|\\QProject(EXPR$0=[$0], rownum=[$1])\\E"},
new String[]{"\\QProject(EXPR$0=[$0], EXPR$1=[$1], EXPR$3=[$1])\\E|\\QProject(EXPR$0=[$1], EXPR$1=[$0], EXPR$3=[$0])\\E"});
}
@@ -40,7 +40,7 @@ public class TestFlattenPlanning extends PlanTestBase {
final String query =
" select comp, rownum " +
" from (select flatten(complex) comp, rownum " +
- " from cp.`/store/json/test_flatten_mappify2.json`) " +
+ " from cp.`store/json/test_flatten_mappify2.json`) " +
" where comp > 1 " + // should not be pushed down
" and rownum = 100"; // should be pushed down.
@@ -54,7 +54,7 @@ public class TestFlattenPlanning extends PlanTestBase {
final String query =
" select comp, rownum " +
" from (select flatten(complex) comp, rownum " +
- " from cp.`/store/json/test_flatten_mappify2.json`) " +
+ " from cp.`store/json/test_flatten_mappify2.json`) " +
" where comp > 1 " + // should NOT be pushed down
" OR rownum = 100"; // should NOT be pushed down.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
index a9fb80d..cee84dc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
@@ -26,8 +26,8 @@ import java.util.List;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.common.util.DrillFileUtils;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -70,7 +70,7 @@ public class TestHashJoin extends PopUnitTestBase {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(physicalPlan), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(physicalPlan), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -120,9 +120,9 @@ public class TestHashJoin extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/hash_join.json"), Charsets.UTF_8)
- .replace("#{TEST_FILE_1}", FileUtils.getResourceAsFile("/build_side_input.json").toURI().toString())
- .replace("#{TEST_FILE_2}", FileUtils.getResourceAsFile("/probe_side_input.json").toURI().toString()));
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/hash_join.json"), Charsets.UTF_8)
+ .replace("#{TEST_FILE_1}", DrillFileUtils.getResourceAsFile("/build_side_input.json").toURI().toString())
+ .replace("#{TEST_FILE_2}", DrillFileUtils.getResourceAsFile("/probe_side_input.json").toURI().toString()));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
@@ -163,7 +163,7 @@ public class TestHashJoin extends PopUnitTestBase {
bit.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/hj_exchanges.json"), Charsets.UTF_8));
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/hj_exchanges.json"), Charsets.UTF_8));
int count = 0;
for (final QueryDataBatch b : results) {
@@ -191,9 +191,9 @@ public class TestHashJoin extends PopUnitTestBase {
bit.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/hj_multi_condition_join.json"), Charsets.UTF_8)
- .replace("#{TEST_FILE_1}", FileUtils.getResourceAsFile("/build_side_input.json").toURI().toString())
- .replace("#{TEST_FILE_2}", FileUtils.getResourceAsFile("/probe_side_input.json").toURI().toString()));
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/hj_multi_condition_join.json"), Charsets.UTF_8)
+ .replace("#{TEST_FILE_1}", DrillFileUtils.getResourceAsFile("/build_side_input.json").toURI().toString())
+ .replace("#{TEST_FILE_2}", DrillFileUtils.getResourceAsFile("/probe_side_input.json").toURI().toString()));
final RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
@@ -237,7 +237,7 @@ public class TestHashJoin extends PopUnitTestBase {
bit.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/hj_exchanges1.json"), Charsets.UTF_8));
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/hj_exchanges1.json"), Charsets.UTF_8));
int count = 0;
for (final QueryDataBatch b : results) {
@@ -262,7 +262,7 @@ public class TestHashJoin extends PopUnitTestBase {
bit1.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/hashJoinExpr.json"), Charsets.UTF_8));
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/hashJoinExpr.json"), Charsets.UTF_8));
int count = 0;
for (final QueryDataBatch b : results) {
if (b.getHeader().getRowCount() != 0) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java
index 205daa2..49aefe6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java
@@ -19,7 +19,7 @@
package org.apache.drill.exec.physical.impl.join;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.categories.UnlikelyTest;
import org.junit.AfterClass;
@@ -141,32 +141,23 @@ public class TestHashJoinAdvanced extends BaseTestQuery {
@Test //DRILL-2197 Left Join with complex type in projection
@Category(UnlikelyTest.class)
public void testJoinWithMapAndDotField() throws Exception {
- File directory = new File(BaseTestQuery.getTempDir("json/input"));
- try {
- directory.mkdirs();
- String fileName = "table.json";
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
- writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
- }
-
- String query = String.format("select t1.m.`a.b` as a,\n" +
- "t2.m.a.b as b,\n" +
- "t1.m['a.b'] as c,\n" +
- "t2.rk.q as d,\n" +
- "t1.`rk.q` as e\n" +
- "from dfs_test.`%1$s/%2$s` t1,\n" +
- "dfs_test.`%1$s/%2$s` t2\n" +
- "where t1.m.`a.b`=t2.m.`a.b` and t1.m.a.b=t2.m.a.b",
- directory.toPath().toString(), fileName);
- testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .baselineColumns("a", "b", "c", "d", "e")
- .baselineValues("1", "2", "1", null, "a")
- .go();
-
- } finally {
- org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ String fileName = "table.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), fileName)))) {
+ writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
}
+
+ testBuilder()
+ .sqlQuery("select t1.m.`a.b` as a,\n" +
+ "t2.m.a.b as b,\n" +
+ "t1.m['a.b'] as c,\n" +
+ "t2.rk.q as d,\n" +
+ "t1.`rk.q` as e\n" +
+ "from dfs.`%1$s` t1,\n" +
+ "dfs.`%1$s` t2\n" +
+ "where t1.m.`a.b`=t2.m.`a.b` and t1.m.a.b=t2.m.a.b", fileName)
+ .unOrdered()
+ .baselineColumns("a", "b", "c", "d", "e")
+ .baselineValues("1", "2", "1", null, "a")
+ .go();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
index 25e8ba5..1c16fa4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -68,7 +68,7 @@ public class TestMergeJoin extends PopUnitTestBase {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/join/merge_join.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/join/merge_join.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -118,9 +118,9 @@ public class TestMergeJoin extends PopUnitTestBase {
new StoragePluginRegistryImpl(bitContext));
final PhysicalPlan plan = reader.readPhysicalPlan(
Files.toString(
- FileUtils.getResourceAsFile("/join/merge_single_batch.json"), Charsets.UTF_8)
- .replace("#{LEFT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString())
- .replace("#{RIGHT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString()));
+ DrillFileUtils.getResourceAsFile("/join/merge_single_batch.json"), Charsets.UTF_8)
+ .replace("#{LEFT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString())
+ .replace("#{RIGHT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString()));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -169,9 +169,9 @@ public class TestMergeJoin extends PopUnitTestBase {
new StoragePluginRegistryImpl(bitContext));
final PhysicalPlan plan = reader.readPhysicalPlan(
Files.toString(
- FileUtils.getResourceAsFile("/join/merge_inner_single_batch.json"), Charsets.UTF_8)
- .replace("#{LEFT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString())
- .replace("#{RIGHT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString()));
+ DrillFileUtils.getResourceAsFile("/join/merge_inner_single_batch.json"), Charsets.UTF_8)
+ .replace("#{LEFT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString())
+ .replace("#{RIGHT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString()));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -220,9 +220,9 @@ public class TestMergeJoin extends PopUnitTestBase {
new StoragePluginRegistryImpl(bitContext));
final PhysicalPlan plan = reader.readPhysicalPlan(
Files.toString(
- FileUtils.getResourceAsFile("/join/merge_multi_batch.json"), Charsets.UTF_8)
- .replace("#{LEFT_FILE}", FileUtils.getResourceAsFile("/join/merge_multi_batch.left.json").toURI().toString())
- .replace("#{RIGHT_FILE}", FileUtils.getResourceAsFile("/join/merge_multi_batch.right.json").toURI().toString()));
+ DrillFileUtils.getResourceAsFile("/join/merge_multi_batch.json"), Charsets.UTF_8)
+ .replace("#{LEFT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_multi_batch.left.json").toURI().toString())
+ .replace("#{RIGHT_FILE}", DrillFileUtils.getResourceAsFile("/join/merge_multi_batch.right.json").toURI().toString()));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -265,7 +265,7 @@ public class TestMergeJoin extends PopUnitTestBase {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/join/join_batchsize.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/join/join_batchsize.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -290,7 +290,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bit1.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/merge_join_empty_batch.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/merge_join_empty_batch.json"),
Charsets.UTF_8)
.replace("${JOIN_TYPE}", "INNER"));
int count = 0;
@@ -314,7 +314,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bit1.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/merge_join_empty_batch.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/merge_join_empty_batch.json"),
Charsets.UTF_8)
.replace("${JOIN_TYPE}", "LEFT"));
int count = 0;
@@ -338,7 +338,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bit1.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/merge_join_empty_batch.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/merge_join_empty_batch.json"),
Charsets.UTF_8)
.replace("${JOIN_TYPE}", "RIGHT"));
int count = 0;
@@ -362,7 +362,7 @@ public class TestMergeJoin extends PopUnitTestBase {
bit1.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/mergeJoinExpr.json"), Charsets.UTF_8));
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/mergeJoinExpr.json"), Charsets.UTF_8));
int count = 0;
for (final QueryDataBatch b : results) {
if (b.getHeader().getRowCount() != 0) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java
index d01b030..310b331 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinAdvanced.java
@@ -17,12 +17,11 @@
*/
package org.apache.drill.exec.physical.impl.join;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.apache.drill.test.OperatorFixture;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -35,10 +34,17 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.Random;
@Category(OperatorTest.class)
public class TestMergeJoinAdvanced extends BaseTestQuery {
+ private static final String LEFT = "merge-join-left.json";
+ private static final String RIGHT = "merge-join-right.json";
+
+ private static File leftFile;
+ private static File rightFile;
+
@Rule
public final TestRule TIMEOUT = TestTools.getTimeoutRule(120000); // Longer timeout than usual.
@@ -46,6 +52,11 @@ public class TestMergeJoinAdvanced extends BaseTestQuery {
@BeforeClass
public static void disableMergeJoin() throws Exception {
test("alter session set `planner.enable_hashjoin` = false");
+
+ leftFile = new File(dirTestWatcher.getRootDir(), LEFT);
+ rightFile = new File(dirTestWatcher.getRootDir(), RIGHT);
+
+ dirTestWatcher.copyResourceToRoot(Paths.get("join"));
}
@AfterClass
@@ -100,11 +111,8 @@ public class TestMergeJoinAdvanced extends BaseTestQuery {
setSessionOption(ExecConstants.SLICE_TARGET, "1");
setSessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, "23");
- final String TEST_RES_PATH = TestTools.getWorkingPath() + "/src/test/resources";
-
try {
- test("select * from dfs_test.`%s/join/j1` j1 left outer join dfs_test.`%s/join/j2` j2 on (j1.c_varchar = j2.c_varchar)",
- TEST_RES_PATH, TEST_RES_PATH);
+ test("select * from dfs.`join/j1` j1 left outer join dfs.`join/j2` j2 on (j1.c_varchar = j2.c_varchar)");
} finally {
test("ALTER SESSION RESET ALL");
}
@@ -131,13 +139,11 @@ public class TestMergeJoinAdvanced extends BaseTestQuery {
private static void testMultipleBatchJoin(final long right, final long left,
final String joinType, final long expected) throws Exception {
- final String leftSide = BaseTestQuery.getTempDir("merge-join-left.json");
- final String rightSide = BaseTestQuery.getTempDir("merge-join-right.json");
- final BufferedWriter leftWriter = new BufferedWriter(new FileWriter(new File(leftSide)));
- final BufferedWriter rightWriter = new BufferedWriter(new FileWriter(new File(rightSide)));
+ final BufferedWriter leftWriter = new BufferedWriter(new FileWriter(leftFile));
+ final BufferedWriter rightWriter = new BufferedWriter(new FileWriter(rightFile));
generateData(leftWriter, rightWriter, left, right);
- final String query1 = String.format("select count(*) c1 from dfs_test.`%s` L %s join dfs_test.`%s` R on L.k=R.k1",
- leftSide, joinType, rightSide);
+ final String query1 = String.format("select count(*) c1 from dfs.`%s` L %s join dfs.`%s` R on L.k=R.k1",
+ LEFT, joinType, RIGHT);
testBuilder()
.sqlQuery(query1)
.optionSettingQueriesForTestQuery("alter session set `planner.enable_hashjoin` = false")
@@ -218,10 +224,8 @@ public class TestMergeJoinAdvanced extends BaseTestQuery {
@Test
public void testDrill4196() throws Exception {
- final String leftSide = BaseTestQuery.getTempDir("merge-join-left.json");
- final String rightSide = BaseTestQuery.getTempDir("merge-join-right.json");
- final BufferedWriter leftWriter = new BufferedWriter(new FileWriter(new File(leftSide)));
- final BufferedWriter rightWriter = new BufferedWriter(new FileWriter(new File(rightSide)));
+ final BufferedWriter leftWriter = new BufferedWriter(new FileWriter(leftFile));
+ final BufferedWriter rightWriter = new BufferedWriter(new FileWriter(rightFile));
// output batch is 32k, create 60k left batch
leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 9999, 9999));
@@ -239,8 +243,8 @@ public class TestMergeJoinAdvanced extends BaseTestQuery {
leftWriter.close();
rightWriter.close();
- final String query1 = String.format("select count(*) c1 from dfs_test.`%s` L %s join dfs_test.`%s` R on L.k=R.k1",
- leftSide, "inner", rightSide);
+ final String query1 = String.format("select count(*) c1 from dfs.`%s` L %s join dfs.`%s` R on L.k=R.k1",
+ LEFT, "inner", RIGHT);
testBuilder()
.sqlQuery(query1)
.optionSettingQueriesForTestQuery("alter session set `planner.enable_hashjoin` = false")
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java
index 48ad5f1..54b7bea 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinMulCondition.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.categories.SlowTest;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.common.util.DrillFileUtils;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
@@ -60,7 +60,7 @@ public class TestMergeJoinMulCondition extends PopUnitTestBase {
bit1.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/mj_multi_condition.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/mj_multi_condition.json"),
Charsets.UTF_8));
int count = 0;
for (QueryDataBatch b : results) {
@@ -85,7 +85,7 @@ public class TestMergeJoinMulCondition extends PopUnitTestBase {
bit1.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/merge_join_nullkey.json"), Charsets.UTF_8).replace("${JOIN_TYPE}", "INNER"));
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/merge_join_nullkey.json"), Charsets.UTF_8).replace("${JOIN_TYPE}", "INNER"));
int count = 0;
for (QueryDataBatch b : results) {
if (b.getHeader().getRowCount() != 0) {
@@ -110,7 +110,7 @@ public class TestMergeJoinMulCondition extends PopUnitTestBase {
bit1.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/join/merge_join_nullkey.json"), Charsets.UTF_8).replace("${JOIN_TYPE}", "LEFT"));
+ Files.toString(DrillFileUtils.getResourceAsFile("/join/merge_join_nullkey.json"), Charsets.UTF_8).replace("${JOIN_TYPE}", "LEFT"));
int count = 0;
for (QueryDataBatch b : results) {
if (b.getHeader().getRowCount() != 0) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinWithSchemaChanges.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinWithSchemaChanges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinWithSchemaChanges.java
index 86a136c..5136628 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinWithSchemaChanges.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoinWithSchemaChanges.java
@@ -17,56 +17,78 @@
*/
package org.apache.drill.exec.physical.impl.join;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.TestBuilder;
+import org.apache.drill.test.SubDirTestWatcher;
+import org.junit.BeforeClass;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
+import java.nio.file.Path;
+import java.nio.file.Paths;
@Category(OperatorTest.class)
public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
+ public static final Path LEFT_DIR = Paths.get("mergejoin-schemachanges-left");
+ public static final Path RIGHT_DIR = Paths.get("mergejoin-schemachanges-right");
+
+ private static File leftDir;
+ private static File rightDir;
+
+ @BeforeClass
+ public static void getDirs() {
+ leftDir = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(LEFT_DIR)
+ .toFile();
+ rightDir = dirTestWatcher.getRootDir()
+ .toPath()
+ .resolve(RIGHT_DIR)
+ .toFile();
+ }
+
+ @Rule
+ public final SubDirTestWatcher subDirTestWatcher = new SubDirTestWatcher.Builder(dirTestWatcher.getRootDir())
+ .addSubDir(LEFT_DIR)
+ .addSubDir(RIGHT_DIR)
+ .build();
@Test
- //@Ignore
public void testNumericTypes() throws Exception {
- final File left_dir = new File(BaseTestQuery.getTempDir("mergejoin-schemachanges-left"));
- final File right_dir = new File(BaseTestQuery.getTempDir("mergejoin-schemachanges-right"));
- left_dir.mkdirs();
- right_dir.mkdirs();
-
// First create data for numeric types.
// left side int and float vs right side float
- BufferedWriter writer = new BufferedWriter(new FileWriter(new File(left_dir, "l1.json")));
+ BufferedWriter writer = new BufferedWriter(new FileWriter(new File(leftDir, "l1.json")));
for (int i = 0; i < 5000; ++i) {
writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(left_dir, "l2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(leftDir, "l2.json")));
for (int i = 1000; i < 6000; ++i) {
writer.write(String.format("{ \"kl\" : %f , \"vl\": %f }\n", (float) i, (float) i));
}
writer.close();
// right side is int and float
- writer = new BufferedWriter(new FileWriter(new File(right_dir, "r1.json")));
+ writer = new BufferedWriter(new FileWriter(new File(rightDir, "r1.json")));
for (int i = 2000; i < 7000; ++i) {
writer.write(String.format("{ \"kr\" : %d , \"vr\": %d }\n", i, i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(right_dir, "r2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(rightDir, "r2.json")));
for (int i = 3000; i < 8000; ++i) {
writer.write(String.format("{ \"kr\" : %f, \"vr\": %f }\n", (float) i, (float) i));
}
writer.close();
// INNER JOIN
- String query = String.format("select * from dfs_test.`%s` L %s join dfs_test.`%s` R on L.kl=R.kr",
- left_dir.toPath().toString(), "inner", right_dir.toPath().toString());
+ String query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr",
+ LEFT_DIR, "inner", RIGHT_DIR);
TestBuilder builder = testBuilder()
.sqlQuery(query)
@@ -92,8 +114,8 @@ public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
builder.go();
// LEFT JOIN
- query = String.format("select * from dfs_test.`%s` L %s join dfs_test.`%s` R on L.kl=R.kr",
- left_dir.toPath().toString(), "left", right_dir.toPath().toString());
+ query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr",
+ LEFT_DIR, "left", RIGHT_DIR);
builder = testBuilder()
.sqlQuery(query)
@@ -125,40 +147,34 @@ public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
}
@Test
- //@Ignore
public void testNumericStringTypes() throws Exception {
- final File left_dir = new File(BaseTestQuery.getTempDir("mergejoin-schemachanges-left"));
- final File right_dir = new File(BaseTestQuery.getTempDir("mergejoin-schemachanges-right"));
- left_dir.mkdirs();
- right_dir.mkdirs();
-
// left side int and strings
- BufferedWriter writer = new BufferedWriter(new FileWriter(new File(left_dir, "l1.json")));
+ BufferedWriter writer = new BufferedWriter(new FileWriter(new File(leftDir, "l1.json")));
for (int i = 0; i < 5000; ++i) {
writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(left_dir, "l2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(leftDir, "l2.json")));
for (int i = 1000; i < 6000; ++i) {
writer.write(String.format("{ \"kl\" : \"%s\" , \"vl\": \"%s\" }\n", i, i));
}
writer.close();
// right side is float and strings
- writer = new BufferedWriter(new FileWriter(new File(right_dir, "r1.json")));
+ writer = new BufferedWriter(new FileWriter(new File(rightDir, "r1.json")));
for (int i = 2000; i < 7000; ++i) {
writer.write(String.format("{ \"kr\" : %f , \"vr\": %f }\n", (float)i, (float)i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(right_dir, "r2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(rightDir, "r2.json")));
for (int i = 3000; i < 8000; ++i) {
writer.write(String.format("{ \"kr\" : \"%s\", \"vr\": \"%s\" }\n", i, i));
}
writer.close();
// INNER JOIN
- String query = String.format("select * from dfs_test.`%s` L %s join dfs_test.`%s` R on L.kl=R.kr",
- left_dir.toPath().toString(), "inner", right_dir.toPath().toString());
+ String query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr",
+ LEFT_DIR, "inner", RIGHT_DIR);
TestBuilder builder = testBuilder()
.sqlQuery(query)
@@ -176,8 +192,8 @@ public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
builder.go();
// RIGHT JOIN
- query = String.format("select * from dfs_test.`%s` L %s join dfs_test.`%s` R on L.kl=R.kr",
- left_dir.toPath().toString(), "right", right_dir.toPath().toString());
+ query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr",
+ LEFT_DIR, "right", RIGHT_DIR);
builder = testBuilder()
.sqlQuery(query)
@@ -205,54 +221,47 @@ public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
@Ignore("DRILL-5612")
@Test
public void testMissingAndNewColumns() throws Exception {
- final File left_dir = new File(BaseTestQuery.getTempDir("mergejoin-schemachanges-left"));
- final File right_dir = new File(BaseTestQuery.getTempDir("mergejoin-schemachanges-right"));
- left_dir.mkdirs();
- right_dir.mkdirs();
- System.out.println(left_dir);
- System.out.println(right_dir);
-
// missing column kl
- BufferedWriter writer = new BufferedWriter(new FileWriter(new File(left_dir, "l1.json")));
+ BufferedWriter writer = new BufferedWriter(new FileWriter(new File(leftDir, "l1.json")));
for (int i = 0; i < 50; ++i) {
writer.write(String.format("{ \"kl1\" : %d , \"vl1\": %d }\n", i, i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(left_dir, "l2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(leftDir, "l2.json")));
for (int i = 50; i < 100; ++i) {
writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(left_dir, "l3.json")));
+ writer = new BufferedWriter(new FileWriter(new File(leftDir, "l3.json")));
for (int i = 100; i < 150; ++i) {
writer.write(String.format("{ \"kl2\" : %d , \"vl2\": %d }\n", i, i));
}
writer.close();
// right missing column kr
- writer = new BufferedWriter(new FileWriter(new File(right_dir, "r1.json")));
+ writer = new BufferedWriter(new FileWriter(new File(rightDir, "r1.json")));
for (int i = 0; i < 50; ++i) {
writer.write(String.format("{ \"kr1\" : %f , \"vr1\": %f }\n", (float)i, (float)i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(right_dir, "r2.json")));
+ writer = new BufferedWriter(new FileWriter(new File(rightDir, "r2.json")));
for (int i = 50; i < 100; ++i) {
writer.write(String.format("{ \"kr\" : %f , \"vr\": %f }\n", (float)i, (float)i));
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(right_dir, "r3.json")));
+ writer = new BufferedWriter(new FileWriter(new File(rightDir, "r3.json")));
for (int i = 100; i < 150; ++i) {
writer.write(String.format("{ \"kr2\" : %f , \"vr2\": %f }\n", (float)i, (float)i));
}
writer.close();
// INNER JOIN
- String query = String.format("select * from dfs_test.`%s` L %s join dfs_test.`%s` R on L.kl=R.kr",
- left_dir.toPath().toString(), "inner", right_dir.toPath().toString());
+ String query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr",
+ LEFT_DIR, "inner", RIGHT_DIR);
TestBuilder builder = testBuilder()
.sqlQuery(query)
@@ -266,8 +275,8 @@ public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
builder.go();
// LEFT JOIN
- query = String.format("select * from dfs_test.`%s` L %s join dfs_test.`%s` R on L.kl=R.kr",
- left_dir.toPath().toString(), "left", right_dir.toPath().toString());
+ query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr",
+ LEFT_DIR, "left", RIGHT_DIR);
builder = testBuilder()
.sqlQuery(query)
@@ -287,8 +296,8 @@ public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
builder.go();
// RIGHT JOIN
- query = String.format("select * from dfs_test.`%s` L %s join dfs_test.`%s` R on L.kl=R.kr",
- left_dir.toPath().toString(), "right", right_dir.toPath().toString());
+ query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr",
+ LEFT_DIR, "right", RIGHT_DIR);
builder = testBuilder()
.sqlQuery(query)
@@ -309,16 +318,8 @@ public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
}
@Test
- //@Ignore
public void testOneSideSchemaChanges() throws Exception {
- final File left_dir = new File(BaseTestQuery.getTempDir("mergejoin-schemachanges-left"));
- final File right_dir = new File(BaseTestQuery.getTempDir("mergejoin-schemachanges-right"));
- left_dir.mkdirs();
- right_dir.mkdirs();
- System.out.println(left_dir);
- System.out.println(right_dir);
-
- BufferedWriter writer = new BufferedWriter(new FileWriter(new File(left_dir, "l1.json")));
+ BufferedWriter writer = new BufferedWriter(new FileWriter(new File(leftDir, "l1.json")));
for (int i = 0; i < 50; ++i) {
writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i));
}
@@ -327,14 +328,14 @@ public class TestMergeJoinWithSchemaChanges extends BaseTestQuery {
}
writer.close();
- writer = new BufferedWriter(new FileWriter(new File(right_dir, "r1.json")));
+ writer = new BufferedWriter(new FileWriter(new File(rightDir, "r1.json")));
for (int i = 0; i < 50; ++i) {
writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i));
}
writer.close();
- String query = String.format("select * from dfs_test.`%s` L %s join dfs_test.`%s` R on L.kl=R.kl",
- left_dir.toPath().toString(), "inner", right_dir.toPath().toString());
+ String query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kl",
+ LEFT_DIR, "inner", RIGHT_DIR);
TestBuilder builder = testBuilder()
.sqlQuery(query)
.optionSettingQueriesForTestQuery("alter session set `planner.enable_hashjoin` = false; alter session set `exec.enable_union_type` = true")
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java
index 87c9501..a67a484 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestNestedLoopJoin.java
@@ -21,10 +21,12 @@ package org.apache.drill.exec.physical.impl.join;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.PlanTestBase;
import org.apache.drill.common.exceptions.UserRemoteException;
-import org.apache.drill.common.util.TestTools;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertThat;
@@ -32,8 +34,6 @@ import static org.junit.Assert.assertThat;
public class TestNestedLoopJoin extends PlanTestBase {
private static String nlpattern = "NestedLoopJoin";
- private static final String WORKING_PATH = TestTools.getWorkingPath();
- private static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
private static final String DISABLE_HJ = "alter session set `planner.enable_hashjoin` = false";
private static final String ENABLE_HJ = "alter session set `planner.enable_hashjoin` = true";
@@ -79,6 +79,11 @@ public class TestNestedLoopJoin extends PlanTestBase {
private static final String testNlJoinWithLargeRightInput = "select * from cp.`tpch/region.parquet`r " +
"left join cp.`tpch/nation.parquet` n on r.r_regionkey <> n.n_regionkey";
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel", "parquet"));
+ }
+
@Test
public void testNlJoinExists_1_planning() throws Exception {
testPlanMatchingPatterns(testNlJoinExists_1, new String[]{nlpattern}, new String[]{});
@@ -157,9 +162,9 @@ public class TestNestedLoopJoin extends PlanTestBase {
@Test // equality join and non-scalar right input, hj and mj disabled, enforce exchanges
public void testNlJoinEqualityNonScalar_2_planning() throws Exception {
- String query = String.format("select n.n_nationkey from cp.`tpch/nation.parquet` n, "
- + " dfs_test.`%s/multilevel/parquet` o "
- + " where n.n_regionkey = o.o_orderkey and o.o_custkey > 5", TEST_RES_PATH);
+ String query = "select n.n_nationkey from cp.`tpch/nation.parquet` n, "
+ + " dfs.`multilevel/parquet` o "
+ + " where n.n_regionkey = o.o_orderkey and o.o_custkey > 5";
test("alter session set `planner.slice_target` = 1");
test(DISABLE_HJ);
test(DISABLE_MJ);
[11/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 9715d14..e3fc833 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -17,14 +17,15 @@
*/
package org.apache.drill.exec.physical.impl.writer;
-import static org.apache.drill.TestBuilder.convertToLocalTimestamp;
import static org.apache.drill.exec.store.parquet.ParquetRecordWriter.DRILL_VERSION_PROPERTY;
+import static org.apache.drill.test.TestBuilder.convertToLocalTimestamp;
import static org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.FileWriter;
import java.math.BigDecimal;
+import java.nio.file.Paths;
import java.sql.Date;
import java.util.ArrayList;
import java.util.Arrays;
@@ -33,13 +34,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.drill.BaseTestQuery;
+import com.google.common.base.Joiner;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.ParquetTest;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.util.DrillVersionInfo;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.fn.interp.TestConstantFolding;
import org.apache.drill.exec.planner.physical.PlannerSettings;
@@ -54,26 +55,24 @@ import org.joda.time.Period;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
-import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import com.google.common.base.Joiner;
-
@RunWith(Parameterized.class)
@Category({SlowTest.class, ParquetTest.class})
public class TestParquetWriter extends BaseTestQuery {
-
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] { {100} });
}
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("parquet", "int96_dict_change"));
+ }
+
static FileSystem fs;
// Map storing a convenient name as well as the cast type necessary
@@ -116,8 +115,7 @@ public class TestParquetWriter extends BaseTestQuery {
allTypesSelection = Joiner.on(",").join(allTypeSelectsAndCasts);
}
-
- private String allTypesTable = "cp.`/parquet/alltypes.json`";
+ private String allTypesTable = "cp.`parquet/alltypes.json`";
@Parameterized.Parameter
public int repeat = 1;
@@ -136,7 +134,7 @@ public class TestParquetWriter extends BaseTestQuery {
@Test
public void testSmallFileValueReadWrite() throws Exception {
String selection = "key";
- String inputTable = "cp.`/store/json/intData.json`";
+ String inputTable = "cp.`store/json/intData.json`";
runTestAndValidate(selection, selection, inputTable, "smallFileTest");
}
@@ -166,14 +164,17 @@ public class TestParquetWriter extends BaseTestQuery {
colNames[numCols - 1] = "col_" + (numCols - 1);
values[numCols - 1] = 100l;
+ String path = "test";
+ File pathDir = dirTestWatcher.makeRootSubDir(Paths.get(path));
+
// write it to a file in the temp directory for the test
- new TestConstantFolding.SmallFileCreator(folder).setRecord(sb.toString()).createFiles(1, 1, "json");
+ new TestConstantFolding.SmallFileCreator(pathDir)
+ .setRecord(sb.toString()).createFiles(1, 1, "json");
- String path = folder.getRoot().toPath().toString();
- test("use dfs_test.tmp");
- test("create table WIDE_PARQUET_TABLE_TestParquetWriter_testLargeFooter as select * from dfs.`" + path + "/smallfile/smallfile.json`");
+ test("use dfs.tmp");
+ test("create table WIDE_PARQUET_TABLE_TestParquetWriter_testLargeFooter as select * from dfs.`%s/smallfile/smallfile.json`", path);
testBuilder()
- .sqlQuery("select * from dfs_test.tmp.WIDE_PARQUET_TABLE_TestParquetWriter_testLargeFooter")
+ .sqlQuery("select * from dfs.tmp.WIDE_PARQUET_TABLE_TestParquetWriter_testLargeFooter")
.unOrdered()
.baselineColumns(colNames)
.baselineValues(values)
@@ -377,7 +378,7 @@ public class TestParquetWriter extends BaseTestQuery {
@Ignore("Test file not available")
@Test
public void testBitError_Drill_2031() throws Exception {
- compareParquetReadersHyperVector("*", "dfs.`/tmp/wide2/0_0_3.parquet`");
+ compareParquetReadersHyperVector("*", "dfs.`tmp/wide2/0_0_3.parquet`");
}
@Test
@@ -445,7 +446,7 @@ public class TestParquetWriter extends BaseTestQuery {
String queryFromWriteOut = "select * from " + outputFile;
try {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test(ctasStmt);
testBuilder()
.ordered()
@@ -495,82 +496,82 @@ public class TestParquetWriter extends BaseTestQuery {
@Ignore("Binary file too large for version control")
@Test
public void testReadVoter() throws Exception {
- compareParquetReadersHyperVector("*", "dfs.`/tmp/voter.parquet`");
+ compareParquetReadersHyperVector("*", "dfs.`tmp/voter.parquet`");
}
@Ignore("Test file not available")
@Test
public void testReadSf_100_supplier() throws Exception {
- compareParquetReadersHyperVector("*", "dfs.`/tmp/sf100_supplier.parquet`");
+ compareParquetReadersHyperVector("*", "dfs.`tmp/sf100_supplier.parquet`");
}
@Ignore("Binary file too large for version control")
@Test
public void testParquetRead_checkNulls_NullsFirst() throws Exception {
compareParquetReadersColumnar("*",
- "dfs.`/tmp/parquet_with_nulls_should_sum_100000_nulls_first.parquet`");
+ "dfs.`tmp/parquet_with_nulls_should_sum_100000_nulls_first.parquet`");
}
@Ignore("Test file not available")
@Test
public void testParquetRead_checkNulls() throws Exception {
- compareParquetReadersColumnar("*", "dfs.`/tmp/parquet_with_nulls_should_sum_100000.parquet`");
+ compareParquetReadersColumnar("*", "dfs.`tmp/parquet_with_nulls_should_sum_100000.parquet`");
}
@Ignore("Binary file too large for version control")
@Test
public void test958_sql() throws Exception {
- compareParquetReadersHyperVector("ss_ext_sales_price", "dfs.`/tmp/store_sales`");
+ compareParquetReadersHyperVector("ss_ext_sales_price", "dfs.`tmp/store_sales`");
}
@Ignore("Binary file too large for version control")
@Test
public void testReadSf_1_supplier() throws Exception {
- compareParquetReadersHyperVector("*", "dfs.`/tmp/orders_part-m-00001.parquet`");
+ compareParquetReadersHyperVector("*", "dfs.`tmp/orders_part-m-00001.parquet`");
}
@Ignore("Binary file too large for version control")
@Test
public void test958_sql_all_columns() throws Exception {
- compareParquetReadersHyperVector("*", "dfs.`/tmp/store_sales`");
- compareParquetReadersHyperVector("ss_addr_sk, ss_hdemo_sk", "dfs.`/tmp/store_sales`");
+ compareParquetReadersHyperVector("*", "dfs.`tmp/store_sales`");
+ compareParquetReadersHyperVector("ss_addr_sk, ss_hdemo_sk", "dfs.`tmp/store_sales`");
// TODO - Drill 1388 - this currently fails, but it is an issue with project, not the reader, pulled out the physical plan
// removed the unneeded project in the plan and ran it against both readers, they outputs matched
// compareParquetReadersHyperVector("pig_schema,ss_sold_date_sk,ss_item_sk,ss_cdemo_sk,ss_addr_sk, ss_hdemo_sk",
-// "dfs.`/tmp/store_sales`");
+// "dfs.`tmp/store_sales`");
}
@Ignore("Binary file too large for version control")
@Test
public void testDrill_1314() throws Exception {
- compareParquetReadersColumnar("l_partkey ", "dfs.`/tmp/drill_1314.parquet`");
+ compareParquetReadersColumnar("l_partkey ", "dfs.`tmp/drill_1314.parquet`");
}
@Ignore("Binary file too large for version control")
@Test
public void testDrill_1314_all_columns() throws Exception {
- compareParquetReadersHyperVector("*", "dfs.`/tmp/drill_1314.parquet`");
+ compareParquetReadersHyperVector("*", "dfs.`tmp/drill_1314.parquet`");
compareParquetReadersColumnar(
"l_orderkey,l_partkey,l_suppkey,l_linenumber, l_quantity, l_extendedprice,l_discount,l_tax",
- "dfs.`/tmp/drill_1314.parquet`");
+ "dfs.`tmp/drill_1314.parquet`");
}
@Ignore("Test file not available")
@Test
public void testParquetRead_checkShortNullLists() throws Exception {
- compareParquetReadersColumnar("*", "dfs.`/tmp/short_null_lists.parquet`");
+ compareParquetReadersColumnar("*", "dfs.`tmp/short_null_lists.parquet`");
}
@Ignore("Test file not available")
@Test
public void testParquetRead_checkStartWithNull() throws Exception {
- compareParquetReadersColumnar("*", "dfs.`/tmp/start_with_null.parquet`");
+ compareParquetReadersColumnar("*", "dfs.`tmp/start_with_null.parquet`");
}
@Ignore("Binary file too large for version control")
@Test
public void testParquetReadWebReturns() throws Exception {
- compareParquetReadersColumnar("wr_returning_customer_sk", "dfs.`/tmp/web_returns`");
+ compareParquetReadersColumnar("wr_returning_customer_sk", "dfs.`tmp/web_returns`");
}
@Test
@@ -578,18 +579,16 @@ public class TestParquetWriter extends BaseTestQuery {
String outputTable = "decimal_test";
try {
- String ctas = String.format("use dfs_test.tmp; " +
- "create table %s as select " +
- "cast('1.2' as decimal(38, 2)) col1, cast('1.2' as decimal(28, 2)) col2 " +
- "from cp.`employee.json` limit 1", outputTable);
-
- test(ctas);
+ test("use dfs.tmp; " +
+ "create table %s as select " +
+ "cast('1.2' as decimal(38, 2)) col1, cast('1.2' as decimal(28, 2)) col2 " +
+ "from cp.`employee.json` limit 1", outputTable);
BigDecimal result = new BigDecimal("1.20");
testBuilder()
.unOrdered()
- .sqlQuery(String.format("select col1, col2 from %s ", outputTable))
+ .sqlQuery("select col1, col2 from %s ", outputTable)
.baselineColumns("col1", "col2")
.baselineValues(result, result)
.go();
@@ -604,17 +603,15 @@ public class TestParquetWriter extends BaseTestQuery {
final String newTblName = "testTableOutputSchema";
try {
- final String ctas = String.format("CREATE TABLE dfs_test.tmp.%s(id, name, bday) AS SELECT " +
- "cast(`employee_id` as integer), " +
- "cast(`full_name` as varchar(100)), " +
- "cast(`birth_date` as date) " +
- "FROM cp.`employee.json` ORDER BY `employee_id` LIMIT 1", newTblName);
-
- test(ctas);
+ test("CREATE TABLE dfs.tmp.%s(id, name, bday) AS SELECT " +
+ "cast(`employee_id` as integer), " +
+ "cast(`full_name` as varchar(100)), " +
+ "cast(`birth_date` as date) " +
+ "FROM cp.`employee.json` ORDER BY `employee_id` LIMIT 1", newTblName);
testBuilder()
.unOrdered()
- .sqlQuery(String.format("SELECT * FROM dfs_test.tmp.`%s`", newTblName))
+ .sqlQuery("SELECT * FROM dfs.tmp.`%s`", newTblName)
.baselineColumns("id", "name", "bday")
.baselineValues(1, "Sheri Nowmer", new DateTime(Date.valueOf("1961-08-26").getTime()))
.go();
@@ -629,15 +626,15 @@ public class TestParquetWriter extends BaseTestQuery {
*/
@Test
public void testCTASWithIntervalTypes() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
String tableName = "drill_1980_t1";
// test required interval day type
- test(String.format("create table %s as " +
+ test("create table %s as " +
"select " +
"interval '10 20:30:40.123' day to second col1, " +
"interval '-1000000000 20:12:23.999' day(10) to second col2 " +
- "from cp.`employee.json` limit 2", tableName));
+ "from cp.`employee.json` limit 2", tableName);
Period row1Col1 = new Period(0, 0, 0, 10, 0, 0, 0, 73840123);
Period row1Col2 = new Period(0, 0, 0, -1000000000, 0, 0, 0, -72743999);
@@ -646,11 +643,11 @@ public class TestParquetWriter extends BaseTestQuery {
tableName = "drill_1980_2";
// test required interval year type
- test(String.format("create table %s as " +
+ test("create table %s as " +
"select " +
"interval '10-2' year to month col1, " +
"interval '-100-8' year(3) to month col2 " +
- "from cp.`employee.json` limit 2", tableName));
+ "from cp.`employee.json` limit 2", tableName);
row1Col1 = new Period(0, 122, 0, 0, 0, 0, 0, 0);
row1Col2 = new Period(0, -1208, 0, 0, 0, 0, 0, 0);
@@ -658,11 +655,11 @@ public class TestParquetWriter extends BaseTestQuery {
testParquetReaderHelper(tableName, row1Col1, row1Col2, row1Col1, row1Col2);
// test nullable interval year type
tableName = "drill_1980_t3";
- test(String.format("create table %s as " +
+ test("create table %s as " +
"select " +
"cast (intervalyear_col as interval year) col1," +
"cast(intervalyear_col as interval year) + interval '2' year col2 " +
- "from cp.`parquet/alltypes.json` where tinyint_col = 1 or tinyint_col = 2", tableName));
+ "from cp.`parquet/alltypes.json` where tinyint_col = 1 or tinyint_col = 2", tableName);
row1Col1 = new Period(0, 12, 0, 0, 0, 0, 0, 0);
row1Col2 = new Period(0, 36, 0, 0, 0, 0, 0, 0);
@@ -673,11 +670,11 @@ public class TestParquetWriter extends BaseTestQuery {
// test nullable interval day type
tableName = "drill_1980_t4";
- test(String.format("create table %s as " +
+ test("create table %s as " +
"select " +
"cast(intervalday_col as interval day) col1, " +
"cast(intervalday_col as interval day) + interval '1' day col2 " +
- "from cp.`parquet/alltypes.json` where tinyint_col = 1 or tinyint_col = 2", tableName));
+ "from cp.`parquet/alltypes.json` where tinyint_col = 1 or tinyint_col = 2", tableName);
row1Col1 = new Period(0, 0, 0, 1, 0, 0, 0, 0);
row1Col2 = new Period(0, 0, 0, 2, 0, 0, 0, 0);
@@ -716,7 +713,7 @@ public class TestParquetWriter extends BaseTestQuery {
private static void deleteTableIfExists(String tableName) {
try {
- Path path = new Path(getDfsTestTmpSchemaLocation(), tableName);
+ Path path = new Path(dirTestWatcher.getDfsTestTmpDir().getAbsolutePath(), tableName);
if (fs.exists(path)) {
fs.delete(path, true);
}
@@ -728,24 +725,22 @@ public class TestParquetWriter extends BaseTestQuery {
public void runTestAndValidate(String selection, String validationSelection, String inputTable, String outputFile) throws Exception {
try {
deleteTableIfExists(outputFile);
- test("use dfs_test.tmp");
- String query = String.format("SELECT %s FROM %s", selection, inputTable);
- String create = "CREATE TABLE " + outputFile + " AS " + query;
- String validateQuery = String.format("SELECT %s FROM " + outputFile, validationSelection);
- test(create);
+ final String query = String.format("SELECT %s FROM %s", selection, inputTable);
+
+ test("use dfs.tmp");
+ test("CREATE TABLE %s AS %s", outputFile, query);
testBuilder()
.unOrdered()
- // Validate query is the query on the output file (the one to validate)
- .sqlQuery(validateQuery)
- // The basline query is the query on the input file (the expected values)
- .sqlBaselineQuery(query)
+ .sqlQuery(query)
+ .sqlBaselineQuery("SELECT %s FROM %s", validationSelection, outputFile)
.go();
Configuration hadoopConf = new Configuration();
hadoopConf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
- Path output = new Path(getDfsTestTmpSchemaLocation(), outputFile);
+
@SuppressWarnings("resource")
+ Path output = new Path(dirTestWatcher.getDfsTestTmpDir().getAbsolutePath(), outputFile);
FileSystem fs = output.getFileSystem(hadoopConf);
for (FileStatus file : fs.listStatus(output)) {
ParquetMetadata footer = ParquetFileReader.readFooter(hadoopConf, file, SKIP_ROW_GROUPS);
@@ -787,11 +782,9 @@ public class TestParquetWriter extends BaseTestQuery {
@Test
@Ignore("relies on particular time zone, works for UTC")
public void testImpalaParquetBinaryAsTimeStamp_DictChange() throws Exception {
- final String WORKING_PATH = TestTools.getWorkingPath();
- final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
try {
testBuilder()
- .sqlQuery("select int96_ts from dfs_test.`%s/parquet/int96_dict_change` order by int96_ts", TEST_RES_PATH)
+ .sqlQuery("select int96_ts from dfs.`parquet/int96_dict_change` order by int96_ts")
.optionSettingQueriesForTestQuery(
"alter session set `%s` = true", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP)
.ordered()
@@ -851,12 +844,10 @@ public class TestParquetWriter extends BaseTestQuery {
@Test
@Ignore("relies on particular time zone")
public void testHiveParquetTimestampAsInt96_basic() throws Exception {
- final String q = "SELECT cast(convert_from(timestamp_field, 'TIMESTAMP_IMPALA') as varchar(19)) as timestamp_field "
- + "from cp.`parquet/part1/hive_all_types.parquet` ";
-
testBuilder()
.unOrdered()
- .sqlQuery(q)
+ .sqlQuery("SELECT cast(convert_from(timestamp_field, 'TIMESTAMP_IMPALA') as varchar(19)) as timestamp_field "
+ + "from cp.`parquet/part1/hive_all_types.parquet` ")
.baselineColumns("timestamp_field")
.baselineValues("2013-07-05 17:01:00")
.baselineValues((Object)null)
@@ -999,6 +990,5 @@ public class TestParquetWriter extends BaseTestQuery {
resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java
index 2170f41..bc72234 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java
@@ -17,41 +17,33 @@
*/
package org.apache.drill.exec.physical.impl.writer;
-import org.apache.drill.BaseTestQuery;
+import org.apache.commons.io.FileUtils;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.ParquetTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.test.OperatorFixture;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.io.File;
+
@Category({ParquetTest.class, UnlikelyTest.class})
public class TestParquetWriterEmptyFiles extends BaseTestQuery {
- private static FileSystem fs;
-
@BeforeClass
public static void initFs() throws Exception {
- fs = getLocalFileSystem();
updateTestCluster(3, null);
}
@Test // see DRILL-2408
public void testWriteEmptyFile() throws Exception {
- final String outputFile = "testparquetwriteremptyfiles_testwriteemptyfile";
-
- try {
- test("CREATE TABLE dfs_test.tmp.%s AS SELECT * FROM cp.`employee.json` WHERE 1=0", outputFile);
+ final String outputFileName = "testparquetwriteremptyfiles_testwriteemptyfile";
+ final File outputFile = FileUtils.getFile(dirTestWatcher.getDfsTestTmpDir(), outputFileName);
- final Path path = new Path(getDfsTestTmpSchemaLocation(), outputFile);
- Assert.assertFalse(fs.exists(path));
- } finally {
- deleteTableIfExists(outputFile);
- }
+ test("CREATE TABLE dfs.tmp.%s AS SELECT * FROM cp.`employee.json` WHERE 1=0", outputFileName);
+ Assert.assertFalse(outputFile.exists());
}
@Test
@@ -62,24 +54,23 @@ public class TestParquetWriterEmptyFiles extends BaseTestQuery {
try {
final String query = "SELECT position_id FROM cp.`employee.json` WHERE position_id IN (15, 16) GROUP BY position_id";
- test("CREATE TABLE dfs_test.tmp.%s AS %s", outputFile, query);
+
+ test("CREATE TABLE dfs.tmp.%s AS %s", outputFile, query);
// this query will fail if an "empty" file was created
testBuilder()
.unOrdered()
- .sqlQuery("SELECT * FROM dfs_test.tmp.%s", outputFile)
+ .sqlQuery("SELECT * FROM dfs.tmp.%s", outputFile)
.sqlBaselineQuery(query)
.go();
} finally {
runSQL("alter session set `planner.slice_target` = " + ExecConstants.SLICE_TARGET_DEFAULT);
- deleteTableIfExists(outputFile);
}
}
@Test // see DRILL-2408
public void testWriteEmptyFileAfterFlush() throws Exception {
final String outputFile = "testparquetwriteremptyfiles_test_write_empty_file_after_flush";
- deleteTableIfExists(outputFile);
try {
// this specific value will force a flush just after the final row is written
@@ -87,32 +78,17 @@ public class TestParquetWriterEmptyFiles extends BaseTestQuery {
test("ALTER SESSION SET `store.parquet.block-size` = 19926");
final String query = "SELECT * FROM cp.`employee.json` LIMIT 100";
- test("CREATE TABLE dfs_test.tmp.%s AS %s", outputFile, query);
+ test("CREATE TABLE dfs.tmp.%s AS %s", outputFile, query);
// this query will fail if an "empty" file was created
testBuilder()
.unOrdered()
- .sqlQuery("SELECT * FROM dfs_test.tmp.%s", outputFile)
+ .sqlQuery("SELECT * FROM dfs.tmp.%s", outputFile)
.sqlBaselineQuery(query)
.go();
} finally {
// restore the session option
resetSessionOption(ExecConstants.PARQUET_BLOCK_SIZE);
- deleteTableIfExists(outputFile);
- }
- }
-
- private static boolean deleteTableIfExists(String tableName) {
- try {
- Path path = new Path(getDfsTestTmpSchemaLocation(), tableName);
- if (fs.exists(path)) {
- return fs.delete(path, true);
- }
- } catch (Exception e) {
- // ignore exceptions.
- return false;
}
-
- return true;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
index 9f93eed..4dc2b96 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
@@ -20,45 +20,39 @@ package org.apache.drill.exec.physical.impl.writer;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.File;
+import java.nio.file.Paths;
import java.util.List;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.VarCharVector;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
public class TestWriter extends BaseTestQuery {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestWriter.class);
+ private static final String ROOT_DIR_REPLACEMENT = "%ROOT_DIR%";
+ private static final String TMP_DIR_REPLACEMENT = "%TMP_DIR%";
+ private static final String TEST_DIR_REPLACEMENT = "%TEST_DIR%";
- static FileSystem fs;
- static String ALTER_SESSION = String.format("ALTER SESSION SET `%s` = 'csv'", ExecConstants.OUTPUT_FORMAT_OPTION);
-
- @BeforeClass
- public static void initFs() throws Exception {
- fs = getLocalFileSystem();
- }
+ private static final String ALTER_SESSION = String.format("ALTER SESSION SET `%s` = 'csv'", ExecConstants.OUTPUT_FORMAT_OPTION);
@Test
public void simpleCsv() throws Exception {
- // before executing the test deleting the existing CSV files in /tmp/csvtest
- Path path = new Path("/tmp/csvtest");
- if (fs.exists(path)) {
- fs.delete(path, true);
- }
+ File testDir = dirTestWatcher.makeRootSubDir(Paths.get("csvtest"));
- String plan = Files.toString(FileUtils.getResourceAsFile("/writer/simple_csv_writer.json"), Charsets.UTF_8);
+ String plan = Files.toString(DrillFileUtils.getResourceAsFile("/writer/simple_csv_writer.json"), Charsets.UTF_8);
+ plan = plan
+ .replace(ROOT_DIR_REPLACEMENT, dirTestWatcher.getRootDir().getAbsolutePath())
+ .replace(TMP_DIR_REPLACEMENT, dirTestWatcher.getTmpDir().getAbsolutePath())
+ .replace(TEST_DIR_REPLACEMENT, testDir.getAbsolutePath());
List<QueryDataBatch> results = testPhysicalWithResults(plan);
@@ -77,49 +71,50 @@ public class TestWriter extends BaseTestQuery {
assertEquals(132000, recordWrittenV.getAccessor().get(0));
// now verify csv files are written to disk
- assertTrue(fs.exists(path));
+ assertTrue(testDir.exists());
// expect two files
- FileStatus[] fileStatuses = fs.globStatus(new Path(path.toString(), "*.csv"));
- assertTrue(2 == fileStatuses.length);
+ int count = org.apache.commons.io.FileUtils.listFiles(testDir, new String[]{"csv"}, false).size();
+ assertEquals(2, count);
for (QueryDataBatch b : results) {
b.release();
}
+
batchLoader.clear();
}
@Test
public void simpleCTAS() throws Exception {
final String tableName = "simplectas";
- runSQL("Use dfs_test.tmp");
+ runSQL("Use dfs.tmp");
runSQL(ALTER_SESSION);
final String testQuery = String.format("CREATE TABLE %s AS SELECT * FROM cp.`employee.json`", tableName);
- testCTASQueryHelper(tableName, testQuery, 1155);
+ testCTASQueryHelper(testQuery, 1155);
}
@Test
public void complex1CTAS() throws Exception {
final String tableName = "complex1ctas";
- runSQL("Use dfs_test.tmp");
+ runSQL("Use dfs.tmp");
runSQL(ALTER_SESSION);
final String testQuery = String.format("CREATE TABLE %s AS SELECT first_name, last_name, " +
"position_id FROM cp.`employee.json`", tableName);
- testCTASQueryHelper(tableName, testQuery, 1155);
+ testCTASQueryHelper(testQuery, 1155);
}
@Test
public void complex2CTAS() throws Exception {
- final String tableName = "complex1ctas";
- runSQL("Use dfs_test.tmp");
+ final String tableName = "complex2ctas";
+ runSQL("Use dfs.tmp");
runSQL(ALTER_SESSION);
final String testQuery = String.format("CREATE TABLE %s AS SELECT CAST(`birth_date` as Timestamp) FROM " +
"cp.`employee.json` GROUP BY birth_date", tableName);
- testCTASQueryHelper(tableName, testQuery, 52);
+ testCTASQueryHelper(testQuery, 52);
}
@Test
@@ -127,21 +122,21 @@ public class TestWriter extends BaseTestQuery {
final String tableName = "/test/simplectas2";
runSQL(ALTER_SESSION);
final String testQuery =
- String.format("CREATE TABLE dfs_test.tmp.`%s` AS SELECT * FROM cp.`employee.json`",tableName);
+ String.format("CREATE TABLE dfs.tmp.`%s` AS SELECT * FROM cp.`employee.json`",tableName);
- testCTASQueryHelper(tableName, testQuery, 1155);
+ testCTASQueryHelper(testQuery, 1155);
}
@Test
public void simpleParquetDecimal() throws Exception {
try {
final String tableName = "simpleparquetdecimal";
- final String testQuery = String.format("CREATE TABLE dfs_test.tmp.`%s` AS SELECT cast(salary as " +
+ final String testQuery = String.format("CREATE TABLE dfs.tmp.`%s` AS SELECT cast(salary as " +
"decimal(30,2)) * -1 as salary FROM cp.`employee.json`", tableName);
// enable decimal
test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
- testCTASQueryHelper(tableName, testQuery, 1155);
+ testCTASQueryHelper(testQuery, 1155);
// disable decimal
} finally {
@@ -149,42 +144,31 @@ public class TestWriter extends BaseTestQuery {
}
}
- private void testCTASQueryHelper(String tableName, String testQuery, int expectedOutputCount) throws Exception {
- try {
- List<QueryDataBatch> results = testSqlWithResults(testQuery);
-
- RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
+ private void testCTASQueryHelper(String testQuery, int expectedOutputCount) throws Exception {
+ List<QueryDataBatch> results = testSqlWithResults(testQuery);
- int recordsWritten = 0;
- for (QueryDataBatch batch : results) {
- batchLoader.load(batch.getHeader().getDef(), batch.getData());
+ RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
- if (batchLoader.getRecordCount() <= 0) {
- continue;
- }
+ int recordsWritten = 0;
+ for (QueryDataBatch batch : results) {
+ batchLoader.load(batch.getHeader().getDef(), batch.getData());
- BigIntVector recordWrittenV = (BigIntVector) batchLoader.getValueAccessorById(BigIntVector.class, 1).getValueVector();
+ if (batchLoader.getRecordCount() <= 0) {
+ continue;
+ }
- for (int i = 0; i < batchLoader.getRecordCount(); i++) {
- recordsWritten += recordWrittenV.getAccessor().get(i);
- }
+ BigIntVector recordWrittenV = (BigIntVector) batchLoader
+ .getValueAccessorById(BigIntVector.class, 1)
+ .getValueVector();
- batchLoader.clear();
- batch.release();
+ for (int i = 0; i < batchLoader.getRecordCount(); i++) {
+ recordsWritten += recordWrittenV.getAccessor().get(i);
}
- assertEquals(expectedOutputCount, recordsWritten);
- } finally {
- try {
- Path path = new Path(getDfsTestTmpSchemaLocation(), tableName);
- if (fs.exists(path)) {
- fs.delete(path, true);
- }
- } catch (Exception e) {
- // ignore exceptions.
- logger.warn("Failed to delete the table [{}, {}] created as part of the test",
- getDfsTestTmpSchemaLocation(), tableName);
- }
+ batchLoader.clear();
+ batch.release();
}
+
+ assertEquals(expectedOutputCount, recordsWritten);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java
index 66b1f0e..c52f1a9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java
@@ -17,18 +17,24 @@
*/
package org.apache.drill.exec.physical.impl.xsort;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.TestBuilder;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.test.rowSet.RowSet;
+import org.apache.drill.test.rowSet.RowSetBuilder;
+import org.apache.drill.test.rowSet.SchemaBuilder;
+import org.apache.drill.test.rowSet.file.JsonFileBuilder;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import java.io.BufferedOutputStream;
import java.io.File;
-import java.io.FileOutputStream;
+import java.nio.file.Paths;
@Category({SlowTest.class, OperatorTest.class})
public class TestExternalSort extends BaseTestQuery {
@@ -45,29 +51,48 @@ public class TestExternalSort extends BaseTestQuery {
private void testNumericTypes(boolean testLegacy) throws Exception {
final int record_count = 10000;
- String dfs_temp = getDfsTestTmpSchemaLocation();
- System.out.println(dfs_temp);
- File table_dir = new File(dfs_temp, "numericTypes");
- table_dir.mkdir();
- try(BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")))) {
- String format = "{ a : %d }%n";
+ final String tableDirName = "numericTypes";
+
+ {
+ final BatchSchema schema = new SchemaBuilder()
+ .add("a", Types.required(TypeProtos.MinorType.INT))
+ .build();
+ final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
+
for (int i = 0; i <= record_count; i += 2) {
- os.write(String.format(format, i).getBytes());
+ rowSetBuilder.add(i);
}
+
+ final RowSet rowSet = rowSetBuilder.build();
+ final File tableFile = createTableFile(tableDirName, "a.json");
+ new JsonFileBuilder(rowSet).build(tableFile);
+ rowSet.clear();
}
- try(BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "b.json")))) {
- String format = "{ a : %.2f }%n";
- for (int i = 1; i <= record_count; i+=2) {
- os.write(String.format(format, (float) i).getBytes());
+
+ {
+ final BatchSchema schema = new SchemaBuilder()
+ .add("a", Types.required(TypeProtos.MinorType.FLOAT4))
+ .build();
+ final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
+
+ for (int i = 1; i <= record_count; i += 2) {
+ rowSetBuilder.add((float) i);
}
+
+ final RowSet rowSet = rowSetBuilder.build();
+ final File tableFile = createTableFile(tableDirName, "b.json");
+ new JsonFileBuilder(rowSet)
+ .setCustomFormatter("a", "%.2f")
+ .build(tableFile);
+ rowSet.clear();
}
- String query = "select * from dfs_test.tmp.numericTypes order by a desc";
+
TestBuilder builder = testBuilder()
- .sqlQuery(query)
- .optionSettingQueriesForTestQuery(getOptions(testLegacy))
- .ordered()
- .baselineColumns("a");
- for (int i = record_count; i >= 0;) {
+ .sqlQuery("select * from dfs.`%s` order by a desc", tableDirName)
+ .optionSettingQueriesForTestQuery(getOptions(testLegacy))
+ .ordered()
+ .baselineColumns("a");
+ for (int i = record_count; i >= 0; ) {
builder.baselineValues((long) i--);
if (i >= 0) {
builder.baselineValues((double) i--);
@@ -97,25 +122,44 @@ public class TestExternalSort extends BaseTestQuery {
private void testNumericAndStringTypes(boolean testLegacy) throws Exception {
final int record_count = 10000;
- String dfs_temp = getDfsTestTmpSchemaLocation();
- System.out.println(dfs_temp);
- File table_dir = new File(dfs_temp, "numericAndStringTypes");
- table_dir.mkdir();
- try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")))) {
- String format = "{ a : %d }%n";
+ final String tableDirName = "numericAndStringTypes";
+
+ {
+ final BatchSchema schema = new SchemaBuilder()
+ .add("a", Types.required(TypeProtos.MinorType.INT))
+ .build();
+ final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
+
for (int i = 0; i <= record_count; i += 2) {
- os.write(String.format(format, i).getBytes());
+ rowSetBuilder.add(i);
}
+
+ final RowSet rowSet = rowSetBuilder.build();
+ final File tableFile = createTableFile(tableDirName, "a.json");
+ new JsonFileBuilder(rowSet).build(tableFile);
+ rowSet.clear();
}
- try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "b.json")))) {
- String format = "{ a : \"%05d\" }%n";
- for (int i = 1; i <= record_count; i+=2) {
- os.write(String.format(format, i).getBytes());
+
+ {
+ final BatchSchema schema = new SchemaBuilder()
+ .add("a", Types.required(TypeProtos.MinorType.INT))
+ .build();
+ final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
+
+ for (int i = 1; i <= record_count; i += 2) {
+ rowSetBuilder.add(i);
}
+
+ final RowSet rowSet = rowSetBuilder.build();
+ final File tableFile = createTableFile(tableDirName, "b.json");
+ new JsonFileBuilder(rowSet)
+ .setCustomFormatter("a", "\"%05d\"")
+ .build(tableFile);
+ rowSet.clear();
}
- String query = "select * from dfs_test.tmp.numericAndStringTypes order by a desc";
+
TestBuilder builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from dfs.`%s` order by a desc", tableDirName)
.ordered()
.optionSettingQueriesForTestQuery(getOptions(testLegacy))
.baselineColumns("a");
@@ -138,7 +182,6 @@ public class TestExternalSort extends BaseTestQuery {
testNewColumns(false);
}
-
@Test
public void testNewColumnsLegacy() throws Exception {
testNewColumns(true);
@@ -146,26 +189,45 @@ public class TestExternalSort extends BaseTestQuery {
private void testNewColumns(boolean testLegacy) throws Exception {
final int record_count = 10000;
- String dfs_temp = getDfsTestTmpSchemaLocation();
- System.out.println(dfs_temp);
- File table_dir = new File(dfs_temp, "newColumns");
- table_dir.mkdir();
- try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")))) {
- String format = "{ a : %d, b : %d }%n";
+ final String tableDirName = "newColumns";
+
+ {
+ final BatchSchema schema = new SchemaBuilder()
+ .add("a", Types.required(TypeProtos.MinorType.INT))
+ .add("b", Types.required(TypeProtos.MinorType.INT))
+ .build();
+ final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
+
for (int i = 0; i <= record_count; i += 2) {
- os.write(String.format(format, i, i).getBytes());
+ rowSetBuilder.add(i, i);
}
+
+ final RowSet rowSet = rowSetBuilder.build();
+ final File tableFile = createTableFile(tableDirName, "a.json");
+ new JsonFileBuilder(rowSet).build(tableFile);
+ rowSet.clear();
}
- try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "b.json")))) {
- String format = "{ a : %d, c : %d }%n";
- for (int i = 1; i <= record_count; i+=2) {
- os.write(String.format(format, i, i).getBytes());
+
+ {
+ final BatchSchema schema = new SchemaBuilder()
+ .add("a", Types.required(TypeProtos.MinorType.INT))
+ .add("c", Types.required(TypeProtos.MinorType.INT))
+ .build();
+ final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
+
+ for (int i = 1; i <= record_count; i += 2) {
+ rowSetBuilder.add(i, i);
}
+
+ final RowSet rowSet = rowSetBuilder.build();
+ final File tableFile = createTableFile(tableDirName, "b.json");
+ new JsonFileBuilder(rowSet).build(tableFile);
+ rowSet.clear();
}
- String query = "select a, b, c from dfs_test.tmp.newColumns order by a desc";
-// Test framework currently doesn't handle changing schema (i.e. new columns) on the client side
+
+ // Test framework currently doesn't handle changing schema (i.e. new columns) on the client side
TestBuilder builder = testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select a, b, c from dfs.`%s` order by a desc", tableDirName)
.ordered()
.optionSettingQueriesForTestQuery(getOptions(testLegacy))
.baselineColumns("a", "b", "c");
@@ -176,7 +238,14 @@ public class TestExternalSort extends BaseTestQuery {
}
}
builder.go();
- String newQuery = "select * from dfs_test.tmp.newColumns order by a desc";
- test(newQuery);
+ test("select * from dfs.`%s` order by a desc", tableDirName);
+ }
+
+ private File createTableFile(final String tableDirName, final String fileName) {
+ return dirTestWatcher
+ .getRootDir()
+ .toPath()
+ .resolve(Paths.get(tableDirName, fileName))
+ .toFile();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
index aa0d4ee..21de6c5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
@@ -24,13 +24,14 @@ import java.util.List;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.DrillTest;
@@ -43,9 +44,11 @@ import org.junit.rules.TestRule;
@Category(SlowTest.class)
public class TestSimpleExternalSort extends DrillTest {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleExternalSort.class);
+ @Rule
+ public final TestRule TIMEOUT = TestTools.getTimeoutRule(160_000);
- @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(160_000);
+ @Rule
+ public final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
@Test
public void mergeSortWithSv2Managed() throws Exception {
@@ -69,9 +72,8 @@ public class TestSimpleExternalSort extends DrillTest {
*/
private void mergeSortWithSv2(boolean testLegacy) throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
- .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false)
- ;
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
chooseImpl(client, testLegacy);
@@ -96,9 +98,8 @@ public class TestSimpleExternalSort extends DrillTest {
}
private void sortOneKeyDescendingMergeSort(boolean testLegacy) throws Throwable {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
- .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false)
- ;
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
chooseImpl(client, testLegacy);
@@ -147,12 +148,11 @@ public class TestSimpleExternalSort extends DrillTest {
}
private void sortOneKeyDescendingExternalSort(boolean testLegacy) throws Throwable {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
.configProperty(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD, 4)
.configProperty(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE, 4)
.configProperty(ExecConstants.EXTERNAL_SORT_BATCH_LIMIT, 4)
- .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false)
- ;
+ .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
chooseImpl(client,testLegacy);
@@ -173,14 +173,13 @@ public class TestSimpleExternalSort extends DrillTest {
}
private void outOfMemoryExternalSort(boolean testLegacy) throws Throwable{
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
// Probably do nothing in modern Drill
.configProperty("drill.memory.fragment.max", 50_000_000)
.configProperty("drill.memory.fragment.initial", 2_000_000)
.configProperty("drill.memory.operator.max", 30_000_000)
.configProperty("drill.memory.operator.initial", 2_000_000)
- .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, testLegacy)
- ;
+ .configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, testLegacy);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
chooseImpl(client,testLegacy);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
index 2694f64..cd3f5fd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
@@ -22,18 +22,20 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.nio.file.Paths;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.exceptions.UserRemoteException;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType;
import org.apache.drill.exec.testing.Controls;
import org.apache.drill.exec.testing.ControlsInjectionUtil;
+import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.ClusterFixtureBuilder;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -47,19 +49,21 @@ import org.junit.experimental.categories.Category;
*/
@Category(OperatorTest.class)
public class TestSortSpillWithException extends ClusterTest {
- private static final String TEST_RES_PATH = TestTools.getWorkingPath() + "/src/test/resources";
+ @ClassRule
+ public static final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
@BeforeClass
public static void setup() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ dirTestWatcher.copyResourceToRoot(Paths.get("xsort"));
+
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
.configProperty(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD, 1) // Unmanaged
.configProperty(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE, 1) // Unmanaged
.sessionOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 60 * 1024 * 1024) // Spill early
// Prevent the percent-based memory rule from second-guessing the above.
.sessionOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.0)
.configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false)
- .maxParallelization(1)
- ;
+ .maxParallelization(1);
startCluster(builder);
}
@@ -77,7 +81,7 @@ public class TestSortSpillWithException extends ClusterTest {
ControlsInjectionUtil.setControls(cluster.client(), controls);
// run a simple order by query
try {
- test("select employee_id from dfs_test.`%s/xsort/2batches` order by employee_id", TEST_RES_PATH);
+ test("select employee_id from dfs.`xsort/2batches` order by employee_id");
fail("Query should have failed!");
} catch (UserRemoteException e) {
assertEquals(ErrorType.RESOURCE, e.getErrorType());
@@ -101,7 +105,6 @@ public class TestSortSpillWithException extends ClusterTest {
// run a simple order by query
try {
test("SELECT id_i, name_s250 FROM `mock`.`employee_500K` ORDER BY id_i");
-// test("select employee_id from dfs_test.`%s/xsort/2batches` order by employee_id", TEST_RES_PATH);
fail("Query should have failed!");
} catch (UserRemoteException e) {
assertEquals(ErrorType.RESOURCE, e.getErrorType());
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java
index 62c961d..ec85f21 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/BasicPhysicalOpUnitTest.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.physical.unit;
-import static org.apache.drill.TestBuilder.mapOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java
index ca226e6..754d858 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java
@@ -21,7 +21,7 @@ package org.apache.drill.exec.physical.unit;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import mockit.NonStrictExpectations;
-import org.apache.drill.DrillTestWrapper;
+import org.apache.drill.test.DrillTestWrapper;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.impl.BatchCreator;
@@ -35,7 +35,6 @@ import org.apache.drill.exec.store.dfs.DrillFileSystem;
import org.apache.drill.exec.store.parquet.ParquetDirectByteBufferAllocator;
import org.apache.drill.exec.store.parquet.ParquetReaderUtility;
import org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader;
-import org.apache.drill.exec.util.TestUtilities;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.CodecFactory;
import org.apache.parquet.hadoop.ParquetFileReader;
@@ -51,7 +50,6 @@ import java.util.TreeMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import static org.apache.drill.DrillTestWrapper.addToCombinedVectorResults;
import static org.apache.drill.exec.physical.base.AbstractBase.INIT_ALLOCATION;
import static org.apache.drill.exec.physical.base.AbstractBase.MAX_ALLOCATION;
@@ -403,9 +401,9 @@ public class MiniPlanUnitTestBase extends PhysicalOpUnitTestBase {
Iterator<RecordReader> readers = null;
if (jsonBatches != null) {
- readers = TestUtilities.getJsonReadersFromBatchString(jsonBatches, fragContext, columnsToRead);
+ readers = getJsonReadersFromBatchString(jsonBatches, fragContext, columnsToRead);
} else {
- readers = TestUtilities.getJsonReadersFromInputFiles(fs, inputPaths, fragContext, columnsToRead);
+ readers = getJsonReadersFromInputFiles(fs, inputPaths, fragContext, columnsToRead);
}
List<RecordReader> readerList = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java
index 26345c9..fda4442 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.physical.unit;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import mockit.Delegate;
@@ -26,7 +28,9 @@ import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
import org.apache.calcite.rel.RelFieldCollation;
-import org.apache.drill.DrillTestWrapper;
+import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.exec.store.easy.json.JSONRecordReader;
+import org.apache.drill.test.DrillTestWrapper;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FieldReference;
@@ -64,9 +68,9 @@ import org.apache.drill.exec.physical.impl.project.ProjectorTemplate;
import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.testing.ExecutionControls;
-import org.apache.drill.exec.util.TestUtilities;
import org.junit.Before;
import java.io.IOException;
@@ -82,10 +86,8 @@ import java.util.Map;
* Look! Doesn't extend BaseTestQuery!!
*/
public class PhysicalOpUnitTestBase extends ExecTest {
-// public static long INIT_ALLOCATION = 1_000_000l;
-// public static long MAX_ALLOCATION = 10_000_000L;
-
@Injectable FragmentContext fragContext;
+ @Injectable DrillbitContext drillbitContext;
@Injectable OperatorContext opContext;
@Injectable OperatorStats opStats;
@Injectable PhysicalOperator popConf;
@@ -295,6 +297,7 @@ public class PhysicalOpUnitTestBase extends ExecTest {
// // string options
// optManager.getOption(withAny(new TypeValidators.StringValidator("", "try"))); result = "try";
// optManager.getOption(withAny(new TypeValidators.PositiveLongValidator("", 1l, 1l))); result = 10;
+ drillbitContext.getCompiler(); result = new CodeCompiler(drillConf, optionManager);
fragContext.getOptions(); result = optionManager;
fragContext.getOptionSet(); result = optionManager;
fragContext.getManagedBuffer(); result = bufManager.getManagedBuffer();
@@ -304,10 +307,11 @@ public class PhysicalOpUnitTestBase extends ExecTest {
fragContext.getConfig(); result = drillConf;
fragContext.getHandle(); result = ExecProtos.FragmentHandle.getDefaultInstance();
fragContext.getFunctionRegistry(); result = funcReg;
+ fragContext.getDrillbitContext(); result = drillbitContext;
try {
- CodeGenerator<?> cg = CodeGenerator.get(templateClassDefinition, funcReg);
+ CodeGenerator<?> cg = CodeGenerator.get(templateClassDefinition);
cg.plainJavaCapable(true);
-// cg.saveCodeForDebugging(true);
+ // cg.saveCodeForDebugging(true);
fragContext.getImplementationClass(withAny(cg));
result = new Delegate<Object>()
{
@@ -316,7 +320,7 @@ public class PhysicalOpUnitTestBase extends ExecTest {
return compiler.createInstance(gen);
}
};
- fragContext.getImplementationClass(withAny(CodeGenerator.get(templateClassDefinition, funcReg).getRoot()));
+ fragContext.getImplementationClass(withAny(CodeGenerator.get(templateClassDefinition).getRoot()));
result = new Delegate<Object>()
{
@SuppressWarnings("unused")
@@ -352,7 +356,7 @@ public class PhysicalOpUnitTestBase extends ExecTest {
}
private Iterator<RecordReader> getRecordReadersForJsonBatches(List<String> jsonBatches, FragmentContext fragContext) {
- return TestUtilities.getJsonReadersFromBatchString(jsonBatches, fragContext, Collections.singletonList(SchemaPath.getSimplePath("*")));
+ return getJsonReadersFromBatchString(jsonBatches, fragContext, Collections.singletonList(SchemaPath.getSimplePath("*")));
}
private List<RecordReader> getReaderListForJsonBatches(List<String> jsonBatches, FragmentContext fragContext) {
@@ -364,4 +368,43 @@ public class PhysicalOpUnitTestBase extends ExecTest {
return readerList;
}
+
+ /**
+ * Create JSONRecordReader from input strings.
+ * @param jsonBatches : list of input strings, each element represent a batch. Each string could either
+ * be in the form of "[{...}, {...}, ..., {...}]", or in the form of "{...}".
+ * @param fragContext : fragment context
+ * @param columnsToRead : list of schema paths to read from JSON reader.
+ * @return
+ */
+ public static Iterator<RecordReader> getJsonReadersFromBatchString(List<String> jsonBatches, FragmentContext fragContext, List<SchemaPath> columnsToRead) {
+ ObjectMapper mapper = new ObjectMapper();
+ List<RecordReader> readers = new ArrayList<>();
+ for (String batchJason : jsonBatches) {
+ JsonNode records;
+ try {
+ records = mapper.readTree(batchJason);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ readers.add(new JSONRecordReader(fragContext, records, null, columnsToRead));
+ }
+ return readers.iterator();
+ }
+
+ /**
+ * Create JSONRecordReader from files on a file system.
+ * @param fs : file system.
+ * @param inputPaths : list of .json file paths.
+ * @param fragContext
+ * @param columnsToRead
+ * @return
+ */
+ public static Iterator<RecordReader> getJsonReadersFromInputFiles(DrillFileSystem fs, List<String> inputPaths, FragmentContext fragContext, List<SchemaPath> columnsToRead) {
+ List<RecordReader> readers = new ArrayList<>();
+ for (String inputPath : inputPaths) {
+ readers.add(new JSONRecordReader(fragContext, inputPath, fs, columnsToRead));
+ }
+ return readers.iterator();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java
index 43b1c9d..79b9dbc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestMiniPlan.java
@@ -21,7 +21,7 @@ package org.apache.drill.exec.physical.unit;
import com.google.common.collect.Lists;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.physical.config.Filter;
import org.apache.drill.exec.physical.config.UnionAll;
import org.apache.drill.exec.record.BatchSchema;
@@ -57,7 +57,7 @@ public class TestMiniPlan extends MiniPlanUnitTestBase {
@Test
public void testSimpleParquetScan() throws Exception {
- String file = FileUtils.getResourceAsFile("/tpchmulti/region/01.parquet").toURI().toString();
+ String file = DrillFileUtils.getResourceAsFile("/tpchmulti/region/01.parquet").toURI().toString();
RecordBatch scanBatch = new ParquetScanBuilder()
.fileSystem(fs)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestNullInputMiniPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestNullInputMiniPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestNullInputMiniPlan.java
index 1127314..c8dcd28 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestNullInputMiniPlan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestNullInputMiniPlan.java
@@ -23,7 +23,7 @@ import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.config.ExternalSort;
import org.apache.drill.exec.physical.config.Filter;
@@ -272,7 +272,7 @@ public class TestNullInputMiniPlan extends MiniPlanUnitTestBase{
RecordBatch left = createScanBatchFromJson(SINGLE_EMPTY_JSON);
- String file = FileUtils.getResourceAsFile("/tpchmulti/region/01.parquet").toURI().toString();
+ String file = DrillFileUtils.getResourceAsFile("/tpchmulti/region/01.parquet").toURI().toString();
RecordBatch scanBatch = new ParquetScanBuilder()
.fileSystem(fs)
@@ -558,7 +558,7 @@ public class TestNullInputMiniPlan extends MiniPlanUnitTestBase{
List<String> inputPaths = new ArrayList<>();
for (String resource : resourcePaths) {
- inputPaths.add(FileUtils.getResourceAsFile(resource).toURI().toString());
+ inputPaths.add(DrillFileUtils.getResourceAsFile(resource).toURI().toString());
}
RecordBatch scanBatch = new JsonScanBuilder()
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
index 2407ee4..1a1127d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
@@ -17,30 +17,27 @@
*/
package org.apache.drill.exec.planner;
+import java.io.File;
import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import com.google.common.collect.ImmutableMap;
+import org.apache.commons.io.FileUtils;
import org.apache.drill.PlanTestBase;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.common.exceptions.UserRemoteException;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.fn.interp.TestConstantFolding;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.drill.exec.util.TestUtilities;
+import org.apache.drill.exec.util.StoragePluginTestUtils;
import org.apache.drill.exec.util.Text;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TemporaryFolder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -61,13 +58,11 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
}
private static List<ConstantFoldingTestConfig> tests;
- private String path;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
+ private static Path path;
+ private static File test;
@BeforeClass
- public static void init() {
+ public static void init() throws Exception {
// Need the suffixes to make the names unique in the directory.
// The capitalized name is on the opposite function (imaxdir and mindir)
// because they are looking on opposite ends of the list.
@@ -82,18 +77,16 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
.add(new ConstantFoldingTestConfig("MINDIR", "BIGFILE_2"))
.add(new ConstantFoldingTestConfig("IMINDIR", "bigfile"))
.build();
- }
- @Before
- public void setup() throws Exception {
- new TestConstantFolding.SmallFileCreator(folder).createFiles(1, 1000);
- path = folder.getRoot().toPath().toString();
- }
+ path = Paths.get("test");
+ test = dirTestWatcher.makeRootSubDir(path);
+ dirTestWatcher.copyResourceToRoot(Paths.get("parquet"));
+ new TestConstantFolding.SmallFileCreator(test).createFiles(1, 1000);
+ }
@Test
public void testConstExprFolding_maxDir0() throws Exception {
-
test("use dfs.root");
List<String> allFiles = ImmutableList.<String>builder()
@@ -103,7 +96,7 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
.add("BIGFILE_2")
.build();
- String query = "select * from dfs.`" + path + "/*/*.csv` where dir0 = %s('dfs.root','" + path + "')";
+ String query = "select * from dfs.`%s/*/*.csv` where dir0 = %s('dfs.root','%s')";
for (ConstantFoldingTestConfig config : tests) {
// make all of the other folders unexpected patterns, except for the one expected in this case
List<String> excludedPatterns = Lists.newArrayList();
@@ -114,7 +107,7 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
String[] excludedArray = new String[excludedPatterns.size()];
testPlanMatchingPatterns(
- String.format(query, config.funcName),
+ String.format(query, path, config.funcName, path),
new String[] {config.expectedFolderName},
excludedPatterns.toArray(excludedArray));
}
@@ -126,7 +119,7 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
list.add(new Text("3"));
testBuilder()
- .sqlQuery(String.format(query, tests.get(0).funcName))
+ .sqlQuery(query, path, tests.get(0).funcName, path)
.unOrdered()
.baselineColumns("columns", "dir0")
.baselineValues(list, tests.get(0).expectedFolderName)
@@ -148,7 +141,7 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
for (Map.Entry<String, String> configEntry : configMap.entrySet()) {
for (ConstantFoldingTestConfig functionConfig : tests) {
try {
- test(String.format(configEntry.getKey(), functionConfig.funcName));
+ test(configEntry.getKey(), functionConfig.funcName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(
String.format("Directory explorers [MAXDIR, IMAXDIR, MINDIR, IMINDIR] functions are not supported in %s", configEntry.getValue())));
@@ -164,7 +157,7 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
String query = "select * from dfs.`" + path + "/*/*.csv` where dir0 = %s('dfs.root','" + path + "')";
for (ConstantFoldingTestConfig config : tests) {
try {
- test(String.format(query, config.funcName));
+ test(query, config.funcName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString("Directory explorers [MAXDIR, IMAXDIR, MINDIR, IMINDIR] functions can not be used " +
"when planner.enable_constant_folding option is set to false"));
@@ -180,31 +173,34 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
//Initially update the location of dfs_test.tmp workspace with "path" temp directory just for use in this UTest
final StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
try {
- TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, path);
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry, test, StoragePluginTestUtils.TMP_SCHEMA);
//Results comparison of using Query Directory Functions (MAXDIR, IMAXDIR, MINDIR, IMINDIR) with one and two arguments
- String queryWithTwoArgFunc = "select * from dfs.`" + path + "/*/*.csv` where dir0 = %s('dfs.root','" + path + "')";
- String queryWithOneArgFunc = "select * from dfs.`" + path + "/*/*.csv` where dir0 = %s('dfs_test.tmp')";
- for (ConstantFoldingTestConfig config : tests) {
- testBuilder()
- .sqlQuery(String.format(queryWithOneArgFunc, config.funcName))
- .unOrdered()
- .sqlBaselineQuery(String.format(queryWithTwoArgFunc, config.funcName))
- .go();
- }
+ String queryWithTwoArgFunc = "select * from dfs.`" + path + "/*/*.csv` where dir0 = %s('dfs.root','" + path + "')";
+ String queryWithOneArgFunc = "select * from dfs.`" + path + "/*/*.csv` where dir0 = %s('dfs.tmp')";
+ for (ConstantFoldingTestConfig config : tests) {
+ testBuilder()
+ .sqlQuery(queryWithOneArgFunc, config.funcName)
+ .unOrdered()
+ .sqlBaselineQuery(queryWithTwoArgFunc, config.funcName)
+ .go();
+ }
} finally {
- TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, getDfsTestTmpSchemaLocation());
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry, dirTestWatcher.getDfsTestTmpDir(), StoragePluginTestUtils.TMP_SCHEMA);
}
}
@Test // DRILL-4720
public void testDirectoryUDFsWithAndWithoutMetadataCache() throws Exception {
- FileSystem fs = getLocalFileSystem();
// prepare test table with partitions
- Path table = new Path(getTempDir("table_with_partitions"));
- String tablePath = table.toUri().getPath();
- Path dataFile = new Path(TestTools.getWorkingPath(),"src/test/resources/parquet/alltypes_required.parquet");
- createPartitions(fs, table, dataFile, 2);
+ Path tableRelPath = Paths.get("table_with_partitions");
+ File tableDir = dirTestWatcher.makeRootSubDir(tableRelPath);
+ File dataFile = dirTestWatcher
+ .getRootDir()
+ .toPath()
+ .resolve(Paths.get("parquet", "alltypes_required.parquet"))
+ .toFile();
+ createPartitions(tableDir.toPath(), dataFile, 2);
Map<String, String> configurations = ImmutableMap.<String, String>builder()
.put("mindir", "part_1")
@@ -218,7 +214,7 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
// run tests without metadata cache
for (Map.Entry<String, String> entry : configurations.entrySet()) {
testBuilder()
- .sqlQuery(query, tablePath, entry.getKey(), tablePath)
+ .sqlQuery(query, tableRelPath, entry.getKey(), tableRelPath)
.unOrdered()
.baselineColumns("dir0")
.baselineValues(entry.getValue())
@@ -226,12 +222,12 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
}
// generate metadata
- test("refresh table metadata dfs.`%s`", tablePath);
+ test("refresh table metadata dfs.`%s`", tableRelPath);
// run tests with metadata cache
for (Map.Entry<String, String> entry : configurations.entrySet()) {
testBuilder()
- .sqlQuery(query, tablePath, entry.getKey(), tablePath)
+ .sqlQuery(query, tableRelPath, entry.getKey(), tableRelPath)
.unOrdered()
.baselineColumns("dir0")
.baselineValues(entry.getValue())
@@ -239,12 +235,11 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
}
}
- private void createPartitions(FileSystem fs, Path table, Path dataFile, int number) throws IOException {
+ private void createPartitions(Path table, File dataFile, int number) throws IOException {
for (int i = 1; i <= number; i++) {
- Path partition = new Path(table, "part_" + i);
- fs.mkdirs(partition);
- FileUtil.copy(fs, dataFile, fs, partition, false, true, fs.getConf());
+ final Path partition = table.resolve("part_" + i);
+ final File destFile = partition.resolve(dataFile.getName()).toFile();
+ FileUtils.copyFile(dataFile, destFile);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java
index 631cabd..fe0d433 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestCaseNullableTypes.java
@@ -16,7 +16,7 @@
*/
package org.apache.drill.exec.planner.logical;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlTest;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java
index 87c2c1c..cb7f22c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/TestConvertCountToDirectScan.java
@@ -20,17 +20,25 @@ package org.apache.drill.exec.planner.logical;
import org.apache.drill.PlanTestBase;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.exec.ExecConstants;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
@Category(PlannerTest.class)
public class TestConvertCountToDirectScan extends PlanTestBase {
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("directcount.parquet"));
+ }
+
@Test
public void ensureCaseDoesntConvertToDirectScan() throws Exception {
testPlanMatchingPatterns(
"select count(case when n_name = 'ALGERIA' and n_regionkey = 2 then n_nationkey else null end) as cnt\n" +
- "from dfs.`${WORKING_PATH}/src/test/resources/directcount.parquet`",
+ "from dfs.`directcount.parquet`",
new String[] { "CASE" },
new String[]{});
}
@@ -117,7 +125,7 @@ public class TestConvertCountToDirectScan extends PlanTestBase {
@Test
public void ensureConvertForSeveralColumns() throws Exception {
- test("use %s", TEMP_SCHEMA);
+ test("use dfs.tmp");
final String tableName = "parquet_table_counts";
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestDrillSQLWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestDrillSQLWorker.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestDrillSQLWorker.java
index cddc3ae..57c5de5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestDrillSQLWorker.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestDrillSQLWorker.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.calcite.sql.parser.SqlParserPos;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java
index b76aad3..bbe2b63 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/PopUnitTestBase.java
@@ -20,9 +20,9 @@ package org.apache.drill.exec.pop;
import java.io.IOException;
import java.util.Properties;
-import org.apache.drill.QueryTestUtil;
+import org.apache.drill.test.QueryTestUtil;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.exception.FragmentSetupException;
@@ -70,7 +70,7 @@ public abstract class PopUnitTestBase extends ExecTest{
public static Fragment getRootFragment(PhysicalPlanReader reader, String file) throws FragmentSetupException,
IOException, ForemanSetupException {
- return getRootFragmentFromPlanString(reader, Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
+ return getRootFragmentFromPlanString(reader, Files.toString(DrillFileUtils.getResourceAsFile(file), Charsets.UTF_8));
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java
index 2637a84..be326a2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
@@ -52,7 +52,7 @@ public class TestInjectionValue extends ExecTest {
@Test
public void testInjected() throws Exception{
PhysicalPlanReader r = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(config);
- PhysicalPlan p = r.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/physical_screen.json"), Charsets.UTF_8));
+ PhysicalPlan p = r.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/physical_screen.json"), Charsets.UTF_8));
List<PhysicalOperator> o = p.getSortedOperators(false);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestRecordIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestRecordIterator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestRecordIterator.java
index b3dd4c7..c07dbaa 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestRecordIterator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestRecordIterator.java
@@ -26,7 +26,7 @@ import mockit.Injectable;
import org.apache.drill.categories.VectorTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OpProfileDef;
@@ -65,7 +65,7 @@ public class TestRecordIterator extends PopUnitTestBase {
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final String planStr = Files.toString(FileUtils.getResourceAsFile("/record/test_recorditerator.json"), Charsets.UTF_8);
+ final String planStr = Files.toString(DrillFileUtils.getResourceAsFile("/record/test_recorditerator.json"), Charsets.UTF_8);
final PhysicalPlan plan = reader.readPhysicalPlan(planStr);
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
@@ -122,7 +122,7 @@ public class TestRecordIterator extends PopUnitTestBase {
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final String planStr = Files.toString(FileUtils.getResourceAsFile("/record/test_recorditerator.json"), Charsets.UTF_8);
+ final String planStr = Files.toString(DrillFileUtils.getResourceAsFile("/record/test_recorditerator.json"), Charsets.UTF_8);
final PhysicalPlan plan = reader.readPhysicalPlan(planStr);
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
index 930bbbc..3fa9541 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
@@ -24,7 +24,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.drill.categories.VectorTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -58,7 +58,7 @@ public class TestDateTypes extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/record/vector/test_date.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/record/vector/test_date.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/test_simple_date.json"));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
@@ -92,7 +92,7 @@ public class TestDateTypes extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/record/vector/test_sort_date.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/record/vector/test_sort_date.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/test_simple_date.json"));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
@@ -126,7 +126,7 @@ public class TestDateTypes extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/record/vector/test_timestamp.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/record/vector/test_timestamp.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/test_simple_date.json"));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
@@ -160,7 +160,7 @@ public class TestDateTypes extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/record/vector/test_interval.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/record/vector/test_interval.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/test_simple_interval.json"));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
@@ -218,7 +218,7 @@ public class TestDateTypes extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/record/vector/test_all_date_literals.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/record/vector/test_all_date_literals.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/test_simple_date.json"));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
@@ -257,7 +257,7 @@ public class TestDateTypes extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/record/vector/test_date_add.json"), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile("/record/vector/test_date_add.json"), Charsets.UTF_8)
.replace("#{TEST_FILE}", "/test_simple_date.json"));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
[20/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
deleted file mode 100644
index 05ca6ea..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ /dev/null
@@ -1,713 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill;
-
-import static org.hamcrest.core.StringContains.containsString;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.URL;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.drill.DrillTestWrapper.TestServices;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.config.DrillProperties;
-import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.scanner.ClassPathScanner;
-import org.apache.drill.common.scanner.persistence.ScanResult;
-import org.apache.drill.common.util.TestTools;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.ExecTest;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.memory.RootAllocatorFactory;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
-import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
-import org.apache.drill.exec.proto.UserBitShared.QueryType;
-import org.apache.drill.exec.proto.UserProtos.PreparedStatementHandle;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.rpc.ConnectionThrottle;
-import org.apache.drill.exec.rpc.user.AwaitableUserResultsListener;
-import org.apache.drill.exec.rpc.user.QueryDataBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.store.StoragePluginRegistry;
-import org.apache.drill.exec.util.TestUtilities;
-import org.apache.drill.exec.util.VectorUtil;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.rules.TestRule;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.io.Resources;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.test.ClusterFixture;
-
-public class BaseTestQuery extends ExecTest {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class);
-
- public static final String TEST_SCHEMA = "dfs_test";
- public static final String TEMP_SCHEMA = TEST_SCHEMA + ".tmp";
-
- private static final int MAX_WIDTH_PER_NODE = 2;
-
- @SuppressWarnings("serial")
- private static final Properties TEST_CONFIGURATIONS = new Properties() {
- {
- put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false");
- put(ExecConstants.HTTP_ENABLE, "false");
- // Increasing retry attempts for testing
- put(ExecConstants.UDF_RETRY_ATTEMPTS, "10");
- put(ExecConstants.SSL_USE_HADOOP_CONF, "false");
- }
- };
-
- public final TestRule resetWatcher = new TestWatcher() {
- @Override
- protected void failed(Throwable e, Description description) {
- try {
- resetClientAndBit();
- } catch (Exception e1) {
- throw new RuntimeException("Failure while resetting client.", e1);
- }
- }
- };
-
- protected static DrillClient client;
- protected static Drillbit[] bits;
- protected static RemoteServiceSet serviceSet;
- protected static DrillConfig config;
- protected static BufferAllocator allocator;
-
- /**
- * Number of Drillbits in test cluster. Default is 1.
- *
- * Tests can update the cluster size through {@link #updateTestCluster(int, DrillConfig)}
- */
- private static int drillbitCount = 1;
-
- /**
- * Location of the dfs_test.tmp schema on local filesystem.
- */
- private static String dfsTestTmpSchemaLocation;
-
- private int[] columnWidths = new int[] { 8 };
-
- private static ScanResult classpathScan;
-
- private static FileSystem fs;
-
- @BeforeClass
- public static void setupDefaultTestCluster() throws Exception {
- config = DrillConfig.create(TEST_CONFIGURATIONS);
- classpathScan = ClassPathScanner.fromPrescan(config);
- openClient();
- // turns on the verbose errors in tests
- // sever side stacktraces are added to the message before sending back to the client
- test("ALTER SESSION SET `exec.errors.verbose` = true");
- fs = getLocalFileSystem();
- }
-
- protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig) {
- updateTestCluster(newDrillbitCount, newConfig, null);
- }
-
- protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig, Properties properties) {
- Preconditions.checkArgument(newDrillbitCount > 0, "Number of Drillbits must be at least one");
- if (drillbitCount != newDrillbitCount || config != null) {
- // TODO: Currently we have to shutdown the existing Drillbit cluster before starting a new one with the given
- // Drillbit count. Revisit later to avoid stopping the cluster.
- try {
- closeClient();
- drillbitCount = newDrillbitCount;
- if (newConfig != null) {
- // For next test class, updated DrillConfig will be replaced by default DrillConfig in BaseTestQuery as part
- // of the @BeforeClass method of test class.
- config = newConfig;
- }
- openClient(properties);
- } catch(Exception e) {
- throw new RuntimeException("Failure while updating the test Drillbit cluster.", e);
- }
- }
- }
-
- /**
- * Useful for tests that require a DrillbitContext to get/add storage plugins, options etc.
- *
- * @return DrillbitContext of first Drillbit in the cluster.
- */
- protected static DrillbitContext getDrillbitContext() {
- Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup.");
- return bits[0].getContext();
- }
-
- protected static Properties cloneDefaultTestConfigProperties() {
- final Properties props = new Properties();
- for(String propName : TEST_CONFIGURATIONS.stringPropertyNames()) {
- props.put(propName, TEST_CONFIGURATIONS.getProperty(propName));
- }
-
- return props;
- }
-
- protected static String getDfsTestTmpSchemaLocation() {
- return dfsTestTmpSchemaLocation;
- }
-
- private static void resetClientAndBit() throws Exception{
- closeClient();
- openClient();
- }
-
- private static void openClient() throws Exception {
- openClient(null);
- }
-
- private static void openClient(Properties properties) throws Exception {
- if (properties == null) {
- properties = new Properties();
- }
-
- allocator = RootAllocatorFactory.newRoot(config);
- serviceSet = RemoteServiceSet.getLocalServiceSet();
-
- dfsTestTmpSchemaLocation = TestUtilities.createTempDir();
-
- bits = new Drillbit[drillbitCount];
- for(int i = 0; i < drillbitCount; i++) {
- bits[i] = new Drillbit(config, serviceSet, classpathScan);
- bits[i].run();
-
- @SuppressWarnings("resource")
- final StoragePluginRegistry pluginRegistry = bits[i].getContext().getStorage();
- TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTmpSchemaLocation);
- TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry);
- }
-
- if (!properties.containsKey(DrillProperties.DRILLBIT_CONNECTION)) {
- properties.setProperty(DrillProperties.DRILLBIT_CONNECTION,
- String.format("localhost:%s", bits[0].getUserPort()));
- }
-
- DrillConfig clientConfig = DrillConfig.forClient();
- client = QueryTestUtil.createClient(clientConfig, serviceSet, MAX_WIDTH_PER_NODE, properties);
- }
-
- /**
- * Close the current <i>client</i> and open a new client using the given <i>properties</i>. All tests executed
- * after this method call use the new <i>client</i>.
- *
- * @param properties
- */
- public static void updateClient(Properties properties) throws Exception {
- Preconditions.checkState(bits != null && bits[0] != null, "Drillbits are not setup.");
- if (client != null) {
- client.close();
- client = null;
- }
-
- DrillConfig clientConfig = DrillConfig.forClient();
- client = QueryTestUtil.createClient(clientConfig, serviceSet, MAX_WIDTH_PER_NODE, properties);
- }
-
- /*
- * Close the current <i>client</i> and open a new client for the given user. All tests executed
- * after this method call use the new <i>client</i>.
- * @param user
- */
- public static void updateClient(String user) throws Exception {
- updateClient(user, null);
- }
-
- /*
- * Close the current <i>client</i> and open a new client for the given user and password credentials. Tests
- * executed after this method call use the new <i>client</i>.
- * @param user
- */
- public static void updateClient(final String user, final String password) throws Exception {
- final Properties props = new Properties();
- props.setProperty(DrillProperties.USER, user);
- if (password != null) {
- props.setProperty(DrillProperties.PASSWORD, password);
- }
- updateClient(props);
- }
-
- protected static BufferAllocator getAllocator() {
- return allocator;
- }
-
- public static int getUserPort() {
- return bits[0].getUserPort();
- }
-
- public static TestBuilder newTest() {
- return testBuilder();
- }
-
-
- public static class ClassicTestServices implements TestServices {
- @Override
- public BufferAllocator allocator() {
- return allocator;
- }
-
- @Override
- public void test(String query) throws Exception {
- BaseTestQuery.test(query);
- }
-
- @Override
- public List<QueryDataBatch> testRunAndReturn(final QueryType type, final Object query) throws Exception {
- return BaseTestQuery.testRunAndReturn(type, query);
- }
- }
-
- public static TestBuilder testBuilder() {
- return new TestBuilder(new ClassicTestServices());
- }
-
- @AfterClass
- public static void closeClient() throws Exception {
- if (client != null) {
- client.close();
- }
-
- if (bits != null) {
- for(final Drillbit bit : bits) {
- if (bit != null) {
- bit.close();
- }
- }
- }
-
- if(serviceSet != null) {
- serviceSet.close();
- }
- if (allocator != null) {
- allocator.close();
- }
- }
-
- @AfterClass
- public static void resetDrillbitCount() {
- // some test classes assume this value to be 1 and will fail if run along other tests that increase it
- drillbitCount = 1;
- }
-
- protected static void runSQL(String sql) throws Exception {
- final AwaitableUserResultsListener listener = new AwaitableUserResultsListener(new SilentListener());
- testWithListener(QueryType.SQL, sql, listener);
- listener.await();
- }
-
- protected static List<QueryDataBatch> testSqlWithResults(String sql) throws Exception{
- return testRunAndReturn(QueryType.SQL, sql);
- }
-
- protected static List<QueryDataBatch> testLogicalWithResults(String logical) throws Exception{
- return testRunAndReturn(QueryType.LOGICAL, logical);
- }
-
- protected static List<QueryDataBatch> testPhysicalWithResults(String physical) throws Exception{
- return testRunAndReturn(QueryType.PHYSICAL, physical);
- }
-
- public static List<QueryDataBatch> testRunAndReturn(QueryType type, Object query) throws Exception{
- if (type == QueryType.PREPARED_STATEMENT) {
- Preconditions.checkArgument(query instanceof PreparedStatementHandle,
- "Expected an instance of PreparedStatement as input query");
- return testPreparedStatement((PreparedStatementHandle)query);
- } else {
- Preconditions.checkArgument(query instanceof String, "Expected a string as input query");
- query = QueryTestUtil.normalizeQuery((String)query);
- return client.runQuery(type, (String)query);
- }
- }
-
- public static List<QueryDataBatch> testPreparedStatement(PreparedStatementHandle handle) throws Exception {
- return client.executePreparedStatement(handle);
- }
-
- public static int testRunAndPrint(final QueryType type, final String query) throws Exception {
- return QueryTestUtil.testRunAndPrint(client, type, query);
- }
-
- protected static void testWithListener(QueryType type, String query, UserResultsListener resultListener) {
- QueryTestUtil.testWithListener(client, type, query, resultListener);
- }
-
- public static void testNoResult(String query, Object... args) throws Exception {
- testNoResult(1, query, args);
- }
-
- public static void alterSession(String option, Object value) {
- String valueStr = ClusterFixture.stringify(value);
- try {
- test("ALTER SESSION SET `%s` = %s", option, valueStr);
- } catch(final Exception e) {
- fail(String.format("Failed to set session option `%s` = %s, Error: %s",
- option, valueStr, e.toString()));
- }
- }
-
- public static void resetSessionOption(String option) {
- try {
- test("ALTER SESSION RESET `%s`", option);
- } catch(final Exception e) {
- fail(String.format("Failed to reset session option `%s`, Error: %s",
- option, e.toString()));
- }
- }
-
- protected static void testNoResult(int interation, String query, Object... args) throws Exception {
- query = String.format(query, args);
- logger.debug("Running query:\n--------------\n" + query);
- for (int i = 0; i < interation; i++) {
- final List<QueryDataBatch> results = client.runQuery(QueryType.SQL, query);
- for (final QueryDataBatch queryDataBatch : results) {
- queryDataBatch.release();
- }
- }
- }
-
- public static void test(String query, Object... args) throws Exception {
- QueryTestUtil.test(client, String.format(query, args));
- }
-
- public static void test(final String query) throws Exception {
- QueryTestUtil.test(client, query);
- }
-
- protected static int testLogical(String query) throws Exception{
- return testRunAndPrint(QueryType.LOGICAL, query);
- }
-
- protected static int testPhysical(String query) throws Exception{
- return testRunAndPrint(QueryType.PHYSICAL, query);
- }
-
- protected static int testSql(String query) throws Exception{
- return testRunAndPrint(QueryType.SQL, query);
- }
-
- protected static void testPhysicalFromFile(String file) throws Exception{
- testPhysical(getFile(file));
- }
-
- protected static List<QueryDataBatch> testPhysicalFromFileWithResults(String file) throws Exception {
- return testRunAndReturn(QueryType.PHYSICAL, getFile(file));
- }
-
- protected static void testLogicalFromFile(String file) throws Exception{
- testLogical(getFile(file));
- }
-
- protected static void testSqlFromFile(String file) throws Exception{
- test(getFile(file));
- }
-
- /**
- * Utility method which tests given query produces a {@link UserException} and the exception message contains
- * the given message.
- * @param testSqlQuery Test query
- * @param expectedErrorMsg Expected error message.
- */
- protected static void errorMsgTestHelper(final String testSqlQuery, final String expectedErrorMsg) throws Exception {
- try {
- test(testSqlQuery);
- fail("Expected a UserException when running " + testSqlQuery);
- } catch (final UserException actualException) {
- try {
- assertThat("message of UserException when running " + testSqlQuery, actualException.getMessage(), containsString(expectedErrorMsg));
- } catch (AssertionError e) {
- e.addSuppressed(actualException);
- throw e;
- }
- }
- }
-
- /**
- * Utility method which tests given query produces a {@link UserException}
- * with {@link org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType} being DrillPBError.ErrorType.PARSE
- * the given message.
- * @param testSqlQuery Test query
- */
- protected static void parseErrorHelper(final String testSqlQuery) throws Exception {
- errorMsgTestHelper(testSqlQuery, UserBitShared.DrillPBError.ErrorType.PARSE.name());
- }
-
- public static String getFile(String resource) throws IOException{
- final URL url = Resources.getResource(resource);
- if (url == null) {
- throw new IOException(String.format("Unable to find path %s.", resource));
- }
- return Resources.toString(url, Charsets.UTF_8);
- }
-
- /**
- * Copy the resource (ex. file on classpath) to a physical file on FileSystem.
- * @param resource
- * @return the file path
- * @throws IOException
- */
- public static String getPhysicalFileFromResource(final String resource) throws IOException {
- final File file = File.createTempFile("tempfile", ".txt");
- file.deleteOnExit();
- final PrintWriter printWriter = new PrintWriter(file);
- printWriter.write(BaseTestQuery.getFile(resource));
- printWriter.close();
-
- return file.getPath();
- }
-
- protected static void setSessionOption(final String option, final boolean value) {
- alterSession(option, value);
- }
-
- protected static void setSessionOption(final String option, final long value) {
- alterSession(option, value);
- }
-
- protected static void setSessionOption(final String option, final double value) {
- alterSession(option, value);
- }
-
- protected static void setSessionOption(final String option, final String value) {
- alterSession(option, value);
- }
-
- public static class SilentListener implements UserResultsListener {
- private final AtomicInteger count = new AtomicInteger();
-
- @Override
- public void submissionFailed(UserException ex) {
- logger.debug("Query failed: " + ex.getMessage());
- }
-
- @Override
- public void queryCompleted(QueryState state) {
- logger.debug("Query completed successfully with row count: " + count.get());
- }
-
- @Override
- public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) {
- final int rows = result.getHeader().getRowCount();
- if (result.getData() != null) {
- count.addAndGet(rows);
- }
- result.release();
- }
-
- @Override
- public void queryIdArrived(QueryId queryId) {}
-
- }
-
- protected void setColumnWidth(int columnWidth) {
- this.columnWidths = new int[] { columnWidth };
- }
-
- protected void setColumnWidths(int[] columnWidths) {
- this.columnWidths = columnWidths;
- }
-
- protected int printResult(List<QueryDataBatch> results) throws SchemaChangeException {
- int rowCount = 0;
- final RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
- for(final QueryDataBatch result : results) {
- rowCount += result.getHeader().getRowCount();
- loader.load(result.getHeader().getDef(), result.getData());
- // TODO: Clean: DRILL-2933: That load(...) no longer throws
- // SchemaChangeException, so check/clean throw clause above.
- VectorUtil.showVectorAccessibleContent(loader, columnWidths);
- loader.clear();
- result.release();
- }
- System.out.println("Total record count: " + rowCount);
- return rowCount;
- }
-
- protected static String getResultString(List<QueryDataBatch> results, String delimiter)
- throws SchemaChangeException {
- final StringBuilder formattedResults = new StringBuilder();
- boolean includeHeader = true;
- final RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
- for(final QueryDataBatch result : results) {
- loader.load(result.getHeader().getDef(), result.getData());
- if (loader.getRecordCount() <= 0) {
- continue;
- }
- VectorUtil.appendVectorAccessibleContent(loader, formattedResults, delimiter, includeHeader);
- if (!includeHeader) {
- includeHeader = false;
- }
- loader.clear();
- result.release();
- }
-
- return formattedResults.toString();
- }
-
-
- public class TestResultSet {
-
- private final List<List<String>> rows;
-
- public TestResultSet() {
- rows = new ArrayList<>();
- }
-
- public TestResultSet(List<QueryDataBatch> batches) throws SchemaChangeException {
- rows = new ArrayList<>();
- convert(batches);
- }
-
- public void addRow(String... cells) {
- List<String> newRow = Arrays.asList(cells);
- rows.add(newRow);
- }
-
- public int size() {
- return rows.size();
- }
-
- @Override public boolean equals(Object o) {
- boolean result = false;
-
- if (this == o) {
- result = true;
- } else if (o instanceof TestResultSet) {
- TestResultSet that = (TestResultSet) o;
- assertEquals(this.size(), that.size());
- for (int i = 0; i < this.rows.size(); i++) {
- assertEquals(this.rows.get(i).size(), that.rows.get(i).size());
- for (int j = 0; j < this.rows.get(i).size(); ++j) {
- assertEquals(this.rows.get(i).get(j), that.rows.get(i).get(j));
- }
- }
- result = true;
- }
-
- return result;
- }
-
- private void convert(List<QueryDataBatch> batches) throws SchemaChangeException {
- RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
- for (QueryDataBatch batch : batches) {
- int rc = batch.getHeader().getRowCount();
- if (batch.getData() != null) {
- loader.load(batch.getHeader().getDef(), batch.getData());
- for (int i = 0; i < rc; ++i) {
- List<String> newRow = new ArrayList<>();
- rows.add(newRow);
- for (VectorWrapper<?> vw : loader) {
- ValueVector.Accessor accessor = vw.getValueVector().getAccessor();
- Object o = accessor.getObject(i);
- newRow.add(o == null ? null : o.toString());
- }
- }
- }
- loader.clear();
- batch.release();
- }
- }
- }
-
- private static String replaceWorkingPathInString(String orig) {
- return orig.replaceAll(Pattern.quote("[WORKING_PATH]"), Matcher.quoteReplacement(TestTools.getWorkingPath()));
- }
-
- protected static void copyDirectoryIntoTempSpace(String resourcesDir) throws IOException {
- copyDirectoryIntoTempSpace(resourcesDir, null);
- }
-
- protected static void copyDirectoryIntoTempSpace(String resourcesDir, String destinationSubDir) throws IOException {
- Path destination = destinationSubDir != null ? new Path(getDfsTestTmpSchemaLocation(), destinationSubDir)
- : new Path(getDfsTestTmpSchemaLocation());
- fs.copyFromLocalFile(
- new Path(replaceWorkingPathInString(resourcesDir)),
- destination);
- }
-
- protected static void copyMetaDataCacheToTempReplacingInternalPaths(String srcFileOnClassPath, String destFolderInTmp,
- String metaFileName) throws IOException {
- copyMetaDataCacheToTempWithReplacements(srcFileOnClassPath, destFolderInTmp, metaFileName, null);
- }
-
- protected static void copyMetaDataCacheToTempReplacingInternalPaths(Path srcFileOnClassPath, String destFolderInTmp,
- String metaFileName) throws IOException {
- copyMetaDataCacheToTempReplacingInternalPaths(srcFileOnClassPath.toUri().getPath(), destFolderInTmp, metaFileName);
- }
-
- /**
- * Old metadata cache files include full paths to the files that have been scanned.
- * <p>
- * There is no way to generate a metadata cache file with absolute paths that
- * will be guaranteed to be available on an arbitrary test machine.
- * <p>
- * To enable testing older metadata cache files, they were generated manually
- * using older drill versions, and the absolute path up to the folder where
- * the metadata cache file appeared was manually replaced with the string
- * REPLACED_IN_TEST. Here the file is re-written into the given temporary
- * location after the REPLACED_IN_TEST string has been replaced by the actual
- * location generated during this run of the tests.
- *
- * @param srcFileOnClassPath the source path of metadata cache file, which should be replaced
- * @param destFolderInTmp the parent folder name of the metadata cache file
- * @param metaFileName the name of metadata cache file depending on the type of the metadata
- * @param customStringReplacement custom string to replace the "CUSTOM_REPLACED" target string in metadata file
- * @throws IOException if a create or write errors occur
- */
- protected static void copyMetaDataCacheToTempWithReplacements(String srcFileOnClassPath,
- String destFolderInTmp, String metaFileName, String customStringReplacement) throws IOException {
- String metadataFileContents = getFile(srcFileOnClassPath);
- Path rootMeta = new Path(dfsTestTmpSchemaLocation, destFolderInTmp);
- Path newMetaCache = new Path(rootMeta, metaFileName);
- try (FSDataOutputStream outSteam = fs.create(newMetaCache)) {
- if (customStringReplacement != null) {
- metadataFileContents = metadataFileContents.replace("CUSTOM_STRING_REPLACEMENT", customStringReplacement);
- }
- outSteam.writeBytes(metadataFileContents.replace("REPLACED_IN_TEST", dfsTestTmpSchemaLocation));
- }
- }
-
- }
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
deleted file mode 100644
index 990a24d..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
+++ /dev/null
@@ -1,808 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.HyperVectorValueIterator;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.proto.UserBitShared.QueryType;
-import org.apache.drill.exec.record.BatchSchema;
-import org.apache.drill.exec.record.HyperVectorWrapper;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorAccessible;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.record.selection.SelectionVector2;
-import org.apache.drill.exec.record.selection.SelectionVector4;
-import org.apache.drill.exec.rpc.user.QueryDataBatch;
-import org.apache.drill.exec.util.Text;
-import org.apache.drill.exec.vector.ValueVector;
-
-/**
- * An object to encapsulate the options for a Drill unit test, as well as the execution methods to perform the tests and
- * validation of results.
- *
- * To construct an instance easily, look at the TestBuilder class. From an implementation of
- * the BaseTestQuery class, and instance of the builder is accessible through the testBuilder() method.
- */
-public class DrillTestWrapper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class);
-
- public interface TestServices {
- BufferAllocator allocator();
-
- void test(String query) throws Exception;
-
- List<QueryDataBatch> testRunAndReturn(QueryType type, Object query) throws Exception;
- }
-
- // TODO - when in JSON, read baseline in all text mode to avoid precision loss for decimal values
-
- // This flag will enable all of the values that are validated to be logged. For large validations this is time consuming
- // so this is not exposed in a way that it can be enabled for an individual test. It can be changed here while debugging
- // a test to see all of the output, but as this framework is doing full validation, there is no reason to keep it on as
- // it will only make the test slower.
- private static boolean VERBOSE_DEBUG = false;
-
- // Unit test doesn't expect any specific batch count
- public static final int EXPECTED_BATCH_COUNT_NOT_SET = -1;
-
- // The motivation behind the TestBuilder was to provide a clean API for test writers. The model is mostly designed to
- // prepare all of the components necessary for running the tests, before the TestWrapper is initialized. There is however
- // one case where the setup for the baseline is driven by the test query results, and this is implicit type enforcement
- // for the baseline data. In this case there needs to be a call back into the TestBuilder once we know the type information
- // from the test query.
- private TestBuilder testBuilder;
- /**
- * Test query to run. Type of object depends on the {@link #queryType}
- */
- private Object query;
- // The type of query provided
- private UserBitShared.QueryType queryType;
- // The type of query provided for the baseline
- private UserBitShared.QueryType baselineQueryType;
- // should ordering be enforced in the baseline check
- private boolean ordered;
- private TestServices services;
- // queries to run before the baseline or test queries, can be used to set options
- private String baselineOptionSettingQueries;
- private String testOptionSettingQueries;
- // two different methods are available for comparing ordered results, the default reads all of the records
- // into giant lists of objects, like one giant on-heap batch of 'vectors'
- // this flag enables the other approach which iterates through a hyper batch for the test query results and baseline
- // while this does work faster and use less memory, it can be harder to debug as all of the elements are not in a
- // single list
- private boolean highPerformanceComparison;
- // if the baseline is a single option test writers can provide the baseline values and columns
- // without creating a file, these are provided to the builder in the baselineValues() and baselineColumns() methods
- // and translated into a map in the builder
- private List<Map<String, Object>> baselineRecords;
-
- private int expectedNumBatches;
-
- public DrillTestWrapper(TestBuilder testBuilder, TestServices services, Object query, QueryType queryType,
- String baselineOptionSettingQueries, String testOptionSettingQueries,
- QueryType baselineQueryType, boolean ordered, boolean highPerformanceComparison,
- List<Map<String, Object>> baselineRecords, int expectedNumBatches) {
- this.testBuilder = testBuilder;
- this.services = services;
- this.query = query;
- this.queryType = queryType;
- this.baselineQueryType = baselineQueryType;
- this.ordered = ordered;
- this.baselineOptionSettingQueries = baselineOptionSettingQueries;
- this.testOptionSettingQueries = testOptionSettingQueries;
- this.highPerformanceComparison = highPerformanceComparison;
- this.baselineRecords = baselineRecords;
- this.expectedNumBatches = expectedNumBatches;
- }
-
- public void run() throws Exception {
- if (testBuilder.getExpectedSchema() != null) {
- compareSchemaOnly();
- } else {
- if (ordered) {
- compareOrderedResults();
- } else {
- compareUnorderedResults();
- }
- }
- }
-
- private BufferAllocator getAllocator() {
- return services.allocator();
- }
-
- private void compareHyperVectors(Map<String, HyperVectorValueIterator> expectedRecords,
- Map<String, HyperVectorValueIterator> actualRecords) throws Exception {
- for (String s : expectedRecords.keySet()) {
- assertNotNull("Expected column '" + s + "' not found.", actualRecords.get(s));
- assertEquals(expectedRecords.get(s).getTotalRecords(), actualRecords.get(s).getTotalRecords());
- HyperVectorValueIterator expectedValues = expectedRecords.get(s);
- HyperVectorValueIterator actualValues = actualRecords.get(s);
- int i = 0;
- while (expectedValues.hasNext()) {
- compareValuesErrorOnMismatch(expectedValues.next(), actualValues.next(), i, s);
- i++;
- }
- }
- cleanupHyperValueIterators(expectedRecords.values());
- cleanupHyperValueIterators(actualRecords.values());
- }
-
- private void cleanupHyperValueIterators(Collection<HyperVectorValueIterator> hyperBatches) {
- for (HyperVectorValueIterator hvi : hyperBatches) {
- for (ValueVector vv : hvi.getHyperVector().getValueVectors()) {
- vv.clear();
- }
- }
- }
-
- public static void compareMergedVectors(Map<String, List<Object>> expectedRecords, Map<String, List<Object>> actualRecords) throws Exception {
- for (String s : actualRecords.keySet()) {
- assertNotNull("Unexpected extra column " + s + " returned by query.", expectedRecords.get(s));
- assertEquals("Incorrect number of rows returned by query.", expectedRecords.get(s).size(), actualRecords.get(s).size());
- List<?> expectedValues = expectedRecords.get(s);
- List<?> actualValues = actualRecords.get(s);
- assertEquals("Different number of records returned", expectedValues.size(), actualValues.size());
-
- for (int i = 0; i < expectedValues.size(); i++) {
- try {
- compareValuesErrorOnMismatch(expectedValues.get(i), actualValues.get(i), i, s);
- } catch (Exception ex) {
- throw new Exception(ex.getMessage() + "\n\n" + printNearbyRecords(expectedRecords, actualRecords, i), ex);
- }
- }
- }
- if (actualRecords.size() < expectedRecords.size()) {
- throw new Exception(findMissingColumns(expectedRecords.keySet(), actualRecords.keySet()));
- }
- }
-
- private static String printNearbyRecords(Map<String, List<Object>> expectedRecords, Map<String, List<Object>> actualRecords, int offset) {
- StringBuilder expected = new StringBuilder();
- StringBuilder actual = new StringBuilder();
- expected.append("Expected Records near verification failure:\n");
- actual.append("Actual Records near verification failure:\n");
- int firstRecordToPrint = Math.max(0, offset - 5);
- List<?> expectedValuesInFirstColumn = expectedRecords.get(expectedRecords.keySet().iterator().next());
- List<?> actualValuesInFirstColumn = expectedRecords.get(expectedRecords.keySet().iterator().next());
- int numberOfRecordsToPrint = Math.min(Math.min(10, expectedValuesInFirstColumn.size()), actualValuesInFirstColumn.size());
- for (int i = firstRecordToPrint; i < numberOfRecordsToPrint; i++) {
- expected.append("Record Number: ").append(i).append(" { ");
- actual.append("Record Number: ").append(i).append(" { ");
- for (String s : actualRecords.keySet()) {
- List<?> actualValues = actualRecords.get(s);
- actual.append(s).append(" : ").append(actualValues.get(i)).append(",");
- }
- for (String s : expectedRecords.keySet()) {
- List<?> expectedValues = expectedRecords.get(s);
- expected.append(s).append(" : ").append(expectedValues.get(i)).append(",");
- }
- expected.append(" }\n");
- actual.append(" }\n");
- }
-
- return expected.append("\n\n").append(actual).toString();
-
- }
-
- private Map<String, HyperVectorValueIterator> addToHyperVectorMap(final List<QueryDataBatch> records,
- final RecordBatchLoader loader)
- throws SchemaChangeException, UnsupportedEncodingException {
- // TODO - this does not handle schema changes
- Map<String, HyperVectorValueIterator> combinedVectors = new TreeMap<>();
-
- long totalRecords = 0;
- QueryDataBatch batch;
- int size = records.size();
- for (int i = 0; i < size; i++) {
- batch = records.get(i);
- loader.load(batch.getHeader().getDef(), batch.getData());
- logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
- totalRecords += loader.getRecordCount();
- for (VectorWrapper<?> w : loader) {
- String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr();
- if (!combinedVectors.containsKey(field)) {
- MaterializedField mf = w.getField();
- ValueVector[] vvList = (ValueVector[]) Array.newInstance(mf.getValueClass(), 1);
- vvList[0] = w.getValueVector();
- combinedVectors.put(field, new HyperVectorValueIterator(mf, new HyperVectorWrapper<>(mf, vvList)));
- } else {
- combinedVectors.get(field).getHyperVector().addVector(w.getValueVector());
- }
-
- }
- }
- for (HyperVectorValueIterator hvi : combinedVectors.values()) {
- hvi.determineTotalSize();
- }
- return combinedVectors;
- }
-
- private static class BatchIterator implements Iterable<VectorAccessible>, AutoCloseable {
- private final List<QueryDataBatch> dataBatches;
- private final RecordBatchLoader batchLoader;
-
- public BatchIterator(List<QueryDataBatch> dataBatches, RecordBatchLoader batchLoader) {
- this.dataBatches = dataBatches;
- this.batchLoader = batchLoader;
- }
-
- @Override
- public Iterator<VectorAccessible> iterator() {
- return new Iterator<VectorAccessible>() {
-
- int index = -1;
-
- @Override
- public boolean hasNext() {
- return index < dataBatches.size() - 1;
- }
-
- @Override
- public VectorAccessible next() {
- index++;
- if (index == dataBatches.size()) {
- throw new RuntimeException("Tried to call next when iterator had no more items.");
- }
- batchLoader.clear();
- QueryDataBatch batch = dataBatches.get(index);
- try {
- batchLoader.load(batch.getHeader().getDef(), batch.getData());
- } catch (SchemaChangeException e) {
- throw new RuntimeException(e);
- }
- return batchLoader;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("Removing is not supported");
- }
- };
- }
-
- @Override
- public void close() throws Exception {
- batchLoader.clear();
- }
-
- }
-
- /**
- * Iterate over batches, and combine the batches into a map, where key is schema path, and value is
- * the list of column values across all the batches.
- * @param batches
- * @return
- * @throws SchemaChangeException
- * @throws UnsupportedEncodingException
- */
- public static Map<String, List<Object>> addToCombinedVectorResults(Iterable<VectorAccessible> batches)
- throws SchemaChangeException, UnsupportedEncodingException {
- Map<String, List<Object>> combinedVectors = new TreeMap<>();
- addToCombinedVectorResults(batches, null, combinedVectors);
- return combinedVectors;
- }
-
- /**
- * Add to result vectors and compare batch schema against expected schema while iterating batches.
- * @param batches
- * @param expectedSchema: the expected schema the batches should contain. Through SchemaChangeException
- * if encounter different batch schema.
- * @param combinedVectors: the vectors to hold the values when iterate the batches.
- *
- * @return number of batches
- * @throws SchemaChangeException
- * @throws UnsupportedEncodingException
- */
- public static int addToCombinedVectorResults(Iterable<VectorAccessible> batches, BatchSchema expectedSchema, Map<String, List<Object>> combinedVectors)
- throws SchemaChangeException, UnsupportedEncodingException {
- // TODO - this does not handle schema changes
- int numBatch = 0;
- long totalRecords = 0;
- BatchSchema schema = null;
- for (VectorAccessible loader : batches) {
- numBatch++;
- if (expectedSchema != null) {
- if (! expectedSchema.equals(loader.getSchema())) {
- throw new SchemaChangeException(String.format("Batch schema does not match expected schema\n" +
- "Actual schema: %s. Expected schema : %s",
- loader.getSchema(), expectedSchema));
- }
- }
-
- // TODO: Clean: DRILL-2933: That load(...) no longer throws
- // SchemaChangeException, so check/clean throws clause above.
- if (schema == null) {
- schema = loader.getSchema();
- for (MaterializedField mf : schema) {
- combinedVectors.put(SchemaPath.getSimplePath(mf.getName()).toExpr(), new ArrayList<>());
- }
- } else {
- // TODO - actually handle schema changes, this is just to get access to the SelectionVectorMode
- // of the current batch, the check for a null schema is used to only mutate the schema once
- // need to add new vectors and null fill for previous batches? distinction between null and non-existence important?
- schema = loader.getSchema();
- }
- logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
- totalRecords += loader.getRecordCount();
- for (VectorWrapper<?> w : loader) {
- String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr();
- ValueVector[] vectors;
- if (w.isHyper()) {
- vectors = w.getValueVectors();
- } else {
- vectors = new ValueVector[] {w.getValueVector()};
- }
- SelectionVector2 sv2 = null;
- SelectionVector4 sv4 = null;
- switch(schema.getSelectionVectorMode()) {
- case TWO_BYTE:
- sv2 = loader.getSelectionVector2();
- break;
- case FOUR_BYTE:
- sv4 = loader.getSelectionVector4();
- break;
- }
- if (sv4 != null) {
- for (int j = 0; j < sv4.getCount(); j++) {
- int complexIndex = sv4.get(j);
- int batchIndex = complexIndex >> 16;
- int recordIndexInBatch = complexIndex & 65535;
- Object obj = vectors[batchIndex].getAccessor().getObject(recordIndexInBatch);
- if (obj != null) {
- if (obj instanceof Text) {
- obj = obj.toString();
- }
- }
- combinedVectors.get(field).add(obj);
- }
- }
- else {
- for (ValueVector vv : vectors) {
- for (int j = 0; j < loader.getRecordCount(); j++) {
- int index;
- if (sv2 != null) {
- index = sv2.getIndex(j);
- } else {
- index = j;
- }
- Object obj = vv.getAccessor().getObject(index);
- if (obj != null) {
- if (obj instanceof Text) {
- obj = obj.toString();
- }
- }
- combinedVectors.get(field).add(obj);
- }
- }
- }
- }
- }
- return numBatch;
- }
-
- protected void compareSchemaOnly() throws Exception {
- RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
- List<QueryDataBatch> actual = null;
- QueryDataBatch batch = null;
- try {
- test(testOptionSettingQueries);
- actual = testRunAndReturn(queryType, query);
- batch = actual.get(0);
- loader.load(batch.getHeader().getDef(), batch.getData());
-
- final BatchSchema schema = loader.getSchema();
- final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = testBuilder.getExpectedSchema();
- if (schema.getFieldCount() != expectedSchema.size()) {
- throw new Exception("Expected and actual numbers of columns do not match.");
- }
-
- for (int i = 0; i < schema.getFieldCount(); ++i) {
- final String actualSchemaPath = schema.getColumn(i).getName();
- final TypeProtos.MajorType actualMajorType = schema.getColumn(i).getType();
-
- final String expectedSchemaPath = expectedSchema.get(i).getLeft().getRootSegmentPath();
- final TypeProtos.MajorType expectedMajorType = expectedSchema.get(i).getValue();
-
- if (!actualSchemaPath.equals(expectedSchemaPath)
- || !actualMajorType.equals(expectedMajorType)) {
- throw new Exception(String.format("Schema path or type mismatch for column #%d:\n" +
- "Expected schema path: %s\nActual schema path: %s\nExpected type: %s\nActual type: %s",
- i, expectedSchemaPath, actualSchemaPath, Types.toString(expectedMajorType),
- Types.toString(actualMajorType)));
- }
- }
-
- } finally {
- if (actual != null) {
- for (QueryDataBatch b : actual) {
- b.release();
- }
- }
- loader.clear();
- }
- }
-
- /**
- * Use this method only if necessary to validate one query against another. If you are just validating against a
- * baseline file use one of the simpler interfaces that will write the validation query for you.
- *
- * @throws Exception
- */
- protected void compareUnorderedResults() throws Exception {
- RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
-
- List<QueryDataBatch> actual = Collections.emptyList();
- List<QueryDataBatch> expected = Collections.emptyList();
- List<Map<String, Object>> expectedRecords = new ArrayList<>();
- List<Map<String, Object>> actualRecords = new ArrayList<>();
-
- try {
- test(testOptionSettingQueries);
- actual = testRunAndReturn(queryType, query);
-
- checkNumBatches(actual);
-
- addTypeInfoIfMissing(actual.get(0), testBuilder);
- addToMaterializedResults(actualRecords, actual, loader);
-
- // If baseline data was not provided to the test builder directly, we must run a query for the baseline, this includes
- // the cases where the baseline is stored in a file.
- if (baselineRecords == null) {
- test(baselineOptionSettingQueries);
- expected = testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery());
- addToMaterializedResults(expectedRecords, expected, loader);
- } else {
- expectedRecords = baselineRecords;
- }
-
- compareResults(expectedRecords, actualRecords);
- } finally {
- cleanupBatches(actual, expected);
- }
- }
-
- /**
- * Use this method only if necessary to validate one query against another. If you are just validating against a
- * baseline file use one of the simpler interfaces that will write the validation query for you.
- *
- * @throws Exception
- */
- protected void compareOrderedResults() throws Exception {
- if (highPerformanceComparison) {
- if (baselineQueryType == null) {
- throw new Exception("Cannot do a high performance comparison without using a baseline file");
- }
- compareResultsHyperVector();
- } else {
- compareMergedOnHeapVectors();
- }
- }
-
- public void compareMergedOnHeapVectors() throws Exception {
- RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
-
- List<QueryDataBatch> actual = Collections.emptyList();
- List<QueryDataBatch> expected = Collections.emptyList();
- Map<String, List<Object>> actualSuperVectors;
- Map<String, List<Object>> expectedSuperVectors;
-
- try {
- test(testOptionSettingQueries);
- actual = testRunAndReturn(queryType, query);
-
- checkNumBatches(actual);
-
- // To avoid extra work for test writers, types can optionally be inferred from the test query
- addTypeInfoIfMissing(actual.get(0), testBuilder);
-
- BatchIterator batchIter = new BatchIterator(actual, loader);
- actualSuperVectors = addToCombinedVectorResults(batchIter);
- batchIter.close();
-
- // If baseline data was not provided to the test builder directly, we must run a query for the baseline, this includes
- // the cases where the baseline is stored in a file.
- if (baselineRecords == null) {
- test(baselineOptionSettingQueries);
- expected = testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery());
- BatchIterator exBatchIter = new BatchIterator(expected, loader);
- expectedSuperVectors = addToCombinedVectorResults(exBatchIter);
- exBatchIter.close();
- } else {
- // data is built in the TestBuilder in a row major format as it is provided by the user
- // translate it here to vectorized, the representation expected by the ordered comparison
- expectedSuperVectors = translateRecordListToHeapVectors(baselineRecords);
- }
-
- compareMergedVectors(expectedSuperVectors, actualSuperVectors);
- } catch (Exception e) {
- throw new Exception(e.getMessage() + "\nFor query: " + query , e);
- } finally {
- cleanupBatches(expected, actual);
- }
- }
-
- public static Map<String, List<Object>> translateRecordListToHeapVectors(List<Map<String, Object>> records) {
- Map<String, List<Object>> ret = new TreeMap<>();
- for (String s : records.get(0).keySet()) {
- ret.put(s, new ArrayList<>());
- }
- for (Map<String, Object> m : records) {
- for (String s : m.keySet()) {
- ret.get(s).add(m.get(s));
- }
- }
- return ret;
- }
-
- public void compareResultsHyperVector() throws Exception {
- RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
-
- test(testOptionSettingQueries);
- List<QueryDataBatch> results = testRunAndReturn(queryType, query);
-
- checkNumBatches(results);
-
- // To avoid extra work for test writers, types can optionally be inferred from the test query
- addTypeInfoIfMissing(results.get(0), testBuilder);
-
- Map<String, HyperVectorValueIterator> actualSuperVectors = addToHyperVectorMap(results, loader);
-
- test(baselineOptionSettingQueries);
- List<QueryDataBatch> expected = testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery());
-
- Map<String, HyperVectorValueIterator> expectedSuperVectors = addToHyperVectorMap(expected, loader);
-
- compareHyperVectors(expectedSuperVectors, actualSuperVectors);
- cleanupBatches(results, expected);
- }
-
- private void checkNumBatches(final List<QueryDataBatch> results) {
- if (expectedNumBatches != EXPECTED_BATCH_COUNT_NOT_SET) {
- final int actualNumBatches = results.size();
- assertEquals(String.format("Expected %d batches but query returned %d non empty batch(es)%n", expectedNumBatches,
- actualNumBatches), expectedNumBatches, actualNumBatches);
- }
- }
-
- private void addTypeInfoIfMissing(QueryDataBatch batch, TestBuilder testBuilder) {
- if (! testBuilder.typeInfoSet()) {
- Map<SchemaPath, TypeProtos.MajorType> typeMap = getTypeMapFromBatch(batch);
- testBuilder.baselineTypes(typeMap);
- }
-
- }
-
- private Map<SchemaPath, TypeProtos.MajorType> getTypeMapFromBatch(QueryDataBatch batch) {
- Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
- for (int i = 0; i < batch.getHeader().getDef().getFieldCount(); i++) {
- typeMap.put(SchemaPath.getSimplePath(MaterializedField.create(batch.getHeader().getDef().getField(i)).getName()),
- batch.getHeader().getDef().getField(i).getMajorType());
- }
- return typeMap;
- }
-
- @SafeVarargs
- private final void cleanupBatches(List<QueryDataBatch>... results) {
- for (List<QueryDataBatch> resultList : results ) {
- for (QueryDataBatch result : resultList) {
- result.release();
- }
- }
- }
-
- public static void addToMaterializedResults(List<Map<String, Object>> materializedRecords,
- List<QueryDataBatch> records,
- RecordBatchLoader loader)
- throws SchemaChangeException, UnsupportedEncodingException {
- long totalRecords = 0;
- QueryDataBatch batch;
- int size = records.size();
- for (int i = 0; i < size; i++) {
- batch = records.get(0);
- loader.load(batch.getHeader().getDef(), batch.getData());
- // TODO: Clean: DRILL-2933: That load(...) no longer throws
- // SchemaChangeException, so check/clean throws clause above.
- logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
- totalRecords += loader.getRecordCount();
- for (int j = 0; j < loader.getRecordCount(); j++) {
- Map<String, Object> record = new TreeMap<>();
- for (VectorWrapper<?> w : loader) {
- Object obj = w.getValueVector().getAccessor().getObject(j);
- if (obj != null) {
- if (obj instanceof Text) {
- obj = obj.toString();
- }
- record.put(SchemaPath.getSimplePath(w.getField().getName()).toExpr(), obj);
- }
- record.put(SchemaPath.getSimplePath(w.getField().getName()).toExpr(), obj);
- }
- materializedRecords.add(record);
- }
- records.remove(0);
- batch.release();
- loader.clear();
- }
- }
-
- public static boolean compareValuesErrorOnMismatch(Object expected, Object actual, int counter, String column) throws Exception {
-
- if (compareValues(expected, actual, counter, column)) {
- return true;
- }
- if (expected == null) {
- throw new Exception("at position " + counter + " column '" + column + "' mismatched values, expected: null " +
- "but received " + actual + "(" + actual.getClass().getSimpleName() + ")");
- }
- if (actual == null) {
- throw new Exception("unexpected null at position " + counter + " column '" + column + "' should have been: " + expected);
- }
- if (actual instanceof byte[]) {
- throw new Exception("at position " + counter + " column '" + column + "' mismatched values, expected: "
- + new String((byte[])expected, "UTF-8") + " but received " + new String((byte[])actual, "UTF-8"));
- }
- if (!expected.equals(actual)) {
- throw new Exception("at position " + counter + " column '" + column + "' mismatched values, expected: "
- + expected + "(" + expected.getClass().getSimpleName() + ") but received " + actual + "(" + actual.getClass().getSimpleName() + ")");
- }
- return true;
- }
-
- public static boolean compareValues(Object expected, Object actual, int counter, String column) throws Exception {
- if (expected == null) {
- if (actual == null) {
- if (VERBOSE_DEBUG) {
- logger.debug("(1) at position " + counter + " column '" + column + "' matched value: " + expected );
- }
- return true;
- } else {
- return false;
- }
- }
- if (actual == null) {
- return false;
- }
- if (actual instanceof byte[]) {
- if ( ! Arrays.equals((byte[]) expected, (byte[]) actual)) {
- return false;
- } else {
- if (VERBOSE_DEBUG) {
- logger.debug("at position " + counter + " column '" + column + "' matched value " + new String((byte[])expected, "UTF-8"));
- }
- return true;
- }
- }
- if (!expected.equals(actual)) {
- return false;
- } else {
- if (VERBOSE_DEBUG) {
- logger.debug("at position " + counter + " column '" + column + "' matched value: " + expected );
- }
- }
- return true;
- }
-
- /**
- * Compare two result sets, ignoring ordering.
- *
- * @param expectedRecords - list of records from baseline
- * @param actualRecords - list of records from test query, WARNING - this list is destroyed in this method
- * @throws Exception
- */
- private void compareResults(List<Map<String, Object>> expectedRecords, List<Map<String, Object>> actualRecords) throws Exception {
-
- assertEquals("Different number of records returned", expectedRecords.size(), actualRecords.size());
-
- int i = 0;
- int counter = 0;
- boolean found;
- for (Map<String, Object> expectedRecord : expectedRecords) {
- i = 0;
- found = false;
- findMatch:
- for (Map<String, Object> actualRecord : actualRecords) {
- for (String s : actualRecord.keySet()) {
- if (!expectedRecord.containsKey(s)) {
- throw new Exception("Unexpected column '" + s + "' returned by query.");
- }
- if ( ! compareValues(expectedRecord.get(s), actualRecord.get(s), counter, s)) {
- i++;
- continue findMatch;
- }
- }
- if (actualRecord.size() < expectedRecord.size()) {
- throw new Exception(findMissingColumns(expectedRecord.keySet(), actualRecord.keySet()));
- }
- found = true;
- break;
- }
- if (!found) {
- StringBuilder sb = new StringBuilder();
- for (int expectedRecordDisplayCount = 0;
- expectedRecordDisplayCount < 10 && expectedRecordDisplayCount < expectedRecords.size();
- expectedRecordDisplayCount++) {
- sb.append(printRecord(expectedRecords.get(expectedRecordDisplayCount)));
- }
- String expectedRecordExamples = sb.toString();
- sb.setLength(0);
- for (int actualRecordDisplayCount = 0;
- actualRecordDisplayCount < 10 && actualRecordDisplayCount < actualRecords.size();
- actualRecordDisplayCount++) {
- sb.append(printRecord(actualRecords.get(actualRecordDisplayCount)));
- }
- String actualRecordExamples = sb.toString();
- throw new Exception(String.format("After matching %d records, did not find expected record in result set:\n %s\n\n" +
- "Some examples of expected records:\n%s\n\n Some examples of records returned by the test query:\n%s",
- counter, printRecord(expectedRecord), expectedRecordExamples, actualRecordExamples));
- } else {
- actualRecords.remove(i);
- counter++;
- }
- }
- assertEquals(0, actualRecords.size());
- }
-
- private static String findMissingColumns(Set<String> expected, Set<String> actual) {
- String missingCols = "";
- for (String colName : expected) {
- if (!actual.contains(colName)) {
- missingCols += colName + ", ";
- }
- }
- return "Expected column(s) " + missingCols + " not found in result set: " + actual + ".";
- }
-
- private String printRecord(Map<String, ?> record) {
- String ret = "";
- for (String s : record.keySet()) {
- ret += s + " : " + record.get(s) + ", ";
- }
- return ret + "\n";
- }
-
- private void test(String query) throws Exception {
- services.test(query);
- }
-
- private List<QueryDataBatch> testRunAndReturn(QueryType type, Object query) throws Exception {
- return services.testRunAndReturn(type, query);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
index eb11532..091d567 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
@@ -36,6 +36,8 @@ import org.apache.calcite.sql.SqlExplain.Depth;
import org.apache.calcite.sql.SqlExplainLevel;
import com.google.common.base.Strings;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.QueryTestUtil;
public class PlanTestBase extends BaseTestQuery {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
index b00d4d6..ca643f5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
@@ -29,7 +29,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.LogicalPlanPersistence;
import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.common.scanner.persistence.ScanResult;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.memory.BufferAllocator;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
deleted file mode 100644
index 26a0537..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill;
-
-import java.util.List;
-import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.drill.BaseTestQuery.SilentListener;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.TestTools;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.client.PrintingResultsListener;
-import org.apache.drill.exec.client.QuerySubmitter.Format;
-import org.apache.drill.exec.compile.ClassTransformer;
-import org.apache.drill.exec.exception.OutOfMemoryException;
-import org.apache.drill.exec.proto.UserBitShared.QueryType;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.AwaitableUserResultsListener;
-import org.apache.drill.exec.rpc.user.QueryDataBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.server.options.OptionManager;
-import org.apache.drill.exec.server.options.OptionValue;
-import org.apache.drill.exec.server.options.SystemOptionManager;
-import org.apache.drill.exec.util.VectorUtil;
-
-/**
- * Utilities useful for tests that issue SQL queries.
- */
-public class QueryTestUtil {
-
- public static final String TEST_QUERY_PRINTING_SILENT = "drill.test.query.printing.silent";
-
- /**
- * Constructor. All methods are static.
- */
- private QueryTestUtil() {
- }
-
- /**
- * Create a DrillClient that can be used to query a drill cluster.
- *
- * @param drillConfig
- * @param remoteServiceSet remote service set
- * @param maxWidth maximum width per node
- * @param props Connection properties contains properties such as "user", "password", "schema" etc
- * @return the newly created client
- * @throws RpcException if there is a problem setting up the client
- */
- public static DrillClient createClient(final DrillConfig drillConfig, final RemoteServiceSet remoteServiceSet,
- final int maxWidth, final Properties props) throws RpcException, OutOfMemoryException {
- final DrillClient drillClient = new DrillClient(drillConfig, remoteServiceSet.getCoordinator());
- drillClient.connect(props);
-
- final List<QueryDataBatch> results = drillClient.runQuery(
- QueryType.SQL, String.format("alter session set `%s` = %d",
- ExecConstants.MAX_WIDTH_PER_NODE_KEY, maxWidth));
- for (QueryDataBatch queryDataBatch : results) {
- queryDataBatch.release();
- }
-
- return drillClient;
- }
-
- /**
- * Normalize the query relative to the test environment.
- *
- * <p>Looks for "${WORKING_PATH}" in the query string, and replaces it the current
- * working patch obtained from {@link org.apache.drill.common.util.TestTools#getWorkingPath()}.
- *
- * @param query the query string
- * @return the normalized query string
- */
- public static String normalizeQuery(final String query) {
- if (query.contains("${WORKING_PATH}")) {
- return query.replaceAll(Pattern.quote("${WORKING_PATH}"), Matcher.quoteReplacement(TestTools.getWorkingPath()));
- } else if (query.contains("[WORKING_PATH]")) {
- return query.replaceAll(Pattern.quote("[WORKING_PATH]"), Matcher.quoteReplacement(TestTools.getWorkingPath()));
- }
- return query;
- }
-
- /**
- * Execute a SQL query, and print the results.
- *
- * @param drillClient drill client to use
- * @param type type of the query
- * @param queryString query string
- * @return number of rows returned
- * @throws Exception
- */
- public static int testRunAndPrint(
- final DrillClient drillClient, final QueryType type, final String queryString) throws Exception {
- final String query = normalizeQuery(queryString);
- DrillConfig config = drillClient.getConfig();
- AwaitableUserResultsListener resultListener =
- new AwaitableUserResultsListener(
- config.getBoolean(TEST_QUERY_PRINTING_SILENT) ?
- new SilentListener() :
- new PrintingResultsListener(config, Format.TSV, VectorUtil.DEFAULT_COLUMN_WIDTH)
- );
- drillClient.runQuery(type, query, resultListener);
- return resultListener.await();
- }
-
- /**
- * Execute one or more queries separated by semicolons, and print the results.
- *
- * @param drillClient drill client to use
- * @param queryString the query string
- * @throws Exception
- */
- public static void test(final DrillClient drillClient, final String queryString) throws Exception{
- final String query = normalizeQuery(queryString);
- String[] queries = query.split(";");
- for (String q : queries) {
- final String trimmedQuery = q.trim();
- if (trimmedQuery.isEmpty()) {
- continue;
- }
- testRunAndPrint(drillClient, QueryType.SQL, trimmedQuery);
- }
- }
-
- /**
- * Execute one or more queries separated by semicolons, and print the results, with the option to
- * add formatted arguments to the query string.
- *
- * @param drillClient drill client to use
- * @param query the query string; may contain formatting specifications to be used by
- * {@link String#format(String, Object...)}.
- * @param args optional args to use in the formatting call for the query string
- * @throws Exception
- */
- public static void test(final DrillClient drillClient, final String query, Object... args) throws Exception {
- test(drillClient, String.format(query, args));
- }
-
- /**
- * Execute a single query with a user supplied result listener.
- *
- * @param drillClient drill client to use
- * @param type type of query
- * @param queryString the query string
- * @param resultListener the result listener
- */
- public static void testWithListener(final DrillClient drillClient, final QueryType type,
- final String queryString, final UserResultsListener resultListener) {
- final String query = QueryTestUtil.normalizeQuery(queryString);
- drillClient.runQuery(type, query, resultListener);
- }
-
- /**
- * Set up the options to test the scalar replacement retry option (see
- * ClassTransformer.java). Scalar replacement rewrites bytecode to replace
- * value holders (essentially boxed values) with their member variables as
- * locals. There is still one pattern that doesn't work, and occasionally new
- * ones are introduced. This can be used in tests that exercise failing patterns.
- *
- * <p>This also flushes the compiled code cache.
- *
- * @param drillbit the drillbit
- * @param srOption the scalar replacement option value to use
- * @return the original scalar replacement option setting (so it can be restored)
- */
- @SuppressWarnings("resource")
- public static OptionValue setupScalarReplacementOption(
- final Drillbit drillbit, final ClassTransformer.ScalarReplacementOption srOption) {
- // set the system option
- final DrillbitContext drillbitContext = drillbit.getContext();
- final SystemOptionManager optionManager = drillbitContext.getOptionManager();
- final OptionValue originalOptionValue = optionManager.getOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION);
- optionManager.setLocalOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase());
-
- // flush the code cache
- drillbitContext.getCompiler().flushCache();
-
- return originalOptionValue;
- }
-
- /**
- * Restore the original scalar replacement option returned from
- * setupScalarReplacementOption().
- *
- * <p>This also flushes the compiled code cache.
- *
- * @param drillbit the drillbit
- * @param srOption the scalar replacement option value to use
- */
- public static void restoreScalarReplacementOption(final Drillbit drillbit, final String srOption) {
- @SuppressWarnings("resource")
- final DrillbitContext drillbitContext = drillbit.getContext();
- @SuppressWarnings("resource")
- final OptionManager optionManager = drillbitContext.getOptionManager();
- optionManager.setLocalOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption);
-
- // flush the code cache
- drillbitContext.getCompiler().flushCache();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java b/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java
index 4085fb4..95e011b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestAggNullable.java
@@ -20,30 +20,25 @@ package org.apache.drill;
import static org.junit.Assert.assertEquals;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(OperatorTest.class)
-public class TestAggNullable extends BaseTestQuery{
+public class TestAggNullable extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAggNullable.class);
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
-
private static void enableAggr(boolean ha, boolean sa) throws Exception {
- test(String.format("alter session set `planner.enable_hashagg` = %s", ha ? "true":"false"));
- test(String.format("alter session set `planner.enable_streamagg` = %s", sa ? "true":"false"));
+ test("alter session set `planner.enable_hashagg` = %s", ha);
+ test("alter session set `planner.enable_streamagg` = %s", sa);
test("alter session set `planner.slice_target` = 1");
}
@Test // HashAgg on nullable columns
public void testHashAggNullableColumns() throws Exception {
- String query1 = String.format("select t2.b2 from dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " group by t2.b2", TEST_RES_PATH);
- String query2 = String.format("select t2.a2, t2.b2 from dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " group by t2.a2, t2.b2", TEST_RES_PATH);
+ String query1 = "select t2.b2 from cp.`jsoninput/nullable2.json` t2 group by t2.b2";
+ String query2 = "select t2.a2, t2.b2 from cp.`jsoninput/nullable2.json` t2 group by t2.a2, t2.b2";
int actualRecordCount;
int expectedRecordCount = 2;
@@ -61,10 +56,8 @@ public class TestAggNullable extends BaseTestQuery{
@Test // StreamingAgg on nullable columns
public void testStreamAggNullableColumns() throws Exception {
- String query1 = String.format("select t2.b2 from dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " group by t2.b2", TEST_RES_PATH);
- String query2 = String.format("select t2.a2, t2.b2 from dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " group by t2.a2, t2.b2", TEST_RES_PATH);
+ String query1 = "select t2.b2 from cp.`jsoninput/nullable2.json` t2 group by t2.b2";
+ String query2 = "select t2.a2, t2.b2 from cp.`jsoninput/nullable2.json` t2 group by t2.a2, t2.b2";
int actualRecordCount;
int expectedRecordCount = 2;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java
index cc32dbe..3631ccf 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestAltSortQueries.java
@@ -19,24 +19,33 @@ package org.apache.drill;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.categories.SqlTest;
+import org.apache.drill.test.BaseTestQuery;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
@Category({SqlTest.class, OperatorTest.class})
-public class TestAltSortQueries extends BaseTestQuery{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAltSortQueries.class);
+public class TestAltSortQueries extends BaseTestQuery {
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data", "region.parquet"));
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data", "regionsSF"));
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data", "nation.parquet"));
+ }
@Test
public void testOrderBy() throws Exception{
test("select R_REGIONKEY " +
- "from dfs_test.`[WORKING_PATH]/../../sample-data/region.parquet` " +
+ "from dfs.`sample-data/region.parquet` " +
"order by R_REGIONKEY");
}
@Test
public void testOrderBySingleFile() throws Exception{
test("select R_REGIONKEY " +
- "from dfs_test.`[WORKING_PATH]/../../sample-data/regionsSF/` " +
+ "from dfs.`sample-data/regionsSF/` " +
"order by R_REGIONKEY");
}
@@ -64,12 +73,11 @@ public class TestAltSortQueries extends BaseTestQuery{
" nations.N_NAME,\n" +
" regions.R_NAME\n" +
"FROM\n" +
- " dfs_test.`[WORKING_PATH]/../../sample-data/nation.parquet` nations\n" +
+ " dfs.`sample-data/nation.parquet` nations\n" +
"JOIN\n" +
- " dfs_test.`[WORKING_PATH]/../../sample-data/region.parquet` regions\n" +
+ " dfs.`sample-data/region.parquet` regions\n" +
" on nations.N_REGIONKEY = regions.R_REGIONKEY" +
" order by regions.R_NAME, nations.N_NAME " +
" limit 5");
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
index 8b608c6..e88e5a4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
@@ -20,12 +20,14 @@ package org.apache.drill;
import com.google.common.collect.ImmutableList;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.test.BaseTestQuery;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -33,8 +35,11 @@ import java.util.Map;
@Category(UnlikelyTest.class)
public class TestBugFixes extends BaseTestQuery {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBugFixes.class);
- private static final String WORKING_PATH = TestTools.getWorkingPath();
- private static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
+
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("bugs", "DRILL-4192"));
+ }
@Test
public void leak1() throws Exception {
@@ -157,19 +162,16 @@ public class TestBugFixes extends BaseTestQuery {
@Test
public void testDRILL4192() throws Exception {
- String query = (String.format("select dir0, dir1 from dfs_test.`%s/bugs/DRILL-4192` order by dir1", TEST_RES_PATH));
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select dir0, dir1 from dfs.`bugs/DRILL-4192` order by dir1")
.unOrdered()
.baselineColumns("dir0", "dir1")
.baselineValues("single_top_partition", "nested_partition_1")
.baselineValues("single_top_partition", "nested_partition_2")
.go();
- query = (String.format("select dir0, dir1 from dfs_test.`%s/bugs/DRILL-4192/*/nested_partition_1` order by dir1", TEST_RES_PATH));
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select dir0, dir1 from dfs.`bugs/DRILL-4192/*/nested_partition_1` order by dir1")
.unOrdered()
.baselineColumns("dir0", "dir1")
.baselineValues("single_top_partition", "nested_partition_1")
@@ -219,11 +221,10 @@ public class TestBugFixes extends BaseTestQuery {
List<Map<String, Object>> baseline = baselineBuilder.build();
testBuilder()
- .sqlQuery(String.format("select cast(id as int) as id from dfs_test.`%s/bugs/DRILL-4884/limit_test_parquet/test0_0_0.parquet` group by id order by 1 limit %s",
- TEST_RES_PATH, limit))
- .unOrdered()
- .baselineRecords(baseline)
- .go();
+ .sqlQuery("select cast(id as int) as id from cp.`bugs/DRILL-4884/limit_test_parquet/test0_0_0.parquet` group by id order by 1 limit %s", limit)
+ .unOrdered()
+ .baselineRecords(baseline)
+ .go();
}
@Test
[17/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java b/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java
index 0ff7a12..6f9f088 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java
@@ -17,25 +17,19 @@
*/
package org.apache.drill;
-
import static org.junit.Assert.assertEquals;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.categories.UnlikelyTest;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.common.util.TestTools;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
@Category({SqlTest.class, PlannerTest.class})
public class TestPartitionFilter extends PlanTestBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestPartitionFilter.class);
-
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
-
private static void testExcludeFilter(String query, int expectedNumFiles,
String excludedFilterPattern, int expectedRowCount) throws Exception {
int actualRowCount = testSql(query);
@@ -54,291 +48,289 @@ public class TestPartitionFilter extends PlanTestBase {
@BeforeClass
public static void createParquetTable() throws Exception {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel"));
+
test("alter session set `planner.disable_exchanges` = true");
- test(String.format("create table dfs_test.tmp.parquet partition by (yr, qrtr) as select o_orderkey, o_custkey, " +
+ test("create table dfs.tmp.parquet partition by (yr, qrtr) as select o_orderkey, o_custkey, " +
"o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, cast(dir0 as int) yr, dir1 qrtr " +
- "from dfs_test.`%s/multilevel/parquet`", TEST_RES_PATH));
+ "from dfs.`multilevel/parquet`");
test("alter session set `planner.disable_exchanges` = false");
}
@Test //Parquet: basic test with dir0 and dir1 filters
public void testPartitionFilter1_Parquet() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/parquet` where dir0=1994 and dir1='Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 10);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/parquet` where dir0=1994 and dir1='Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 10);
}
@Test //Parquet: basic test with dir0 and dir1 filters
public void testPartitionFilter1_Parquet_from_CTAS() throws Exception {
- String query = String.format("select yr, qrtr, o_custkey, o_orderdate from dfs_test.tmp.parquet where yr=1994 and qrtr='Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 10);
+ String query = "select yr, qrtr, o_custkey, o_orderdate from dfs.tmp.parquet where yr=1994 and qrtr='Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 10);
}
@Test //Json: basic test with dir0 and dir1 filters
public void testPartitionFilter1_Json() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/json` where dir0=1994 and dir1='Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 10);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/json` where dir0=1994 and dir1='Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 10);
}
@Test //Json: basic test with dir0 and dir1 filters
public void testPartitionFilter1_JsonFileMixDir() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/jsonFileMixDir` where dir0=1995 and dir1='Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 10);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/jsonFileMixDir` where dir0=1995 and dir1='Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 10);
}
@Test //Json: basic test with dir0 = and dir1 is null filters
public void testPartitionFilterIsNull_JsonFileMixDir() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/jsonFileMixDir` where dir0=1995 and dir1 is null", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 5);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/jsonFileMixDir` where dir0=1995 and dir1 is null";
+ testExcludeFilter(query, 1, "Filter\\(", 5);
}
@Test //Json: basic test with dir0 = and dir1 is not null filters
public void testPartitionFilterIsNotNull_JsonFileMixDir() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/jsonFileMixDir` where dir0=1995 and dir1 is not null", TEST_RES_PATH);
- testExcludeFilter(query, 4, "Filter", 40);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/jsonFileMixDir` where dir0=1995 and dir1 is not null";
+ testExcludeFilter(query, 4, "Filter\\(", 40);
}
@Test //CSV: basic test with dir0 and dir1 filters in
public void testPartitionFilter1_Csv() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/csv` where dir0=1994 and dir1='Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 10);
+ String query = "select * from dfs.`multilevel/csv` where dir0=1994 and dir1='Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 10);
}
@Test //Parquet: partition filters are combined with regular columns in an AND
public void testPartitionFilter2_Parquet() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/parquet` where o_custkey < 1000 and dir0=1994 and dir1='Q1'", TEST_RES_PATH);
- testIncludeFilter(query, 1, "Filter", 5);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/parquet` where o_custkey < 1000 and dir0=1994 and dir1='Q1'";
+ testIncludeFilter(query, 1, "Filter\\(", 5);
}
@Test //Parquet: partition filters are combined with regular columns in an AND
public void testPartitionFilter2_Parquet_from_CTAS() throws Exception {
- String query = String.format("select yr, qrtr, o_custkey, o_orderdate from dfs_test.tmp.parquet where o_custkey < 1000 and yr=1994 and qrtr='Q1'", TEST_RES_PATH);
- testIncludeFilter(query, 1, "Filter", 5);
+ String query = "select yr, qrtr, o_custkey, o_orderdate from dfs.tmp.parquet where o_custkey < 1000 and yr=1994 and qrtr='Q1'";
+ testIncludeFilter(query, 1, "Filter\\(", 5);
}
@Test //Json: partition filters are combined with regular columns in an AND
public void testPartitionFilter2_Json() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/json` where o_custkey < 1000 and dir0=1994 and dir1='Q1'", TEST_RES_PATH);
- testIncludeFilter(query, 1, "Filter", 5);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/json` where o_custkey < 1000 and dir0=1994 and dir1='Q1'";
+ testIncludeFilter(query, 1, "Filter\\(", 5);
}
@Test //CSV: partition filters are combined with regular columns in an AND
public void testPartitionFilter2_Csv() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/csv` where columns[1] < 1000 and dir0=1994 and dir1='Q1'", TEST_RES_PATH);
- testIncludeFilter(query, 1, "Filter", 5);
+ String query = "select * from dfs.`multilevel/csv` where columns[1] < 1000 and dir0=1994 and dir1='Q1'";
+ testIncludeFilter(query, 1, "Filter\\(", 5);
}
@Test //Parquet: partition filters are ANDed and belong to a top-level OR
public void testPartitionFilter3_Parquet() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/parquet` where (dir0=1994 and dir1='Q1' and o_custkey < 500) or (dir0=1995 and dir1='Q2' and o_custkey > 500)", TEST_RES_PATH);
- testIncludeFilter(query, 2, "Filter", 8);
+ String query = "select * from dfs.`multilevel/parquet` where (dir0=1994 and dir1='Q1' and o_custkey < 500) or (dir0=1995 and dir1='Q2' and o_custkey > 500)";
+ testIncludeFilter(query, 2, "Filter\\(", 8);
}
@Test //Parquet: partition filters are ANDed and belong to a top-level OR
public void testPartitionFilter3_Parquet_from_CTAS() throws Exception {
- String query = String.format("select * from dfs_test.tmp.parquet where (yr=1994 and qrtr='Q1' and o_custkey < 500) or (yr=1995 and qrtr='Q2' and o_custkey > 500)", TEST_RES_PATH);
- testIncludeFilter(query, 2, "Filter", 8);
+ String query = "select * from dfs.tmp.parquet where (yr=1994 and qrtr='Q1' and o_custkey < 500) or (yr=1995 and qrtr='Q2' and o_custkey > 500)";
+ testIncludeFilter(query, 2, "Filter\\(", 8);
}
@Test //Json: partition filters are ANDed and belong to a top-level OR
public void testPartitionFilter3_Json() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/json` where (dir0=1994 and dir1='Q1' and o_custkey < 500) or (dir0=1995 and dir1='Q2' and o_custkey > 500)", TEST_RES_PATH);
- testIncludeFilter(query, 2, "Filter", 8);
+ String query = "select * from dfs.`multilevel/json` where (dir0=1994 and dir1='Q1' and o_custkey < 500) or (dir0=1995 and dir1='Q2' and o_custkey > 500)";
+ testIncludeFilter(query, 2, "Filter\\(", 8);
}
@Test //CSV: partition filters are ANDed and belong to a top-level OR
public void testPartitionFilter3_Csv() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/csv` where (dir0=1994 and dir1='Q1' and columns[1] < 500) or (dir0=1995 and dir1='Q2' and columns[1] > 500)", TEST_RES_PATH);
- testIncludeFilter(query, 2, "Filter", 8);
+ String query = "select * from dfs.`multilevel/csv` where (dir0=1994 and dir1='Q1' and columns[1] < 500) or (dir0=1995 and dir1='Q2' and columns[1] > 500)";
+ testIncludeFilter(query, 2, "Filter\\(", 8);
}
@Test //Parquet: filters contain join conditions and partition filters
public void testPartitionFilter4_Parquet() throws Exception {
- String query1 = String.format("select t1.dir0, t1.dir1, t1.o_custkey, t1.o_orderdate, cast(t2.c_name as varchar(10)) from dfs_test.`%s/multilevel/parquet` t1, cp.`tpch/customer.parquet` t2 where t1.o_custkey = t2.c_custkey and t1.dir0=1994 and t1.dir1='Q1'", TEST_RES_PATH);
+ String query1 = "select t1.dir0, t1.dir1, t1.o_custkey, t1.o_orderdate, cast(t2.c_name as varchar(10)) from dfs.`multilevel/parquet` t1, cp.`tpch/customer.parquet` t2 where" +
+ " t1.o_custkey = t2.c_custkey and t1.dir0=1994 and t1.dir1='Q1'";
test(query1);
}
@Test //Parquet: filters contain join conditions and partition filters
public void testPartitionFilter4_Parquet_from_CTAS() throws Exception {
- String query1 = String.format("select t1.dir0, t1.dir1, t1.o_custkey, t1.o_orderdate, cast(t2.c_name as varchar(10)) from dfs_test.tmp.parquet t1, cp.`tpch/customer.parquet` t2 where t1.o_custkey = t2.c_custkey and t1.yr=1994 and t1.qrtr='Q1'", TEST_RES_PATH);
+ String query1 = "select t1.dir0, t1.dir1, t1.o_custkey, t1.o_orderdate, cast(t2.c_name as varchar(10)) from dfs.tmp.parquet t1, cp.`tpch/customer.parquet` t2 where " +
+ "t1.o_custkey = t2.c_custkey and t1.yr=1994 and t1.qrtr='Q1'";
test(query1);
}
@Test //Json: filters contain join conditions and partition filters
public void testPartitionFilter4_Json() throws Exception {
- String query1 = String.format("select t1.dir0, t1.dir1, t1.o_custkey, t1.o_orderdate, cast(t2.c_name as varchar(10)) from dfs_test.`%s/multilevel/json` t1, cp.`tpch/customer.parquet` t2 where cast(t1.o_custkey as bigint) = cast(t2.c_custkey as bigint) and t1.dir0=1994 and t1.dir1='Q1'", TEST_RES_PATH);
+ String query1 = "select t1.dir0, t1.dir1, t1.o_custkey, t1.o_orderdate, cast(t2.c_name as varchar(10)) from dfs.`multilevel/json` t1, cp.`tpch/customer.parquet` t2 where " +
+ "cast(t1.o_custkey as bigint) = cast(t2.c_custkey as bigint) and t1.dir0=1994 and t1.dir1='Q1'";
test(query1);
}
@Test //CSV: filters contain join conditions and partition filters
public void testPartitionFilter4_Csv() throws Exception {
- String query1 = String.format("select t1.dir0, t1.dir1, t1.columns[1] as o_custkey, t1.columns[4] as o_orderdate, cast(t2.c_name as varchar(10)) from dfs_test.`%s/multilevel/csv` t1, cp.`tpch/customer.parquet` t2 where cast(t1.columns[1] as bigint) = cast(t2.c_custkey as bigint) and t1.dir0=1994 and t1.dir1='Q1'", TEST_RES_PATH);
+ String query1 = "select t1.dir0, t1.dir1, t1.columns[1] as o_custkey, t1.columns[4] as o_orderdate, cast(t2.c_name as varchar(10)) from dfs.`multilevel/csv` t1, cp" +
+ ".`tpch/customer.parquet` t2 where cast(t1.columns[1] as bigint) = cast(t2.c_custkey as bigint) and t1.dir0=1994 and t1.dir1='Q1'";
test(query1);
}
@Test // Parquet: IN filter
public void testPartitionFilter5_Parquet() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/parquet` where dir0 in (1995, 1996)", TEST_RES_PATH);
- testExcludeFilter(query, 8, "Filter", 80);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/parquet` where dir0 in (1995, 1996)";
+ testExcludeFilter(query, 8, "Filter\\(", 80);
}
@Test // Parquet: IN filter
public void testPartitionFilter5_Parquet_from_CTAS() throws Exception {
- String query = String.format("select yr, qrtr, o_custkey, o_orderdate from dfs_test.tmp.parquet where yr in (1995, 1996)", TEST_RES_PATH);
- testExcludeFilter(query, 8, "Filter", 80);
+ String query = "select yr, qrtr, o_custkey, o_orderdate from dfs.tmp.parquet where yr in (1995, 1996)";
+ testExcludeFilter(query, 8, "Filter\\(", 80);
}
@Test // Json: IN filter
public void testPartitionFilter5_Json() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/json` where dir0 in (1995, 1996)", TEST_RES_PATH);
- testExcludeFilter(query, 8, "Filter", 80);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/json` where dir0 in (1995, 1996)";
+ testExcludeFilter(query, 8, "Filter\\(", 80);
}
@Test // CSV: IN filter
public void testPartitionFilter5_Csv() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/csv` where dir0 in (1995, 1996)", TEST_RES_PATH);
- testExcludeFilter(query, 8, "Filter", 80);
+ String query = "select * from dfs.`multilevel/csv` where dir0 in (1995, 1996)";
+ testExcludeFilter(query, 8, "Filter\\(", 80);
}
@Test // Parquet: one side of OR has partition filter only, other side has both partition filter and non-partition filter
public void testPartitionFilter6_Parquet() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/parquet` where (dir0=1995 and o_totalprice < 40000) or dir0=1996", TEST_RES_PATH);
- testIncludeFilter(query, 8, "Filter", 46);
+ String query = "select * from dfs.`multilevel/parquet` where (dir0=1995 and o_totalprice < 40000) or dir0=1996";
+ testIncludeFilter(query, 8, "Filter\\(", 46);
}
@Test // Parquet: one side of OR has partition filter only, other side has both partition filter and non-partition filter
public void testPartitionFilter6_Parquet_from_CTAS() throws Exception {
- String query = String.format("select * from dfs_test.tmp.parquet where (yr=1995 and o_totalprice < 40000) or yr=1996", TEST_RES_PATH);
+ String query = "select * from dfs.tmp.parquet where (yr=1995 and o_totalprice < 40000) or yr=1996";
// Parquet RG filter pushdown further reduces to 6 files.
- testIncludeFilter(query, 6, "Filter", 46);
+ testIncludeFilter(query, 6, "Filter\\(", 46);
}
@Test // Parquet: trivial case with 1 partition filter
public void testPartitionFilter7_Parquet() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/parquet` where dir0=1995", TEST_RES_PATH);
- testExcludeFilter(query, 4, "Filter", 40);
+ String query = "select * from dfs.`multilevel/parquet` where dir0=1995";
+ testExcludeFilter(query, 4, "Filter\\(", 40);
}
@Test // Parquet: trivial case with 1 partition filter
public void testPartitionFilter7_Parquet_from_CTAS() throws Exception {
- String query = String.format("select * from dfs_test.tmp.parquet where yr=1995", TEST_RES_PATH);
- testExcludeFilter(query, 4, "Filter", 40);
+ String query = "select * from dfs.tmp.parquet where yr=1995";
+ testExcludeFilter(query, 4, "Filter\\(", 40);
}
@Test // Parquet: partition filter on subdirectory only
public void testPartitionFilter8_Parquet() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/parquet` where dir1 in ('Q1','Q4')", TEST_RES_PATH);
- testExcludeFilter(query, 6, "Filter", 60);
+ String query = "select * from dfs.`multilevel/parquet` where dir1 in ('Q1','Q4')";
+ testExcludeFilter(query, 6, "Filter\\(", 60);
}
@Test
public void testPartitionFilter8_Parquet_from_CTAS() throws Exception {
- String query = String.format("select * from dfs_test.tmp.parquet where qrtr in ('Q1','Q4')", TEST_RES_PATH);
- testExcludeFilter(query, 6, "Filter", 60);
+ String query = "select * from dfs.tmp.parquet where qrtr in ('Q1','Q4')";
+ testExcludeFilter(query, 6, "Filter\\(", 60);
}
@Test // Parquet: partition filter on subdirectory only plus non-partition filter
public void testPartitionFilter9_Parquet() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/parquet` where dir1 in ('Q1','Q4') and o_totalprice < 40000", TEST_RES_PATH);
+ String query = "select * from dfs.`multilevel/parquet` where dir1 in ('Q1','Q4') and o_totalprice < 40000";
// Parquet RG filter pushdown further reduces to 4 files.
- testIncludeFilter(query, 4, "Filter", 9);
+ testIncludeFilter(query, 4, "Filter\\(", 9);
}
@Test
public void testPartitionFilter9_Parquet_from_CTAS() throws Exception {
- String query = String.format("select * from dfs_test.tmp.parquet where qrtr in ('Q1','Q4') and o_totalprice < 40000", TEST_RES_PATH);
+ String query = "select * from dfs.tmp.parquet where qrtr in ('Q1','Q4') and o_totalprice < 40000";
// Parquet RG filter pushdown further reduces to 4 files.
- testIncludeFilter(query, 4, "Filter", 9);
+ testIncludeFilter(query, 4, "Filter\\(", 9);
}
@Test
public void testPartitoinFilter10_Parquet() throws Exception {
- String query = String.format("select max(o_orderprice) from dfs_test.`%s/multilevel/parquet` where dir0=1994 and dir1='Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 1);
+ String query = "select max(o_orderprice) from dfs.`multilevel/parquet` where dir0=1994 and dir1='Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 1);
}
@Test
public void testPartitoinFilter10_Parquet_from_CTAS() throws Exception {
- String query = String.format("select max(o_orderprice) from dfs_test.tmp.parquet where yr=1994 and qrtr='Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 1);
+ String query = "select max(o_orderprice) from dfs.tmp.parquet where yr=1994 and qrtr='Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 1);
}
@Test // see DRILL-2712
@Category(UnlikelyTest.class)
public void testMainQueryFalseCondition() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/parquet").toURI().toString();
- String query = String.format("select * from (select dir0, o_custkey from dfs_test.`%s` where dir0='1994') t where 1 = 0", root);
+ String query = "select * from (select dir0, o_custkey from dfs.`multilevel/parquet` where dir0='1994') t where 1 = 0";
// the 1 = 0 becomes limit 0, which will require to read only one parquet file, in stead of 4 for year '1994'.
- testExcludeFilter(query, 1, "Filter", 0);
+ testExcludeFilter(query, 1, "Filter\\(", 0);
}
@Test // see DRILL-2712
@Category(UnlikelyTest.class)
public void testMainQueryTrueCondition() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/parquet").toURI().toString();
- String query = String.format("select * from (select dir0, o_custkey from dfs_test.`%s` where dir0='1994' ) t where 0 = 0", root);
- testExcludeFilter(query, 4, "Filter", 40);
+ String query = "select * from (select dir0, o_custkey from dfs.`multilevel/parquet` where dir0='1994' ) t where 0 = 0";
+ testExcludeFilter(query, 4, "Filter\\(", 40);
}
@Test // see DRILL-2712
public void testMainQueryFilterRegularColumn() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/parquet").toURI().toString();
- String query = String.format("select * from (select dir0, o_custkey from dfs_test.`%s` where dir0='1994' and o_custkey = 10) t limit 0", root);
+ String query = "select * from (select dir0, o_custkey from dfs.`multilevel/parquet` where dir0='1994' and o_custkey = 10) t limit 0";
// with Parquet RG filter pushdown, reduce to 1 file ( o_custkey all > 10).
- testIncludeFilter(query, 1, "Filter", 0);
+ testIncludeFilter(query, 1, "Filter\\(", 0);
}
@Test // see DRILL-2852 and DRILL-3591
@Category(UnlikelyTest.class)
public void testPartitionFilterWithCast() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/parquet").toURI().toString();
- String query = String.format("select myyear, myquarter, o_totalprice from (select cast(dir0 as varchar(10)) as myyear, "
- + " cast(dir1 as varchar(10)) as myquarter, o_totalprice from dfs_test.`%s`) where myyear = cast('1995' as varchar(10)) "
- + " and myquarter = cast('Q2' as varchar(10)) and o_totalprice < 40000.0 order by o_totalprice", root);
+ String query = "select myyear, myquarter, o_totalprice from (select cast(dir0 as varchar(10)) as myyear, "
+ + " cast(dir1 as varchar(10)) as myquarter, o_totalprice from dfs.`multilevel/parquet`) where myyear = cast('1995' as varchar(10)) "
+ + " and myquarter = cast('Q2' as varchar(10)) and o_totalprice < 40000.0 order by o_totalprice";
- testIncludeFilter(query, 1, "Filter", 3);
+ testIncludeFilter(query, 1, "Filter\\(", 3);
}
@Test
public void testPPWithNestedExpression() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/parquet").toURI().toString();
- String query = String.format("select * from dfs_test.`%s` where dir0 not in(1994) and o_orderpriority = '2-HIGH'",
- root);
- testIncludeFilter(query, 8, "Filter", 24);
+ String query = "select * from dfs.`multilevel/parquet` where dir0 not in(1994) and o_orderpriority = '2-HIGH'";
+ testIncludeFilter(query, 8, "Filter\\(", 24);
}
@Test
public void testPPWithCase() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/parquet").toURI().toString();
- String query = String.format("select 1 from " +
- "(select CASE WHEN '07' = '13' THEN '13' ELSE CAST(dir0 as VARCHAR(4)) END as YEAR_FILTER from dfs_test.`%s` where o_orderpriority = '2-HIGH') subq" +
- " where subq.YEAR_FILTER not in('1994')", root);
- testIncludeFilter(query, 8, "Filter", 24);
+ String query = "select 1 from (select CASE WHEN '07' = '13' THEN '13' ELSE CAST(dir0 as VARCHAR(4)) END as YEAR_FILTER from dfs.`multilevel/parquet` " +
+ "where o_orderpriority = '2-HIGH') subq where subq.YEAR_FILTER not in('1994')";
+ testIncludeFilter(query, 8, "Filter\\(", 24);
}
@Test // DRILL-3702
@Category(UnlikelyTest.class)
public void testPartitionFilterWithNonNullabeFilterExpr() throws Exception {
- String query = String.format("select dir0, dir1, o_custkey, o_orderdate from dfs_test.`%s/multilevel/parquet` where concat(dir0, '') = '1994' and concat(dir1, '') = 'Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 10);
+ String query = "select dir0, dir1, o_custkey, o_orderdate from dfs.`multilevel/parquet` where concat(dir0, '') = '1994' and concat(dir1, '') = 'Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 10);
}
@Test // DRILL-2748
public void testPartitionFilterAfterPushFilterPastAgg() throws Exception {
- String query = String.format("select dir0, dir1, cnt from (select dir0, dir1, count(*) cnt from dfs_test.`%s/multilevel/parquet` group by dir0, dir1) where dir0 = '1994' and dir1 = 'Q1'", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 1);
+ String query = "select dir0, dir1, cnt from (select dir0, dir1, count(*) cnt from dfs.`multilevel/parquet` group by dir0, dir1) where dir0 = '1994' and dir1 = 'Q1'";
+ testExcludeFilter(query, 1, "Filter\\(", 1);
}
// Coalesce filter is on the non directory column. DRILL-4071
@Test
public void testPartitionWithCoalesceFilter_1() throws Exception {
- String query = String.format("select 1 from dfs_test.`%s/multilevel/parquet` where dir0=1994 and dir1='Q1' and coalesce(o_custkey, 0) = 890", TEST_RES_PATH);
- testIncludeFilter(query, 1, "Filter", 1);
+ String query = "select 1 from dfs.`multilevel/parquet` where dir0=1994 and dir1='Q1' and coalesce(o_custkey, 0) = 890";
+ testIncludeFilter(query, 1, "Filter\\(", 1);
}
// Coalesce filter is on the directory column
@Test
public void testPartitionWithCoalesceFilter_2() throws Exception {
- String query = String.format("select 1 from dfs_test.`%s/multilevel/parquet` where dir0=1994 and o_custkey = 890 and coalesce(dir1, 'NA') = 'Q1'", TEST_RES_PATH);
- testIncludeFilter(query, 1, "Filter", 1);
+ String query = "select 1 from dfs.`multilevel/parquet` where dir0=1994 and o_custkey = 890 and coalesce(dir1, 'NA') = 'Q1'";
+ testIncludeFilter(query, 1, "Filter\\(", 1);
}
@Test //DRILL-4021: Json with complex type and nested flatten functions: dir0 and dir1 filters plus filter involves filter refering to output from nested flatten functions.
@@ -346,70 +338,68 @@ public class TestPartitionFilter extends PlanTestBase {
public void testPartitionFilter_Json_WithFlatten() throws Exception {
// this query expects to have the partition filter pushded.
// With partition pruning, we will end with one file, and one row returned from the query.
- final String query = String.format(
+ final String query =
" select dir0, dir1, o_custkey, o_orderdate, provider from " +
" ( select dir0, dir1, o_custkey, o_orderdate, flatten(items['providers']) as provider " +
" from (" +
" select dir0, dir1, o_custkey, o_orderdate, flatten(o_items) items " +
- " from dfs_test.`%s/multilevel/jsoncomplex`) ) " +
+ " from dfs.`multilevel/jsoncomplex`) ) " +
" where dir0=1995 " + // should be pushed down and used as partitioning filter
" and dir1='Q1' " + // should be pushed down and used as partitioning filter
- " and provider = 'BestBuy'", // should NOT be pushed down.
- TEST_RES_PATH);
+ " and provider = 'BestBuy'"; // should NOT be pushed down.
- testIncludeFilter(query, 1, "Filter", 1);
+ testIncludeFilter(query, 1, "Filter\\(", 1);
}
@Test
public void testLogicalDirPruning() throws Exception {
// 1995/Q1 contains one valid parquet, while 1996/Q1 contains bad format parquet.
// If dir pruning happens in logical, the query will run fine, since the bad parquet has been pruned before we build ParquetGroupScan.
- String query = String.format("select dir0, o_custkey from dfs_test.`%s/multilevel/parquetWithBadFormat` where dir0=1995", TEST_RES_PATH);
- testExcludeFilter(query, 1, "Filter", 10);
+ String query = "select dir0, o_custkey from dfs.`multilevel/parquetWithBadFormat` where dir0=1995";
+ testExcludeFilter(query, 1, "Filter\\(", 10);
}
@Test
public void testLogicalDirPruning2() throws Exception {
// 1995/Q1 contains one valid parquet, while 1996/Q1 contains bad format parquet.
// If dir pruning happens in logical, the query will run fine, since the bad parquet has been pruned before we build ParquetGroupScan.
- String query = String.format("select dir0, o_custkey from dfs_test.`%s/multilevel/parquetWithBadFormat` where dir0=1995 and o_custkey > 0", TEST_RES_PATH);
- testIncludeFilter(query, 1, "Filter", 10);
+ String query = "select dir0, o_custkey from dfs.`multilevel/parquetWithBadFormat` where dir0=1995 and o_custkey > 0";
+ testIncludeFilter(query, 1, "Filter\\(", 10);
}
@Test //DRILL-4665: Partition pruning should occur when LIKE predicate on non-partitioning column
public void testPartitionFilterWithLike() throws Exception {
// Also should be insensitive to the order of the predicates
- String query1 = "select yr, qrtr from dfs_test.tmp.parquet where yr=1994 and o_custkey LIKE '%5%'";
- String query2 = "select yr, qrtr from dfs_test.tmp.parquet where o_custkey LIKE '%5%' and yr=1994";
- testIncludeFilter(query1, 4, "Filter", 9);
- testIncludeFilter(query2, 4, "Filter", 9);
+ String query1 = "select yr, qrtr from dfs.tmp.parquet where yr=1994 and o_custkey LIKE '%5%'";
+ String query2 = "select yr, qrtr from dfs.tmp.parquet where o_custkey LIKE '%5%' and yr=1994";
+ testIncludeFilter(query1, 4, "Filter\\(", 9);
+ testIncludeFilter(query2, 4, "Filter\\(", 9);
// Test when LIKE predicate on partitioning column
- String query3 = "select yr, qrtr from dfs_test.tmp.parquet where yr LIKE '%1995%' and o_custkey LIKE '%3%'";
- String query4 = "select yr, qrtr from dfs_test.tmp.parquet where o_custkey LIKE '%3%' and yr LIKE '%1995%'";
- testIncludeFilter(query3, 4, "Filter", 16);
- testIncludeFilter(query4, 4, "Filter", 16);
+ String query3 = "select yr, qrtr from dfs.tmp.parquet where yr LIKE '%1995%' and o_custkey LIKE '%3%'";
+ String query4 = "select yr, qrtr from dfs.tmp.parquet where o_custkey LIKE '%3%' and yr LIKE '%1995%'";
+ testIncludeFilter(query3, 4, "Filter\\(", 16);
+ testIncludeFilter(query4, 4, "Filter\\(", 16);
}
@Test //DRILL-3710 Partition pruning should occur with varying IN-LIST size
public void testPartitionFilterWithInSubquery() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/parquet` where cast (dir0 as int) IN (1994, 1994, 1994, 1994, 1994, 1994)", TEST_RES_PATH);
+ String query = "select * from dfs.`multilevel/parquet` where cast (dir0 as int) IN (1994, 1994, 1994, 1994, 1994, 1994)";
/* In list size exceeds threshold - no partition pruning since predicate converted to join */
test("alter session set `planner.in_subquery_threshold` = 2");
- testExcludeFilter(query, 12, "Filter", 40);
+ testExcludeFilter(query, 12, "Filter\\(", 40);
/* In list size does not exceed threshold - partition pruning */
test("alter session set `planner.in_subquery_threshold` = 10");
- testExcludeFilter(query, 4, "Filter", 40);
+ testExcludeFilter(query, 4, "Filter\\(", 40);
}
@Test // DRILL-4825: querying same table with different filter in UNION ALL.
public void testPruneSameTableInUnionAll() throws Exception {
- final String query = String.format("select count(*) as cnt from "
- + "( select dir0 from dfs_test.`%s/multilevel/parquet` where dir0 in ('1994') union all "
- + " select dir0 from dfs_test.`%s/multilevel/parquet` where dir0 in ('1995', '1996') )",
- TEST_RES_PATH, TEST_RES_PATH);
+ final String query = "select count(*) as cnt from "
+ + "( select dir0 from dfs.`multilevel/parquet` where dir0 in ('1994') union all "
+ + " select dir0 from dfs.`multilevel/parquet` where dir0 in ('1995', '1996') )";
- String [] excluded = {"Filter"};
+ String [] excluded = {"Filter\\("};
// verify plan that filter is applied in partition pruning.
testPlanMatchingPatterns(query, null, excluded);
@@ -426,13 +416,12 @@ public class TestPartitionFilter extends PlanTestBase {
@Test // DRILL-4825: querying same table with different filter in Join.
public void testPruneSameTableInJoin() throws Exception {
- final String query = String.format("select * from "
- + "( select sum(o_custkey) as x from dfs_test.`%s/multilevel/parquet` where dir0 in ('1994') ) join "
- + " ( select sum(o_custkey) as y from dfs_test.`%s/multilevel/parquet` where dir0 in ('1995', '1996')) "
- + " on x = y ",
- TEST_RES_PATH, TEST_RES_PATH);
+ final String query = "select * from "
+ + "( select sum(o_custkey) as x from dfs.`multilevel/parquet` where dir0 in ('1994') ) join "
+ + " ( select sum(o_custkey) as y from dfs.`multilevel/parquet` where dir0 in ('1995', '1996')) "
+ + " on x = y ";
- String [] excluded = {"Filter"};
+ String [] excluded = {"Filter\\("};
// verify plan that filter is applied in partition pruning.
testPlanMatchingPatterns(query, null, excluded);
@@ -444,5 +433,4 @@ public class TestPartitionFilter extends PlanTestBase {
.run();
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestProjectPushDown.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestProjectPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/TestProjectPushDown.java
index fe7bc08..41d1227 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestProjectPushDown.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestProjectPushDown.java
@@ -19,10 +19,13 @@
package org.apache.drill;
import org.apache.drill.categories.PlannerTest;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
// Test the optimizer plan in terms of project pushdown.
// When a query refers to a subset of columns in a table, optimizer should push the list
// of refereed columns to the SCAN operator, so that SCAN operator would only retrieve
@@ -30,8 +33,11 @@ import org.junit.experimental.categories.Category;
@Category(PlannerTest.class)
public class TestProjectPushDown extends PlanTestBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
- .getLogger(TestProjectPushDown.class);
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data", "nation.parquet"));
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data", "region.parquet"));
+ }
@Test
@Ignore
@@ -76,11 +82,11 @@ public class TestProjectPushDown extends PlanTestBase {
String expectedColNames1 = "\"columns\" : [ \"`N_REGIONKEY`\", \"`N_NAME`\" ]";
String expectedColNames2 = "\"columns\" : [ \"`R_REGIONKEY`\", \"`R_NAME`\" ]";
- testPhysicalPlan("SELECT\n" + " nations.N_NAME,\n" + " regions.R_NAME\n"
+ testPhysicalPlan("SELECT nations.N_NAME, regions.R_NAME "
+ "FROM\n"
- + " dfs_test.`[WORKING_PATH]/../../sample-data/nation.parquet` nations\n"
+ + " dfs.`sample-data/nation.parquet` nations\n"
+ "JOIN\n"
- + " dfs_test.`[WORKING_PATH]/../../sample-data/region.parquet` regions\n"
+ + " dfs.`sample-data/region.parquet` regions\n"
+ " on nations.N_REGIONKEY = regions.R_REGIONKEY", expectedColNames1,
expectedColNames2);
}
@@ -268,10 +274,6 @@ public class TestProjectPushDown extends PlanTestBase {
@Test
public void testSimpleProjectPastJoinPastFilterPastJoinPushDown() throws Exception {
-// String sql = "select * " +
-// "from cp.`%s` t0, cp.`%s` t1, cp.`%s` t2 " +
-// "where t0.fname = t1.sname and t1.slastname = t2.tlastname and t0.fcolumns[0] + t1.scolumns = 100";
-
final String firstExpected = "\"columns\" : [ \"`a`\", \"`fa`\", \"`fcolumns`[0]\" ],";
final String secondExpected = "\"columns\" : [ \"`a`\", \"`b`\", \"`c`\", \"`sa`\" ],";
final String thirdExpected = "\"columns\" : [ \"`d`\", \"`ta`\" ],";
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java
index 4465e79..f404db4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaChange.java
@@ -17,21 +17,24 @@
*/
package org.apache.drill;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.BaseTestQuery;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-public class TestSchemaChange extends BaseTestQuery {
+import java.nio.file.Paths;
- protected static final String WORKING_PATH = TestTools.getWorkingPath();
- protected static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
+public class TestSchemaChange extends BaseTestQuery {
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("schemachange"));
+ }
@Test //DRILL-1605
@Ignore("Until DRILL-2171 is fixed")
public void testMultiFilesWithDifferentSchema() throws Exception {
- final String query = String.format("select a, b from dfs_test.`%s/schemachange/multi/*.json`", TEST_RES_PATH);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select a, b from dfs.`schemachange/multi/*.json`")
.ordered()
.baselineColumns("a", "b")
.baselineValues(1L, null)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestSelectWithOption.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestSelectWithOption.java b/exec/java-exec/src/test/java/org/apache/drill/TestSelectWithOption.java
index 80565a1..5a49ad2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestSelectWithOption.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestSelectWithOption.java
@@ -18,7 +18,7 @@
package org.apache.drill;
import static java.lang.String.format;
-import static org.apache.drill.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.listOf;
import java.io.File;
import java.io.FileWriter;
@@ -26,6 +26,8 @@ import java.io.IOException;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestBuilder;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -34,7 +36,7 @@ public class TestSelectWithOption extends BaseTestQuery {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WorkspaceSchemaFactory.class);
private File genCSVFile(String name, String... rows) throws IOException {
- File file = new File(format("target/%s_%s.csv", this.getClass().getName(), name));
+ File file = new File(format("%s/%s.csv", dirTestWatcher.getRootDir(), name));
try (FileWriter fw = new FileWriter(file)) {
for (int i = 0; i < rows.length; i++) {
fw.append(rows[i] + "\n");
@@ -45,7 +47,7 @@ public class TestSelectWithOption extends BaseTestQuery {
private String genCSVTable(String name, String... rows) throws IOException {
File f = genCSVFile(name, rows);
- return format("dfs.`${WORKING_PATH}/%s`", f.getPath());
+ return format("dfs.`%s`", f.getName());
}
private void testWithResult(String query, Object... expectedResult) throws Exception {
@@ -245,9 +247,6 @@ public class TestSelectWithOption extends BaseTestQuery {
String[] jsonQueries = {
format("select columns from table(%s ('JSON'))", jsonTableName),
format("select columns from table(%s(type => 'JSON'))", jsonTableName),
-// format("select columns from %s ('JSON')", jsonTableName),
-// format("select columns from %s (type => 'JSON')", jsonTableName),
-// format("select columns from %s(type => 'JSON')", jsonTableName),
// we can use named format plugin configurations too!
format("select columns from table(%s(type => 'Named', name => 'json'))", jsonTableName),
};
@@ -260,7 +259,7 @@ public class TestSelectWithOption extends BaseTestQuery {
public void testUse() throws Exception {
File f = genCSVFile("testUse",
"{\"columns\": [\"f\",\"g\"]}");
- String jsonTableName = format("`${WORKING_PATH}/%s`", f.getPath());
+ String jsonTableName = String.format("dfs.`%s`", f.getName());
// the extension is actually csv
test("use dfs");
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java b/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java
index 0c41ae0..de49a5f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestSelectivity.java
@@ -18,6 +18,7 @@
package org.apache.drill;
import org.apache.drill.categories.PlannerTest;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
index b4ac11f..34c8c6c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
@@ -22,20 +22,25 @@ import org.apache.drill.categories.SqlTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.test.rowSet.SchemaBuilder;
+import org.apache.drill.test.BaseTestQuery;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
import static org.junit.Assert.assertEquals;
@Category({SqlTest.class, PlannerTest.class})
-public class TestStarQueries extends BaseTestQuery{
+public class TestStarQueries extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestStarQueries.class);
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
+
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel", "parquet"));
+ }
@Test // see DRILL-2021
@Category(UnlikelyTest.class)
@@ -248,7 +253,7 @@ public class TestStarQueries extends BaseTestQuery{
@Test // DRILL-1293
@Category(UnlikelyTest.class)
public void testStarView1() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view vt1 as select * from cp.`tpch/region.parquet` r, cp.`tpch/nation.parquet` n where r.r_regionkey = n.n_regionkey");
test("select * from vt1");
test("drop view vt1");
@@ -416,11 +421,10 @@ public class TestStarQueries extends BaseTestQuery{
@Test // DRILL-1500
@Category(UnlikelyTest.class)
public void testStarPartitionFilterOrderBy() throws Exception {
- String query = String.format("select * from dfs_test.`%s/multilevel/parquet` where dir0=1994 and dir1='Q1' order by dir0 limit 1", TEST_RES_PATH);
org.joda.time.DateTime mydate = new org.joda.time.DateTime("1994-01-20T00:00:00.000");
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select * from dfs.`multilevel/parquet` where dir0=1994 and dir1='Q1' order by dir0 limit 1")
.ordered()
.baselineColumns("dir0", "dir1", "o_clerk", "o_comment", "o_custkey", "o_orderdate", "o_orderkey", "o_orderpriority", "o_orderstatus", "o_shippriority", "o_totalprice")
.baselineValues("1994", "Q1", "Clerk#000000743", "y pending requests integrate", 1292, mydate, 66, "5-LOW", "F", 0, 104190.66)
@@ -481,19 +485,13 @@ public class TestStarQueries extends BaseTestQuery{
@Test //DRILL-2802
@Category(UnlikelyTest.class)
public void testSelectPartitionColumnOnly() throws Exception {
- final String table = FileUtils.getResourceAsFile("/multilevel/parquet").toURI().toString();
- final String query1 = String.format("select dir0 from dfs_test.`%s` limit 1 ", table);
-
final String[] expectedPlan1 = {".*Project.*dir0=\\[\\$0\\]"};
final String[] excludedPlan1 = {};
- PlanTestBase.testPlanMatchingPatterns(query1, expectedPlan1, excludedPlan1);
-
- final String query2 = String.format("select dir0, dir1 from dfs_test.`%s` limit 1 ", table);
+ PlanTestBase.testPlanMatchingPatterns("select dir0 from dfs.`multilevel/parquet` limit 1", expectedPlan1, excludedPlan1);
final String[] expectedPlan2 = {".*Project.*dir0=\\[\\$0\\], dir1=\\[\\$1\\]"};
final String[] excludedPlan2 = {};
- PlanTestBase.testPlanMatchingPatterns(query2, expectedPlan2, excludedPlan2);
-
+ PlanTestBase.testPlanMatchingPatterns("select dir0, dir1 from dfs.`multilevel/parquet` limit 1", expectedPlan2, excludedPlan2);
}
@Test // DRILL-2053 : column name is case-insensitive when join a CTE with a regluar table.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java b/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java
index 7fd9c8e..d427a3d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTextJoin.java
@@ -17,29 +17,22 @@
*/
package org.apache.drill;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
-public class TestTextJoin extends BaseTestQuery{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTextJoin.class);
-
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
-
+public class TestTextJoin extends BaseTestQuery {
@Test
public void testTextJoin1() throws Exception {
- String query1 = String.format("select r.columns[0] as v, r.columns[1] as w, r.columns[2] as x, u.columns[0] as y, t.columns[0] as z from dfs_test.`%s/uservisits/rankings.tbl` r, "
- + " dfs_test.`%s/uservisits/uservisits.tbl` u, dfs_test.`%s/uservisits/temp1.tbl` t "
- + " where r.columns[1]=u.columns[1] and r.columns[1] = t.columns[1]", TEST_RES_PATH, TEST_RES_PATH, TEST_RES_PATH);
- test(query1);
+ test("select r.columns[0] as v, r.columns[1] as w, r.columns[2] as x, u.columns[0] as y, t.columns[0] as z "
+ + " from cp.`uservisits/rankings.tbl` r, "
+ + " cp.`uservisits/uservisits.tbl` u, cp.`uservisits/temp1.tbl` t "
+ + " where r.columns[1]=u.columns[1] and r.columns[1] = t.columns[1]");
}
@Test
public void testTextJoin2() throws Exception {
- String query1 = String.format("select r.columns[0] as v, r.columns[1] as w, r.columns[2] as x, u.columns[0] as y "
- + " from dfs_test.`%s/uservisits/rankings.tbl` r, dfs_test.`%s/uservisits/uservisits.tbl` u "
- + " where r.columns[1]=u.columns[1] and r.columns[0] < 50", TEST_RES_PATH, TEST_RES_PATH);
- test(query1);
+ test("select r.columns[0] as v, r.columns[1] as w, r.columns[2] as x, u.columns[0] as y "
+ + " from cp.`uservisits/rankings.tbl` r, cp.`uservisits/uservisits.tbl` u "
+ + " where r.columns[1]=u.columns[1] and r.columns[0] < 50");
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
index 5164de5..88a2311 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
@@ -18,6 +18,7 @@
package org.apache.drill;
import org.apache.drill.categories.SlowTest;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
index 07b9eb6..917890b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
@@ -26,10 +26,12 @@ import java.util.concurrent.Semaphore;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.QueryTestUtil;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java
index 94433f4..2a8ccc3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedStreaming.java
@@ -17,10 +17,11 @@
*/
package org.apache.drill;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Ignore;
import org.junit.Test;
-public class TestTpchDistributedStreaming extends BaseTestQuery{
+public class TestTpchDistributedStreaming extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchDistributedStreaming.class);
private void testDistributed(String fileName) throws Exception{
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java
index 1fc437c..204684b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchExplain.java
@@ -18,12 +18,13 @@
package org.apache.drill;
import org.apache.drill.categories.PlannerTest;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(PlannerTest.class)
-public class TestTpchExplain extends BaseTestQuery{
+public class TestTpchExplain extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchExplain.class);
private static final String EXPLAIN_PREFIX = "EXPLAIN PLAN FOR ";
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
index 68ebdda..940b4af 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchLimit0.java
@@ -18,12 +18,13 @@
package org.apache.drill;
import org.apache.drill.categories.PlannerTest;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(PlannerTest.class)
-public class TestTpchLimit0 extends BaseTestQuery{
+public class TestTpchLimit0 extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchLimit0.class);
private void testLimitZero(String fileName) throws Exception {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestTpchPlanning.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchPlanning.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchPlanning.java
index 1c93a07..8299e6c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchPlanning.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchPlanning.java
@@ -18,7 +18,7 @@
package org.apache.drill;
import org.apache.drill.categories.PlannerTest;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java
index abefb16..a76ec8a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchSingleMode.java
@@ -17,10 +17,11 @@
*/
package org.apache.drill;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Ignore;
import org.junit.Test;
-public class TestTpchSingleMode extends BaseTestQuery{
+public class TestTpchSingleMode extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTpchSingleMode.class);
private static final String SINGLE_MODE = "ALTER SESSION SET `planner.disable_exchanges` = true;";
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
index 831b70c..655d036 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
@@ -26,16 +26,17 @@ import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.work.foreman.SqlUnsupportedException;
import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException;
-import org.junit.Ignore;
+import org.apache.drill.test.BaseTestQuery;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
+import java.nio.file.Paths;
import java.util.List;
@Category({SqlTest.class, OperatorTest.class})
@@ -46,6 +47,11 @@ public class TestUnionAll extends BaseTestQuery {
private static final String enableDistribute = "alter session set `planner.enable_unionall_distribute` = true";
private static final String defaultDistribute = "alter session reset `planner.enable_unionall_distribute`";
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel", "parquet"));
+ }
+
@Test // Simple Union-All over two scans
public void testUnionAll1() throws Exception {
String query = "(select n_regionkey from cp.`tpch/nation.parquet`) union all (select r_regionkey from cp.`tpch/region.parquet`)";
@@ -171,13 +177,12 @@ public class TestUnionAll extends BaseTestQuery {
@Test // DRILL-1905: Union-all of * column from JSON files in different directories
@Category(UnlikelyTest.class)
public void testUnionAll9() throws Exception {
- String file0 = FileUtils.getResourceAsFile("/multilevel/json/1994/Q1/orders_94_q1.json").toURI().toString();
- String file1 = FileUtils.getResourceAsFile("/multilevel/json/1995/Q1/orders_95_q1.json").toURI().toString();
- String query = String.format("select o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, o_orderkey from dfs_test.`%s` union all " +
- "select o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, o_orderkey from dfs_test.`%s`", file0, file1);
+ String file0 = "/multilevel/json/1994/Q1/orders_94_q1.json";
+ String file1 = "/multilevel/json/1995/Q1/orders_95_q1.json";
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, o_orderkey from cp.`%s` union " +
+ "all select o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, o_orderkey from cp.`%s`", file0, file1)
.unOrdered()
.csvBaselineFile("testframework/testUnionAllQueries/q9.tsv")
.baselineTypes(TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.VARCHAR, TypeProtos.MinorType.FLOAT8, TypeProtos.MinorType.VARCHAR,
@@ -204,17 +209,17 @@ public class TestUnionAll extends BaseTestQuery {
@Test
public void testUnionAllViewExpandableStar() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testunionall as select n_name, n_nationkey from cp.`tpch/nation.parquet`;");
test("create view region_view_testunionall as select r_name, r_regionkey from cp.`tpch/region.parquet`;");
- String query1 = "(select * from dfs_test.tmp.`nation_view_testunionall`) " +
+ String query1 = "(select * from dfs.tmp.`nation_view_testunionall`) " +
"union all " +
- "(select * from dfs_test.tmp.`region_view_testunionall`) ";
+ "(select * from dfs.tmp.`region_view_testunionall`) ";
String query2 = "(select r_name, r_regionkey from cp.`tpch/region.parquet`) " +
"union all " +
- "(select * from dfs_test.tmp.`nation_view_testunionall`)";
+ "(select * from dfs.tmp.`nation_view_testunionall`)";
try {
testBuilder()
@@ -240,11 +245,11 @@ public class TestUnionAll extends BaseTestQuery {
@Test(expected = UnsupportedRelOperatorException.class) // see DRILL-2002
public void testUnionAllViewUnExpandableStar() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testunionall as select * from cp.`tpch/nation.parquet`;");
try {
- String query = "(select * from dfs_test.tmp.`nation_view_testunionall`) " +
+ String query = "(select * from dfs.tmp.`nation_view_testunionall`) " +
"union all (select * from cp.`tpch/region.parquet`)";
test(query);
} catch(UserException ex) {
@@ -257,7 +262,7 @@ public class TestUnionAll extends BaseTestQuery {
@Test
public void testDiffDataTypesAndModes() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testunionall as select n_name, n_nationkey from cp.`tpch/nation.parquet`;");
test("create view region_view_testunionall as select r_name, r_regionkey from cp.`tpch/region.parquet`;");
@@ -348,23 +353,13 @@ public class TestUnionAll extends BaseTestQuery {
@Test // see DRILL-1977, DRILL-2376, DRILL-2377, DRILL-2378, DRILL-2379
@Category(UnlikelyTest.class)
public void testAggregationOnUnionAllOperator() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString();
- String query1 = String.format(
- "(select calc1, max(b1) as `max`, min(b1) as `min`, count(c1) as `count` " +
- "from (select a1 + 10 as calc1, b1, c1 from dfs_test.`%s` " +
- "union all " +
- "select a1 + 100 as diff1, b1 as diff2, c1 as diff3 from dfs_test.`%s`) " +
- "group by calc1 order by calc1)", root, root);
-
- String query2 = String.format(
- "(select calc1, min(b1) as `min`, max(b1) as `max`, count(c1) as `count` " +
- "from (select a1 + 10 as calc1, b1, c1 from dfs_test.`%s` " +
- "union all " +
- "select a1 + 100 as diff1, b1 as diff2, c1 as diff3 from dfs_test.`%s`) " +
- "group by calc1 order by calc1)", root, root);
+ String root = "/store/text/data/t.json";
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("(select calc1, max(b1) as `max`, min(b1) as `min`, count(c1) as `count` " +
+ "from (select a1 + 10 as calc1, b1, c1 from cp.`%s` " +
+ "union all select a1 + 100 as diff1, b1 as diff2, c1 as diff3 from cp.`%s`) " +
+ "group by calc1 order by calc1)", root, root)
.ordered()
.csvBaselineFile("testframework/testExampleQueries/testAggregationOnUnionAllOperator/q1.tsv")
.baselineTypes(TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT)
@@ -372,7 +367,10 @@ public class TestUnionAll extends BaseTestQuery {
.build().run();
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("(select calc1, min(b1) as `min`, max(b1) as `max`, count(c1) as `count` " +
+ "from (select a1 + 10 as calc1, b1, c1 from cp.`%s` " +
+ "union all select a1 + 100 as diff1, b1 as diff2, c1 as diff3 from cp.`%s`) " +
+ "group by calc1 order by calc1)", root, root)
.ordered()
.csvBaselineFile("testframework/testExampleQueries/testAggregationOnUnionAllOperator/q2.tsv")
.baselineTypes(TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.BIGINT)
@@ -382,40 +380,23 @@ public class TestUnionAll extends BaseTestQuery {
@Test(expected = UserException.class) // see DRILL-2590
public void testUnionAllImplicitCastingFailure() throws Exception {
- String rootInt = FileUtils.getResourceAsFile("/store/json/intData.json").toURI().toString();
- String rootBoolean = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
+ String rootInt = "/store/json/intData.json";
+ String rootBoolean = "/store/json/booleanData.json";
- String query = String.format(
- "(select key from dfs_test.`%s` " +
+ test("(select key from cp.`%s` " +
"union all " +
- "select key from dfs_test.`%s` )", rootInt, rootBoolean);
-
- test(query);
+ "select key from cp.`%s` )", rootInt, rootBoolean);
}
@Test // see DRILL-2591
@Category(UnlikelyTest.class)
public void testDateAndTimestampJson() throws Exception {
- String rootDate = FileUtils.getResourceAsFile("/store/json/dateData.json").toURI().toString();
- String rootTimpStmp = FileUtils.getResourceAsFile("/store/json/timeStmpData.json").toURI().toString();
-
- String query1 = String.format(
- "(select max(key) as key from dfs_test.`%s` " +
- "union all " +
- "select key from dfs_test.`%s`)", rootDate, rootTimpStmp);
-
- String query2 = String.format(
- "select key from dfs_test.`%s` " +
- "union all " +
- "select max(key) as key from dfs_test.`%s`", rootDate, rootTimpStmp);
-
- String query3 = String.format(
- "select key from dfs_test.`%s` " +
- "union all " +
- "select max(key) as key from dfs_test.`%s`", rootDate, rootTimpStmp);
+ String rootDate = "/store/json/dateData.json";
+ String rootTimpStmp = "/store/json/timeStmpData.json";
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("(select max(key) as key from cp.`%s` " +
+ "union all select key from cp.`%s`)", rootDate, rootTimpStmp)
.unOrdered()
.csvBaselineFile("testframework/testUnionAllQueries/q18_1.tsv")
.baselineTypes(TypeProtos.MinorType.VARCHAR)
@@ -423,7 +404,8 @@ public class TestUnionAll extends BaseTestQuery {
.build().run();
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select key from cp.`%s` " +
+ "union all select max(key) as key from cp.`%s`", rootDate, rootTimpStmp)
.unOrdered()
.csvBaselineFile("testframework/testUnionAllQueries/q18_2.tsv")
.baselineTypes(TypeProtos.MinorType.VARCHAR)
@@ -431,7 +413,8 @@ public class TestUnionAll extends BaseTestQuery {
.build().run();
testBuilder()
- .sqlQuery(query3)
+ .sqlQuery("select key from cp.`%s` " +
+ "union all select max(key) as key from cp.`%s`", rootDate, rootTimpStmp)
.unOrdered()
.csvBaselineFile("testframework/testUnionAllQueries/q18_3.tsv")
.baselineTypes(TypeProtos.MinorType.VARCHAR)
@@ -442,21 +425,11 @@ public class TestUnionAll extends BaseTestQuery {
@Test // see DRILL-2637
@Category(UnlikelyTest.class)
public void testUnionAllOneInputContainsAggFunction() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/csv/1994/Q1/orders_94_q1.csv").toURI().toString();
- String query1 = String.format("select * from ((select count(c1) as ct from (select columns[0] c1 from dfs.`%s`)) \n" +
- "union all \n" +
- "(select columns[0] c2 from dfs.`%s`)) order by ct limit 3", root, root);
-
- String query2 = String.format("select * from ((select columns[0] ct from dfs.`%s`)\n" +
- "union all \n" +
- "(select count(c1) as c2 from (select columns[0] c1 from dfs.`%s`))) order by ct limit 3", root, root);
-
- String query3 = String.format("select * from ((select count(c1) as ct from (select columns[0] c1 from dfs.`%s`))\n" +
- "union all \n" +
- "(select count(c1) as c2 from (select columns[0] c1 from dfs.`%s`))) order by ct", root, root);
+ String root = "/multilevel/csv/1994/Q1/orders_94_q1.csv";
testBuilder()
- .sqlQuery(query1)
+ .sqlQuery("select * from ((select count(c1) as ct from (select columns[0] c1 from cp.`%s`)) \n" +
+ "union all (select columns[0] c2 from cp.`%s`)) order by ct limit 3", root, root)
.ordered()
.baselineColumns("ct")
.baselineValues((long) 10)
@@ -465,7 +438,8 @@ public class TestUnionAll extends BaseTestQuery {
.build().run();
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select * from ((select columns[0] ct from cp.`%s`)\n" +
+ "union all (select count(c1) as c2 from (select columns[0] c1 from cp.`%s`))) order by ct limit 3", root, root)
.ordered()
.baselineColumns("ct")
.baselineValues((long) 10)
@@ -474,7 +448,8 @@ public class TestUnionAll extends BaseTestQuery {
.build().run();
testBuilder()
- .sqlQuery(query3)
+ .sqlQuery("select * from ((select count(c1) as ct from (select columns[0] c1 from cp.`%s`))\n" +
+ "union all (select count(c1) as c2 from (select columns[0] c1 from cp.`%s`))) order by ct", root, root)
.ordered()
.baselineColumns("ct")
.baselineValues((long) 10)
@@ -485,19 +460,18 @@ public class TestUnionAll extends BaseTestQuery {
@Test // see DRILL-2717
@Category(UnlikelyTest.class)
public void testUnionInputsGroupByOnCSV() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/csv/1994/Q1/orders_94_q1.csv").toURI().toString();
- String query = String.format("select * from \n" +
- "((select columns[0] as col0 from dfs.`%s` t1 \n" +
+ String root = "/multilevel/csv/1994/Q1/orders_94_q1.csv";
+
+ testBuilder()
+ .sqlQuery("select * from \n" +
+ "((select columns[0] as col0 from cp.`%s` t1 \n" +
"where t1.columns[0] = 66) \n" +
"union all \n" +
- "(select columns[0] c2 from dfs.`%s` t2 \n" +
+ "(select columns[0] c2 from cp.`%s` t2 \n" +
"where t2.columns[0] is not null \n" +
"group by columns[0])) \n" +
- "group by col0"
- , root, root);
-
- testBuilder()
- .sqlQuery(query)
+ "group by col0"
+ , root, root)
.unOrdered()
.baselineColumns("col0")
.baselineValues("290")
@@ -516,12 +490,9 @@ public class TestUnionAll extends BaseTestQuery {
@Test // see DRILL-2639
@Category(UnlikelyTest.class)
public void testUnionAllDiffTypesAtPlanning() throws Exception {
- String query = "select count(c1) as ct from (select cast(r_regionkey as int) c1 from cp.`tpch/region.parquet`) " +
- "union all " +
- "(select cast(r_regionkey as int) c2 from cp.`tpch/region.parquet`)";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select count(c1) as ct from (select cast(r_regionkey as int) c1 from cp.`tpch/region.parquet`) " +
+ "union all (select cast(r_regionkey as int) c2 from cp.`tpch/region.parquet`)")
.ordered()
.baselineColumns("ct")
.baselineValues((long) 5)
@@ -536,18 +507,15 @@ public class TestUnionAll extends BaseTestQuery {
@Test // see DRILL-2612
@Category(UnlikelyTest.class)
public void testUnionAllRightEmptyJson() throws Exception {
- String rootEmpty = FileUtils.getResourceAsFile("/project/pushdown/empty.json").toURI().toString();
- String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- String queryRightEmpty = String.format(
- "select key from dfs_test.`%s` " +
- "union all " +
- "select key from dfs_test.`%s`",
- rootSimple,
- rootEmpty);
+ String rootEmpty = "/project/pushdown/empty.json";
+ String rootSimple = "/store/json/booleanData.json";
testBuilder()
- .sqlQuery(queryRightEmpty)
+ .sqlQuery("select key from cp.`%s` " +
+ "union all " +
+ "select key from cp.`%s`",
+ rootSimple,
+ rootEmpty)
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -557,18 +525,15 @@ public class TestUnionAll extends BaseTestQuery {
@Test
public void testUnionAllLeftEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile("/project/pushdown/empty.json").toURI().toString();
- final String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- final String queryLeftEmpty = String.format(
- "select key from dfs_test.`%s` " +
- "union all " +
- "select key from dfs_test.`%s`",
- rootEmpty,
- rootSimple);
+ final String rootEmpty = "/project/pushdown/empty.json";
+ final String rootSimple = "/store/json/booleanData.json";
testBuilder()
- .sqlQuery(queryLeftEmpty)
+ .sqlQuery("select key from cp.`%s` " +
+ "union all " +
+ "select key from cp.`%s`",
+ rootEmpty,
+ rootSimple)
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -579,13 +544,7 @@ public class TestUnionAll extends BaseTestQuery {
@Test
public void testUnionAllBothEmptyJson() throws Exception {
- final String rootEmpty = FileUtils.getResourceAsFile("/project/pushdown/empty.json").toURI().toString();
- final String query = String.format(
- "select key from dfs_test.`%s` " +
- "union all " +
- "select key from dfs_test.`%s`",
- rootEmpty,
- rootEmpty);
+ final String rootEmpty = "/project/pushdown/empty.json";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
final TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
@@ -595,7 +554,11 @@ public class TestUnionAll extends BaseTestQuery {
expectedSchema.add(Pair.of(SchemaPath.getSimplePath("key"), majorType));
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select key from cp.`%s` " +
+ "union all " +
+ "select key from cp.`%s`",
+ rootEmpty,
+ rootEmpty)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -603,17 +566,14 @@ public class TestUnionAll extends BaseTestQuery {
@Test
public void testUnionAllRightEmptyDataBatch() throws Exception {
- String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- String queryRightEmptyBatch = String.format(
- "select key from dfs_test.`%s` " +
- "union all " +
- "select key from dfs_test.`%s` where 1 = 0",
- rootSimple,
- rootSimple);
+ String rootSimple = "/store/json/booleanData.json";
testBuilder()
- .sqlQuery(queryRightEmptyBatch)
+ .sqlQuery("select key from cp.`%s` " +
+ "union all " +
+ "select key from cp.`%s` where 1 = 0",
+ rootSimple,
+ rootSimple)
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -623,17 +583,14 @@ public class TestUnionAll extends BaseTestQuery {
@Test
public void testUnionAllLeftEmptyDataBatch() throws Exception {
- String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- final String queryLeftBatch = String.format(
- "select key from dfs_test.`%s` where 1 = 0 " +
- "union all " +
- "select key from dfs_test.`%s`",
- rootSimple,
- rootSimple);
+ String rootSimple = "/store/json/booleanData.json";
testBuilder()
- .sqlQuery(queryLeftBatch)
+ .sqlQuery("select key from cp.`%s` where 1 = 0 " +
+ "union all " +
+ "select key from cp.`%s`",
+ rootSimple,
+ rootSimple)
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -644,13 +601,7 @@ public class TestUnionAll extends BaseTestQuery {
@Test
public void testUnionAllBothEmptyDataBatch() throws Exception {
- String rootSimple = FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
- final String query = String.format(
- "select key from dfs_test.`%s` where 1 = 0 " +
- "union all " +
- "select key from dfs_test.`%s` where 1 = 0",
- rootSimple,
- rootSimple);
+ String rootSimple = "/store/json/booleanData.json";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
final TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
@@ -660,7 +611,11 @@ public class TestUnionAll extends BaseTestQuery {
expectedSchema.add(Pair.of(SchemaPath.getSimplePath("key"), majorType));
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select key from cp.`%s` where 1 = 0 " +
+ "union all " +
+ "select key from cp.`%s` where 1 = 0",
+ rootSimple,
+ rootSimple)
.schemaBaseLine(expectedSchema)
.build()
.run();
@@ -733,11 +688,12 @@ public class TestUnionAll extends BaseTestQuery {
@Test // see DRILL-2746
public void testFilterPushDownOverUnionAllCSV() throws Exception {
- String root = FileUtils.getResourceAsFile("/multilevel/csv/1994/Q1/orders_94_q1.csv").toURI().toString();
+ String root = "/multilevel/csv/1994/Q1/orders_94_q1.csv";
+
String query = String.format("select ct \n" +
- "from ((select count(c1) as ct from (select columns[0] c1 from dfs.`%s`)) \n" +
+ "from ((select count(c1) as ct from (select columns[0] c1 from cp.`%s`)) \n" +
"union all \n" +
- "(select columns[0] c2 from dfs.`%s`)) \n" +
+ "(select columns[0] c2 from cp.`%s`)) \n" +
"where ct < 100", root, root);
// Validate the plan
@@ -845,10 +801,10 @@ public class TestUnionAll extends BaseTestQuery {
@Test // see DRILL-3130
public void testProjectDownOverUnionAllImplicitCasting() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/nations.csv").toURI().toString();
+ String root = "/store/text/data/nations.csv";
String query = String.format("select 2 * n_nationkey as col from \n" +
"(select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` \n" +
- "union all select columns[0], columns[1], columns[2] from dfs.`%s`) \n" +
+ "union all select columns[0], columns[1], columns[2] from cp.`%s`) \n" +
"order by col limit 10", root);
// Validate the plan
@@ -1043,11 +999,11 @@ public class TestUnionAll extends BaseTestQuery {
@Test // DRILL-4147 // base case
@Category(UnlikelyTest.class)
public void testDrill4147_1() throws Exception {
- final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
- final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+ final String l = "/multilevel/parquet/1994";
+ final String r = "/multilevel/parquet/1995";
- final String query = String.format("SELECT o_custkey FROM dfs_test.`%s` \n" +
- "Union All SELECT o_custkey FROM dfs_test.`%s`", l, r);
+ final String query = String.format("SELECT o_custkey FROM dfs.`%s` \n" +
+ "Union All SELECT o_custkey FROM dfs.`%s`", l, r);
// Validate the plan
final String[] expectedPlan = {"UnionExchange.*\n",
@@ -1073,12 +1029,12 @@ public class TestUnionAll extends BaseTestQuery {
@Test // DRILL-4147 // group-by on top of union-all
public void testDrill4147_2() throws Exception {
- final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
- final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+ final String l = "/multilevel/parquet/1994";
+ final String r = "/multilevel/parquet/1995";
final String query = String.format("Select o_custkey, count(*) as cnt from \n" +
- " (SELECT o_custkey FROM dfs_test.`%s` \n" +
- "Union All SELECT o_custkey FROM dfs_test.`%s`) \n" +
+ " (SELECT o_custkey FROM dfs.`%s` \n" +
+ "Union All SELECT o_custkey FROM dfs.`%s`) \n" +
"group by o_custkey", l, r);
// Validate the plan
@@ -1104,12 +1060,12 @@ public class TestUnionAll extends BaseTestQuery {
@Test // DRILL-4147 // union-all above a hash join
public void testDrill4147_3() throws Exception {
- final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
- final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+ final String l = "/multilevel/parquet/1994";
+ final String r = "/multilevel/parquet/1995";
final String query = String.format("SELECT o_custkey FROM \n" +
- " (select o1.o_custkey from dfs_test.`%s` o1 inner join dfs_test.`%s` o2 on o1.o_orderkey = o2.o_custkey) \n" +
- " Union All SELECT o_custkey FROM dfs_test.`%s` where o_custkey > 10", l, r, l);
+ " (select o1.o_custkey from dfs.`%s` o1 inner join dfs.`%s` o2 on o1.o_orderkey = o2.o_custkey) \n" +
+ " Union All SELECT o_custkey FROM dfs.`%s` where o_custkey > 10", l, r, l);
// Validate the plan
final String[] expectedPlan = {"(?s)UnionExchange.*UnionAll.*HashJoin.*"};
@@ -1135,12 +1091,12 @@ public class TestUnionAll extends BaseTestQuery {
@Test // DRILL-4833 // limit 1 is on RHS of union-all
@Category(UnlikelyTest.class)
public void testDrill4833_1() throws Exception {
- final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
- final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+ final String l = "/multilevel/parquet/1994";
+ final String r = "/multilevel/parquet/1995";
final String query = String.format("SELECT o_custkey FROM \n" +
- " ((select o1.o_custkey from dfs_test.`%s` o1 inner join dfs_test.`%s` o2 on o1.o_orderkey = o2.o_custkey) \n" +
- " Union All (SELECT o_custkey FROM dfs_test.`%s` limit 1))", l, r, l);
+ " ((select o1.o_custkey from dfs.`%s` o1 inner join dfs.`%s` o2 on o1.o_orderkey = o2.o_custkey) \n" +
+ " Union All (SELECT o_custkey FROM dfs.`%s` limit 1))", l, r, l);
// Validate the plan
final String[] expectedPlan = {"(?s)UnionExchange.*UnionAll.*HashJoin.*"};
@@ -1169,13 +1125,13 @@ public class TestUnionAll extends BaseTestQuery {
@Test // DRILL-4833 // limit 1 is on LHS of union-all
@Category(UnlikelyTest.class)
public void testDrill4833_2() throws Exception {
- final String l = FileUtils.getResourceAsFile("/multilevel/parquet/1994").toURI().toString();
- final String r = FileUtils.getResourceAsFile("/multilevel/parquet/1995").toURI().toString();
+ final String l = "/multilevel/parquet/1994";
+ final String r = "/multilevel/parquet/1995";
final String query = String.format("SELECT o_custkey FROM \n" +
- " ((SELECT o_custkey FROM dfs_test.`%s` limit 1) \n" +
+ " ((SELECT o_custkey FROM dfs.`%s` limit 1) \n" +
" union all \n" +
- " (select o1.o_custkey from dfs_test.`%s` o1 inner join dfs_test.`%s` o2 on o1.o_orderkey = o2.o_custkey))", l, r, l);
+ " (select o1.o_custkey from dfs.`%s` o1 inner join dfs.`%s` o2 on o1.o_orderkey = o2.o_custkey))", l, r, l);
// Validate the plan
final String[] expectedPlan = {"(?s)UnionExchange.*UnionAll.*HashJoin.*"};
@@ -1215,39 +1171,30 @@ public class TestUnionAll extends BaseTestQuery {
@Test // DRILL-4264
@Category(UnlikelyTest.class)
public void testFieldWithDots() throws Exception {
- File directory = new File(BaseTestQuery.getTempDir("json/input"));
- try {
- directory.mkdirs();
- String fileName = "table.json";
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
- writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
- }
-
- String query = String.format("select * from (" +
- "(select t.m.`a.b` as a,\n" +
- "t.m.a.b as b,\n" +
- "t.m['a.b'] as c,\n" +
- "t.rk.q as d,\n" +
- "t.`rk.q` as e\n" +
- "from dfs_test.`%1$s/%2$s` t)\n" +
- "union all\n" +
- "(select t.m.`a.b` as a,\n" +
- "t.m.a.b as b,\n" +
- "t.m['a.b'] as c,\n" +
- "t.rk.q as d,\n" +
- "t.`rk.q` as e\n" +
- "from dfs_test.`%1$s/%2$s` t))",
- directory.toPath().toString(), fileName);
- testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .baselineColumns("a", "b", "c", "d", "e")
- .baselineValues("1", "2", "1", null, "a")
- .baselineValues("1", "2", "1", null, "a")
- .go();
-
- } finally {
- org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ String fileName = "table.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), fileName)))) {
+ writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
}
+
+ testBuilder()
+ .sqlQuery("select * from (" +
+ "(select t.m.`a.b` as a,\n" +
+ "t.m.a.b as b,\n" +
+ "t.m['a.b'] as c,\n" +
+ "t.rk.q as d,\n" +
+ "t.`rk.q` as e\n" +
+ "from dfs.`%1$s` t)\n" +
+ "union all\n" +
+ "(select t.m.`a.b` as a,\n" +
+ "t.m.a.b as b,\n" +
+ "t.m['a.b'] as c,\n" +
+ "t.rk.q as d,\n" +
+ "t.`rk.q` as e\n" +
+ "from dfs.`%1$s` t))", fileName)
+ .unOrdered()
+ .baselineColumns("a", "b", "c", "d", "e")
+ .baselineValues("1", "2", "1", null, "a")
+ .baselineValues("1", "2", "1", null, "a")
+ .go();
}
}
\ No newline at end of file
[15/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
index 6ae8ae1..ebad2f7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.fn.impl;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.PlanTestBase;
import org.apache.drill.categories.PlannerTest;
@@ -28,9 +28,9 @@ import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -38,13 +38,17 @@ import org.junit.experimental.categories.Category;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
@Category({SqlFunctionTest.class, OperatorTest.class, PlannerTest.class})
public class TestAggregateFunctions extends BaseTestQuery {
- private static final String TEST_RES_PATH = TestTools.getWorkingPath() + "/src/test/resources";
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("agg"));
+ }
/*
* Test checks the count of a nullable column within a map
@@ -54,7 +58,7 @@ public class TestAggregateFunctions extends BaseTestQuery {
@Test
public void testCountOnNullableColumn() throws Exception {
testBuilder()
- .sqlQuery("select count(t.x.y) as cnt1, count(`integer`) as cnt2 from cp.`/jsoninput/input2.json` t")
+ .sqlQuery("select count(t.x.y) as cnt1, count(`integer`) as cnt2 from cp.`jsoninput/input2.json` t")
.ordered()
.baselineColumns("cnt1", "cnt2")
.baselineValues(3l, 4l)
@@ -352,23 +356,23 @@ public class TestAggregateFunctions extends BaseTestQuery {
*/
@Test
public void drill3069() throws Exception {
- final String query = "select max(foo) col1 from dfs_test.`%s/agg/bugs/drill3069` where foo = %d";
+ final String query = "select max(foo) col1 from dfs.`agg/bugs/drill3069` where foo = %d";
testBuilder()
- .sqlQuery(String.format(query, TEST_RES_PATH, 2))
+ .sqlQuery(query, 2)
.unOrdered()
.baselineColumns("col1")
.baselineValues(2l)
.go();
testBuilder()
- .sqlQuery(String.format(query, TEST_RES_PATH, 4))
+ .sqlQuery(query, 4)
.unOrdered()
.baselineColumns("col1")
.baselineValues(4l)
.go();
testBuilder()
- .sqlQuery(String.format(query, TEST_RES_PATH, 6))
+ .sqlQuery(query, 6)
.unOrdered()
.baselineColumns("col1")
.baselineValues(6l)
@@ -466,25 +470,22 @@ public class TestAggregateFunctions extends BaseTestQuery {
@Category(UnlikelyTest.class)
// GROUP BY System functions in csv, parquet, json table.
public void testGroupBySystemFuncFileSystemTable() throws Exception {
- final String query = String.format("select count(*) as cnt from dfs_test.`%s/nation/nation.tbl` group by CURRENT_DATE", TEST_RES_PATH);
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select count(*) as cnt from cp.`nation/nation.tbl` group by CURRENT_DATE")
.unOrdered()
.baselineColumns("cnt")
.baselineValues(25l)
.build().run();
- final String query2 = "select count(*) as cnt from cp.`tpch/nation.parquet` group by CURRENT_DATE";
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select count(*) as cnt from cp.`tpch/nation.parquet` group by CURRENT_DATE")
.unOrdered()
.baselineColumns("cnt")
.baselineValues(25l)
.build().run();
- final String query3 = "select count(*) as cnt from cp.`employee.json` group by CURRENT_DATE";
testBuilder()
- .sqlQuery(query3)
+ .sqlQuery("select count(*) as cnt from cp.`employee.json` group by CURRENT_DATE")
.unOrdered()
.baselineColumns("cnt")
.baselineValues(1155l)
@@ -493,7 +494,7 @@ public class TestAggregateFunctions extends BaseTestQuery {
@Test
public void test4443() throws Exception {
- test("SELECT MIN(columns[1]) FROM dfs_test.`%s/agg/4443.csv` GROUP BY columns[0]", TEST_RES_PATH);
+ test("SELECT MIN(columns[1]) FROM cp.`agg/4443.csv` GROUP BY columns[0]");
}
@Test
@@ -590,30 +591,21 @@ public class TestAggregateFunctions extends BaseTestQuery {
@Test // DRILL-4264
@Category(UnlikelyTest.class)
public void testCountOnFieldWithDots() throws Exception {
- File directory = new File(BaseTestQuery.getTempDir("json/input"));
- try {
- directory.mkdirs();
- String fileName = "table.json";
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) {
- writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
- }
-
- String query = String.format("select count(t.m.`a.b`) as a,\n" +
- "count(t.m.a.b) as b,\n" +
- "count(t.m['a.b']) as c,\n" +
- "count(t.rk.q) as d,\n" +
- "count(t.`rk.q`) as e\n" +
- "from dfs_test.`%s/%s` t",
- directory.toPath().toString(), fileName);
- testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .baselineColumns("a", "b", "c", "d", "e")
- .baselineValues(1L, 1L, 1L, 0L, 1L)
- .go();
-
- } finally {
- org.apache.commons.io.FileUtils.deleteQuietly(directory);
+ String fileName = "table.json";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), fileName)))) {
+ writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}");
}
+
+ testBuilder()
+ .sqlQuery("select count(t.m.`a.b`) as a,\n" +
+ "count(t.m.a.b) as b,\n" +
+ "count(t.m['a.b']) as c,\n" +
+ "count(t.rk.q) as d,\n" +
+ "count(t.`rk.q`) as e\n" +
+ "from dfs.`%s` t", fileName)
+ .unOrdered()
+ .baselineColumns("a", "b", "c", "d", "e")
+ .baselineValues(1L, 1L, 1L, 0L, 1L)
+ .go();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
index 29cff1f..576dd48 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,10 +18,9 @@
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -30,76 +29,68 @@ import org.junit.experimental.categories.Category;
@Category({UnlikelyTest.class, SqlFunctionTest.class})
public class TestCastEmptyStrings extends BaseTestQuery {
- // enable decimal data type
- @BeforeClass
- public static void enableDecimalDataType() throws Exception {
- test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
- }
-
- @AfterClass
- public static void disableDecimalDataType() throws Exception {
- test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
- }
-
- @Test // see DRILL-1874
- public void testCastInputTypeNullableVarCharToNumeric() throws Exception {
- String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
-
- // Enable the new cast functions (cast empty string "" to null)
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
-
- // Test Optional VarChar
- test(String.format("select cast(columns[0] as int) from dfs_test.`%s`;", root));
- test(String.format("select cast(columns[0] as bigint) from dfs_test.`%s`;", root));
- test(String.format("select cast(columns[0] as float) from dfs_test.`%s`;", root));
- test(String.format("select cast(columns[0] as double) from dfs_test.`%s`;", root));
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
- }
-
- @Test // see DRILL-1874
- public void testCastInputTypeNonNullableVarCharToNumeric() throws Exception {
- String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
-
- // Enable the new cast functions (cast empty string "" to null)
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
- // Test Required VarChar
- test(String.format("select cast('' as int) from dfs_test.`%s`;", root));
- test(String.format("select cast('' as bigint) from dfs_test.`%s`;", root));
- test(String.format("select cast('' as float) from dfs_test.`%s`;", root));
- test(String.format("select cast('' as double) from dfs_test.`%s`;", root));
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
- }
-
- @Test // see DRILL-1874
- public void testCastInputTypeNullableVarCharToDecimal() throws Exception {
- String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
-
- // Enable the new cast functions (cast empty string "" to null)
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
-
- // Test Optional VarChar
- test(String.format("select cast(columns[0] as decimal) from dfs_test.`%s` where cast(columns[0] as decimal) is null;", root));
- test(String.format("select cast(columns[0] as decimal(9)) from dfs_test.`%s`;", root));
- test(String.format("select cast(columns[0] as decimal(18)) from dfs_test.`%s`;", root));
- test(String.format("select cast(columns[0] as decimal(28)) from dfs_test.`%s`;", root));
- test(String.format("select cast(columns[0] as decimal(38)) from dfs_test.`%s`;", root));
-
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
- }
-
- @Test // see DRILL-1874
- public void testCastInputTypeNonNullableVarCharToDecimal() throws Exception {
- String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
-
- // Enable the new cast functions (cast empty string "" to null)
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
-
- // Test Required VarChar
- test(String.format("select cast('' as decimal) from dfs_test.`%s` where cast('' as decimal) is null;", root));
- test(String.format("select cast('' as decimal(18)) from dfs_test.`%s`;", root));
- test(String.format("select cast('' as decimal(28)) from dfs_test.`%s`;", root));
- test(String.format("select cast('' as decimal(38)) from dfs_test.`%s`;", root));
-
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
- }
+ // enable decimal data type
+ @BeforeClass
+ public static void enableDecimalDataType() throws Exception {
+ test("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY);
+ }
+
+ @AfterClass
+ public static void disableDecimalDataType() throws Exception {
+ test("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY);
+ }
+
+ @Test // see DRILL-1874
+ public void testCastInputTypeNullableVarCharToNumeric() throws Exception {
+ // Enable the new cast functions (cast empty string "" to null)
+ test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+
+ // Test Optional VarChar
+ test("select cast(columns[0] as int) from cp.`emptyStrings.csv`");
+ test("select cast(columns[0] as bigint) from cp.`emptyStrings.csv`");
+ test("select cast(columns[0] as float) from cp.`emptyStrings.csv`");
+ test("select cast(columns[0] as double) from cp.`emptyStrings.csv`");
+ test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+ }
+
+ @Test // see DRILL-1874
+ public void testCastInputTypeNonNullableVarCharToNumeric() throws Exception {
+ // Enable the new cast functions (cast empty string "" to null)
+ test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+ // Test Required VarChar
+ test("select cast('' as int) from cp.`emptyStrings.csv`");
+ test("select cast('' as bigint) from cp.`emptyStrings.csv`");
+ test("select cast('' as float) from cp.`emptyStrings.csv`");
+ test("select cast('' as double) from cp.`emptyStrings.csv`");
+ test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+ }
+
+ @Test // see DRILL-1874
+ public void testCastInputTypeNullableVarCharToDecimal() throws Exception {
+ // Enable the new cast functions (cast empty string "" to null)
+ test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+
+ // Test Optional VarChar
+ test("select cast(columns[0] as decimal) from cp.`emptyStrings.csv` where cast(columns[0] as decimal) is null");
+ test("select cast(columns[0] as decimal(9)) from cp.`emptyStrings.csv`");
+ test("select cast(columns[0] as decimal(18)) from cp.`emptyStrings.csv`");
+ test("select cast(columns[0] as decimal(28)) from cp.`emptyStrings.csv`");
+ test("select cast(columns[0] as decimal(38)) from cp.`emptyStrings.csv`");
+
+ test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+ }
+
+ @Test // see DRILL-1874
+ public void testCastInputTypeNonNullableVarCharToDecimal() throws Exception {
+ // Enable the new cast functions (cast empty string "" to null)
+ test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+
+ // Test Required VarChar
+ test("select cast('' as decimal) from cp.`emptyStrings.csv` where cast('' as decimal) is null");
+ test("select cast('' as decimal(18)) from cp.`emptyStrings.csv`");
+ test("select cast('' as decimal(28)) from cp.`emptyStrings.csv`");
+ test("select cast('' as decimal(38)) from cp.`emptyStrings.csv`");
+
+ test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
index 4aeb396..14a49b5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastFunctions.java
@@ -20,10 +20,9 @@ package org.apache.drill.exec.fn.impl;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import mockit.integration.junit4.JMockit;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
-import org.apache.drill.common.util.FileUtils;
import org.joda.time.DateTime;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -48,13 +47,8 @@ public class TestCastFunctions extends BaseTestQuery {
@Test // DRILL-2827
public void testImplicitCastStringToBoolean() throws Exception {
- String boolTable= FileUtils.getResourceAsFile("/store/json/booleanData.json").toURI().toString();
-
- String query = String.format(
- "(select * from dfs_test.`%s` where key = 'true' or key = 'false')", boolTable);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("(select * from cp.`store/json/booleanData.json` where key = 'true' or key = 'false')")
.unOrdered()
.baselineColumns("key")
.baselineValues(true)
@@ -114,18 +108,18 @@ public class TestCastFunctions extends BaseTestQuery {
for (float value : values.keySet()) {
try {
- test("create table dfs_test.tmp.table_with_float as\n" +
+ test("create table dfs.tmp.table_with_float as\n" +
"(select cast(%1$s as float) c1 from (values(1)))", value);
testBuilder()
- .sqlQuery("select cast(c1 as int) col1 from dfs_test.tmp.table_with_float")
+ .sqlQuery("select cast(c1 as int) col1 from dfs.tmp.table_with_float")
.unOrdered()
.baselineColumns("col1")
.baselineValues(values.get(value))
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_float");
+ test("drop table if exists dfs.tmp.table_with_float");
}
}
}
@@ -144,20 +138,20 @@ public class TestCastFunctions extends BaseTestQuery {
for (int value : values) {
try {
- test("create table dfs_test.tmp.table_with_int as\n" +
+ test("create table dfs.tmp.table_with_int as\n" +
"(select cast(%1$s as int) c1 from (values(1)))", value);
testBuilder()
.sqlQuery("select cast(c1 as float) col1,\n" +
"cast(c1 as double) col2\n" +
- "from dfs_test.tmp.table_with_int")
+ "from dfs.tmp.table_with_int")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues((float) value, (double) value)
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_int");
+ test("drop table if exists dfs.tmp.table_with_int");
}
}
}
@@ -183,18 +177,18 @@ public class TestCastFunctions extends BaseTestQuery {
for (float value : values.keySet()) {
try {
- test("create table dfs_test.tmp.table_with_float as\n" +
+ test("create table dfs.tmp.table_with_float as\n" +
"(select cast(%1$s as float) c1 from (values(1)))", value);
testBuilder()
- .sqlQuery("select cast(c1 as bigInt) col1 from dfs_test.tmp.table_with_float")
+ .sqlQuery("select cast(c1 as bigInt) col1 from dfs.tmp.table_with_float")
.unOrdered()
.baselineColumns("col1")
.baselineValues(values.get(value))
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_float");
+ test("drop table if exists dfs.tmp.table_with_float");
}
}
}
@@ -215,20 +209,20 @@ public class TestCastFunctions extends BaseTestQuery {
for (long value : values) {
try {
- test("create table dfs_test.tmp.table_with_bigint as\n" +
+ test("create table dfs.tmp.table_with_bigint as\n" +
"(select cast(%1$s as bigInt) c1 from (values(1)))", value);
testBuilder()
.sqlQuery("select cast(c1 as float) col1,\n" +
"cast(c1 as double) col2\n" +
- "from dfs_test.tmp.table_with_bigint")
+ "from dfs.tmp.table_with_bigint")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues((float) value, (double) value)
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_bigint");
+ test("drop table if exists dfs.tmp.table_with_bigint");
}
}
}
@@ -254,18 +248,18 @@ public class TestCastFunctions extends BaseTestQuery {
for (double value : values.keySet()) {
try {
- test("create table dfs_test.tmp.table_with_double as\n" +
+ test("create table dfs.tmp.table_with_double as\n" +
"(select cast(%1$s as double) c1 from (values(1)))", value);
testBuilder()
- .sqlQuery("select cast(c1 as int) col1 from dfs_test.tmp.table_with_double")
+ .sqlQuery("select cast(c1 as int) col1 from dfs.tmp.table_with_double")
.unOrdered()
.baselineColumns("col1")
.baselineValues(values.get(value))
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_double");
+ test("drop table if exists dfs.tmp.table_with_double");
}
}
}
@@ -291,18 +285,18 @@ public class TestCastFunctions extends BaseTestQuery {
values.put(Double.MIN_VALUE, 0L);
for (double value : values.keySet()) {
try {
- test("create table dfs_test.tmp.table_with_double as\n" +
+ test("create table dfs.tmp.table_with_double as\n" +
"(select cast(%1$s as double) c1 from (values(1)))", value);
testBuilder()
- .sqlQuery("select cast(c1 as bigInt) col1 from dfs_test.tmp.table_with_double")
+ .sqlQuery("select cast(c1 as bigInt) col1 from dfs.tmp.table_with_double")
.unOrdered()
.baselineColumns("col1")
.baselineValues(values.get(value))
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_double");
+ test("drop table if exists dfs.tmp.table_with_double");
}
}
}
@@ -320,20 +314,20 @@ public class TestCastFunctions extends BaseTestQuery {
for (int value : values) {
try {
- test("create table dfs_test.tmp.table_with_int as\n" +
+ test("create table dfs.tmp.table_with_int as\n" +
"(select cast(%1$s as int) c1, cast(%1$s as bigInt) c2 from (values(1)))", value);
testBuilder()
.sqlQuery("select cast(c1 as bigint) col1,\n" +
"cast(c1 as int) col2\n" +
- "from dfs_test.tmp.table_with_int")
+ "from dfs.tmp.table_with_int")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues((long) value, value)
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_int");
+ test("drop table if exists dfs.tmp.table_with_int");
}
}
}
@@ -356,7 +350,7 @@ public class TestCastFunctions extends BaseTestQuery {
for (double value : values) {
try {
- test("create table dfs_test.tmp.table_with_float as\n" +
+ test("create table dfs.tmp.table_with_float as\n" +
"(select cast(%1$s as float) c1,\n" +
"cast(%1$s as double) c2\n" +
"from (values(1)))", value);
@@ -364,14 +358,14 @@ public class TestCastFunctions extends BaseTestQuery {
testBuilder()
.sqlQuery("select cast(c1 as double) col1,\n" +
"cast(c2 as float) col2\n" +
- "from dfs_test.tmp.table_with_float")
+ "from dfs.tmp.table_with_float")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues((double) ((float) (value)), (float) value)
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_float");
+ test("drop table if exists dfs.tmp.table_with_float");
}
}
}
@@ -406,7 +400,7 @@ public class TestCastFunctions extends BaseTestQuery {
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_int");
+ test("drop table if exists dfs.tmp.table_with_int");
test("alter session reset planner.enable_decimal_data_type");
}
}
@@ -429,7 +423,7 @@ public class TestCastFunctions extends BaseTestQuery {
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_int");
+ test("drop table if exists dfs.tmp.table_with_int");
test("alter session reset planner.enable_decimal_data_type");
}
}
@@ -452,7 +446,7 @@ public class TestCastFunctions extends BaseTestQuery {
.build()
.run();
} finally {
- test("drop table if exists dfs_test.tmp.table_with_int");
+ test("drop table if exists dfs.tmp.table_with_int");
test("alter session reset planner.enable_decimal_data_type");
}
}
@@ -460,7 +454,7 @@ public class TestCastFunctions extends BaseTestQuery {
@Test // DRILL-4970
public void testCastNegativeFloatToInt() throws Exception {
try {
- test("create table dfs_test.tmp.table_with_float as\n" +
+ test("create table dfs.tmp.table_with_float as\n" +
"(select cast(-255.0 as double) as double_col,\n" +
"cast(-255.0 as float) as float_col\n" +
"from (values(1)))");
@@ -473,7 +467,7 @@ public class TestCastFunctions extends BaseTestQuery {
castTypes.add("int");
castTypes.add("bigInt");
- final String query = "select count(*) as c from dfs_test.tmp.table_with_float\n" +
+ final String query = "select count(*) as c from dfs.tmp.table_with_float\n" +
"where (cast(%1$s as %2$s) >= -255 and (%1$s <= -5)) or (%1$s <= -256)";
for (String columnName : columnNames) {
@@ -488,7 +482,7 @@ public class TestCastFunctions extends BaseTestQuery {
}
}
} finally {
- test("drop table if exists dfs_test.tmp.table_with_float");
+ test("drop table if exists dfs.tmp.table_with_float");
}
}
@@ -497,7 +491,7 @@ public class TestCastFunctions extends BaseTestQuery {
try {
test("alter session set planner.enable_decimal_data_type = true");
- test("create table dfs_test.tmp.table_with_decimal as" +
+ test("create table dfs.tmp.table_with_decimal as" +
"(select cast(cast(manager_id as double) * (-1) as decimal(9, 0)) as decimal9_col,\n" +
"cast(cast(manager_id as double) * (-1) as decimal(18, 0)) as decimal18_col\n" +
"from cp.`parquet/fixedlenDecimal.parquet` limit 1)");
@@ -506,7 +500,7 @@ public class TestCastFunctions extends BaseTestQuery {
columnNames.add("decimal9_col");
columnNames.add("decimal18_col");
- final String query = "select count(*) as c from dfs_test.tmp.table_with_decimal\n" +
+ final String query = "select count(*) as c from dfs.tmp.table_with_decimal\n" +
"where (cast(%1$s as varchar) = '-124' and (%1$s <= -5)) or (%1$s <= -256)";
for (String colName : columnNames) {
@@ -519,7 +513,7 @@ public class TestCastFunctions extends BaseTestQuery {
.run();
}
} finally {
- test("drop table if exists dfs_test.tmp.table_with_decimal");
+ test("drop table if exists dfs.tmp.table_with_decimal");
test("alter session reset planner.enable_decimal_data_type");
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
index 6f4bd29..2e39966 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestContextFunctions.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -75,22 +75,22 @@ public class TestContextFunctions extends BaseTestQuery {
@Test
public void currentSchemaUDFWithSingleLevelDefaultSchema() throws Exception {
testBuilder()
- .optionSettingQueriesForTestQuery("USE dfs_test")
+ .optionSettingQueriesForTestQuery("USE dfs")
.sqlQuery("select current_schema from cp.`employee.json` limit 1")
.unOrdered()
.baselineColumns("current_schema")
- .baselineValues("dfs_test")
+ .baselineValues("dfs")
.go();
}
@Test
public void currentSchemaUDFWithMultiLevelDefaultSchema() throws Exception {
testBuilder()
- .optionSettingQueriesForTestQuery("USE dfs_test.tmp")
+ .optionSettingQueriesForTestQuery("USE dfs.tmp")
.sqlQuery("select current_schema from cp.`employee.json` limit 1")
.unOrdered()
.baselineColumns("current_schema")
- .baselineValues("dfs_test.tmp")
+ .baselineValues("dfs.tmp")
.go();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCryptoFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCryptoFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCryptoFunctions.java
index 5cfdf84..77b4fb7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCryptoFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCryptoFunctions.java
@@ -17,7 +17,7 @@
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateAddFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateAddFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateAddFunctions.java
index 8d3c31a..0c8f6d2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateAddFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateAddFunctions.java
@@ -16,7 +16,7 @@
*/
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.joda.time.DateTime;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
index 622d8f4..b1adcc9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateFunctions.java
@@ -22,7 +22,7 @@ import com.google.common.io.Files;
import mockit.integration.junit4.JMockit;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -45,19 +45,16 @@ import static org.junit.Assert.assertTrue;
@RunWith(JMockit.class)
@Category({UnlikelyTest.class, SqlFunctionTest.class})
public class TestDateFunctions extends PopUnitTestBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestDateFunctions.class);
-
public void testCommon(String[] expectedResults, String physicalPlan, String resourceFile) throws Exception {
try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
Drillbit bit = new Drillbit(CONFIG, serviceSet);
DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
-
// run query.
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile(physicalPlan), Charsets.UTF_8)
+ Files.toString(DrillFileUtils.getResourceAsFile(physicalPlan), Charsets.UTF_8)
.replace("#{TEST_FILE}", resourceFile));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateTruncFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateTruncFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateTruncFunctions.java
index c62dc65..9af4e52 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateTruncFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestDateTruncFunctions.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.joda.time.DateTime;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
index 172f35c..5a85319 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java
@@ -25,7 +25,7 @@ import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -59,7 +59,7 @@ public class TestMathFunctions extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/simple_math_functions.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/simple_math_functions.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMultiInputAdd.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMultiInputAdd.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMultiInputAdd.java
index 02aeb0a..237b01a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMultiInputAdd.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMultiInputAdd.java
@@ -26,7 +26,7 @@ import mockit.Injectable;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -62,7 +62,7 @@ public class TestMultiInputAdd extends PopUnitTestBase {
bit.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/functions/multi_input_add_test.json"), Charsets.UTF_8));
+ Files.toString(DrillFileUtils.getResourceAsFile("/functions/multi_input_add_test.json"), Charsets.UTF_8));
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNetworkFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNetworkFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNetworkFunctions.java
index 5beb4d5..737dace 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNetworkFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNetworkFunctions.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
public class TestNetworkFunctions extends BaseTestQuery {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewAggregateFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewAggregateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewAggregateFunctions.java
index 00b0b36..cf5df1f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewAggregateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewAggregateFunctions.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
@@ -55,7 +55,7 @@ public class TestNewAggregateFunctions extends PopUnitTestBase {
client.connect();
List<QueryDataBatch> results = client.runQuery(
QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile(physicalPlan),
+ Files.toString(DrillFileUtils.getResourceAsFile(physicalPlan),
Charsets.UTF_8).replace("#{TEST_FILE}",
inputDataFile));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java
index e339117..238b048 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewDateFunctions.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.joda.time.DateTime;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java
index 9f37312..6b0a685 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewSimpleRepeatedFunctions.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
index c4aabe9..fedd30a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java
@@ -24,7 +24,7 @@ import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -58,7 +58,7 @@ public class TestRepeatedFunction extends ExecTest{
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/physical_repeated_1.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/physical_repeated_1.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestTrigFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestTrigFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestTrigFunctions.java
index 60ca587..f9bbd29 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestTrigFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestTrigFunctions.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
index 4da6db3..cc13039 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
@@ -17,11 +17,10 @@
package org.apache.drill.exec.fn.impl.testing;
import mockit.integration.junit4.JMockit;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserException;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -36,26 +35,19 @@ import static org.junit.Assert.assertThat;
@RunWith(JMockit.class)
@Category({UnlikelyTest.class, SqlFunctionTest.class})
public class TestDateConversions extends BaseTestQuery {
-
- private static String TEMP_DIR;
-
@BeforeClass
public static void generateTestFiles() throws IOException {
- File path = new File(BaseTestQuery.getTempDir("json/input"));
- path.mkdirs();
- TEMP_DIR = path.toPath().toString();
-
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(path, "joda_postgres_date.json")))) {
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "joda_postgres_date.json")))) {
writer.write("{\"date1\" : \"1970-01-02\",\n \"date2\" : \"01021970\",\n \"date3\" : \"32/1970\"\n}\n"
+ "{\"date1\" : \"2010-05-03\",\n \"date2\" : \"01021970\",\n \"date3\" : \"64/2010\"\n}");
}
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(path, "joda_postgres_time.json")))) {
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "joda_postgres_time.json")))) {
writer.write("{\"time1\" : \"23:11:59\",\n \"time2\" : \"11:11:59pm\",\n \"time3\" : \"591111pm\"\n}\n"
+ "{\"time1\" : \"17:33:41\",\n \"time2\" : \"5:33:41am\",\n \"time3\" : \"413305pm\"\n}");
}
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(path, "joda_postgres_date_time.json")))) {
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "joda_postgres_date_time.json")))) {
writer.write("{ \"time1\" : \"1970-01-0223:11:59\",\n \"time2\" : \"0102197011:11:59pm\",\n"
+ " \"time3\" : \"32/1970591111pm\"\n}\n"
+ "{\"time1\" : \"2010-05-0317:33:41\",\n \"time2\" : \"0102197005:33:41am\",\n"
@@ -63,22 +55,13 @@ public class TestDateConversions extends BaseTestQuery {
}
}
- @AfterClass
- public static void deleteTestFiles() throws IOException {
- java.nio.file.Files.delete(new File(TEMP_DIR, "joda_postgres_date.json").toPath());
- java.nio.file.Files.delete(new File(TEMP_DIR, "joda_postgres_time.json").toPath());
- java.nio.file.Files.delete(new File(TEMP_DIR, "joda_postgres_date_time.json").toPath());
- }
-
@Test
public void testJodaDate() throws Exception {
- String query = String.format("SELECT to_date(date1, 'yyyy-dd-MM') = "
- + "to_date(date2, 'ddMMyyyy') as col1, " + "to_date(date1, 'yyyy-dd-MM') = "
- + "to_date(date3, 'D/yyyy') as col2 "
- + "from dfs_test.`%s/joda_postgres_date.json`", TEMP_DIR);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT to_date(date1, 'yyyy-dd-MM') = "
+ + "to_date(date2, 'ddMMyyyy') as col1, " + "to_date(date1, 'yyyy-dd-MM') = "
+ + "to_date(date3, 'D/yyyy') as col2 "
+ + "from dfs.`joda_postgres_date.json`")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues(true, true)
@@ -88,14 +71,12 @@ public class TestDateConversions extends BaseTestQuery {
@Test
public void testPostgresDate() throws Exception {
- String query = String.format("SELECT sql_to_date(date1, 'yyyy-DD-MM') = "
- + "sql_to_date(date2, 'DDMMyyyy') as col1, "
- + "sql_to_date(date1, 'yyyy-DD-MM') = "
- + "sql_to_date(date3, 'DDD/yyyy') as col2 "
- + "from dfs_test.`%s/joda_postgres_date.json`", TEMP_DIR);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT sql_to_date(date1, 'yyyy-DD-MM') = "
+ + "sql_to_date(date2, 'DDMMyyyy') as col1, "
+ + "sql_to_date(date1, 'yyyy-DD-MM') = "
+ + "sql_to_date(date3, 'DDD/yyyy') as col2 "
+ + "from dfs.`joda_postgres_date.json`")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues(true, true)
@@ -107,14 +88,12 @@ public class TestDateConversions extends BaseTestQuery {
public void testJodaTime() throws Exception {
mockUsDateFormatSymbols();
- String query = String.format("SELECT to_time(time1, 'H:m:ss') = "
- + "to_time(time2, 'h:m:ssa') as col1, "
- + "to_time(time1, 'H:m:ss') = "
- + "to_time(time3, 'ssmha') as col2 "
- + "from dfs_test.`%s/joda_postgres_time.json`", TEMP_DIR);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT to_time(time1, 'H:m:ss') = "
+ + "to_time(time2, 'h:m:ssa') as col1, "
+ + "to_time(time1, 'H:m:ss') = "
+ + "to_time(time3, 'ssmha') as col2 "
+ + "from dfs.`joda_postgres_time.json`")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues(true, true)
@@ -126,14 +105,12 @@ public class TestDateConversions extends BaseTestQuery {
public void testPostgresTime() throws Exception {
mockUsDateFormatSymbols();
- String query = String.format("SELECT sql_to_time(time1, 'HH24:MI:SS') = "
- + "sql_to_time(time2, 'HH12:MI:SSam') as col1, "
- + "sql_to_time(time1, 'HH24:MI:SS') = "
- + "sql_to_time(time3, 'SSMIHH12am') as col2 "
- + "from dfs_test.`%s/joda_postgres_time.json`", TEMP_DIR);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT sql_to_time(time1, 'HH24:MI:SS') = "
+ + "sql_to_time(time2, 'HH12:MI:SSam') as col1, "
+ + "sql_to_time(time1, 'HH24:MI:SS') = "
+ + "sql_to_time(time3, 'SSMIHH12am') as col2 "
+ + "from dfs.`joda_postgres_time.json`")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues(true, true)
@@ -145,14 +122,12 @@ public class TestDateConversions extends BaseTestQuery {
public void testPostgresDateTime() throws Exception {
mockUsDateFormatSymbols();
- String query = String.format("SELECT sql_to_timestamp(time1, 'yyyy-DD-MMHH24:MI:SS') = "
- + "sql_to_timestamp(time2, 'DDMMyyyyHH12:MI:SSam') as col1, "
- + "sql_to_timestamp(time1, 'yyyy-DD-MMHH24:MI:SS') = "
- + "sql_to_timestamp(time3, 'DDD/yyyySSMIHH12am') as col2 "
- + "from dfs_test.`%s/joda_postgres_date_time.json`", TEMP_DIR);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT sql_to_timestamp(time1, 'yyyy-DD-MMHH24:MI:SS') = "
+ + "sql_to_timestamp(time2, 'DDMMyyyyHH12:MI:SSam') as col1, "
+ + "sql_to_timestamp(time1, 'yyyy-DD-MMHH24:MI:SS') = "
+ + "sql_to_timestamp(time3, 'DDD/yyyySSMIHH12am') as col2 "
+ + "from dfs.`joda_postgres_date_time.json`")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues(true, true)
@@ -164,14 +139,12 @@ public class TestDateConversions extends BaseTestQuery {
public void testJodaDateTime() throws Exception {
mockUsDateFormatSymbols();
- String query = String.format("SELECT to_timestamp(time1, 'yyyy-dd-MMH:m:ss') = "
- + "to_timestamp(time2, 'ddMMyyyyh:m:ssa') as col1, "
- + "to_timestamp(time1, 'yyyy-dd-MMH:m:ss') = "
- + "to_timestamp(time3, 'DDD/yyyyssmha') as col2 "
- + "from dfs_test.`%s/joda_postgres_date_time.json`", TEMP_DIR);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT to_timestamp(time1, 'yyyy-dd-MMH:m:ss') = "
+ + "to_timestamp(time2, 'ddMMyyyyh:m:ssa') as col1, "
+ + "to_timestamp(time1, 'yyyy-dd-MMH:m:ss') = "
+ + "to_timestamp(time3, 'DDD/yyyyssmha') as col2 "
+ + "from dfs.`joda_postgres_date_time.json`")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues(true, true)
@@ -183,14 +156,12 @@ public class TestDateConversions extends BaseTestQuery {
public void testJodaDateTimeNested() throws Exception {
mockUsDateFormatSymbols();
- String query = String.format("SELECT date_add(to_date(time1, concat('yyyy-dd-MM','H:m:ss')), 22)= "
- + "date_add(to_date(time2, concat('ddMMyyyy', 'h:m:ssa')), 22) as col1, "
- + "date_add(to_date(time1, concat('yyyy-dd-MM', 'H:m:ss')), 22) = "
- + "date_add(to_date(time3, concat('DDD/yyyy', 'ssmha')), 22) as col2 "
- + "from dfs_test.`%s/joda_postgres_date_time.json`", TEMP_DIR);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT date_add(to_date(time1, concat('yyyy-dd-MM','H:m:ss')), 22)= "
+ + "date_add(to_date(time2, concat('ddMMyyyy', 'h:m:ssa')), 22) as col1, "
+ + "date_add(to_date(time1, concat('yyyy-dd-MM', 'H:m:ss')), 22) = "
+ + "date_add(to_date(time3, concat('DDD/yyyy', 'ssmha')), 22) as col2 "
+ + "from dfs.`joda_postgres_date_time.json`")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues(true, true)
@@ -202,14 +173,12 @@ public class TestDateConversions extends BaseTestQuery {
public void testPostgresDateTimeNested() throws Exception {
mockUsDateFormatSymbols();
- String query = String.format("SELECT date_add(sql_to_date(time1, concat('yyyy-DD-MM', 'HH24:MI:SS')), 22) = "
- + "date_add(sql_to_date(time2, concat('DDMMyyyy', 'HH12:MI:SSam')), 22) as col1, "
- + "date_add(sql_to_date(time1, concat('yyyy-DD-MM', 'HH24:MI:SS')), 10) = "
- + "date_add(sql_to_date(time3, concat('DDD/yyyySSMI', 'HH12am')), 10) as col2 "
- + "from dfs_test.`%s/joda_postgres_date_time.json`", TEMP_DIR);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("SELECT date_add(sql_to_date(time1, concat('yyyy-DD-MM', 'HH24:MI:SS')), 22) = "
+ + "date_add(sql_to_date(time2, concat('DDMMyyyy', 'HH12:MI:SSam')), 22) as col1, "
+ + "date_add(sql_to_date(time1, concat('yyyy-DD-MM', 'HH24:MI:SS')), 10) = "
+ + "date_add(sql_to_date(time3, concat('DDD/yyyySSMI', 'HH12am')), 10) as col2 "
+ + "from dfs.`joda_postgres_date_time.json`")
.unOrdered()
.baselineColumns("col1", "col2")
.baselineValues(true, true)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
index 206bf97..0cf62bd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
@@ -37,22 +37,14 @@ import java.util.List;
@Category(SqlTest.class)
public class TestConstantFolding extends PlanTestBase {
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- // This should run as a @BeforeClass, but these methods must be defined static.
- // Unfortunately, the temporary folder with an @Rule annotation cannot be static, this issue
- // has been fixed in a newer version of JUnit
- // http://stackoverflow.com/questions/2722358/junit-rule-temporaryfolder
-
public static class SmallFileCreator {
- private final TemporaryFolder folder;
+ private final File folder;
private static final List<String> values = Lists.newArrayList("1","2","3");
private static final String jsonRecord = "{\"col1\" : 1,\"col2\" : 2, \"col3\" : 3}";
private String record;
- public SmallFileCreator(TemporaryFolder folder) {
+ public SmallFileCreator(File folder) {
this.folder = folder;
this.record = null;
}
@@ -76,7 +68,8 @@ public class TestConstantFolding extends PlanTestBase {
}
PrintWriter out;
for (String fileAndFolderName : new String[]{"bigfile", "BIGFILE_2"}) {
- File bigFolder = folder.newFolder(fileAndFolderName);
+ File bigFolder = new File(folder, fileAndFolderName);
+ bigFolder.mkdirs();
File bigFile = new File (bigFolder, fileAndFolderName + "." + extension);
out = new PrintWriter(bigFile);
for (int i = 0; i < bigFileLines; i++ ) {
@@ -86,7 +79,8 @@ public class TestConstantFolding extends PlanTestBase {
}
for (String fileAndFolderName : new String[]{"smallfile", "SMALLFILE_2"}) {
- File smallFolder = folder.newFolder(fileAndFolderName);
+ File smallFolder = new File(folder, fileAndFolderName);
+ smallFolder.mkdirs();
File smallFile = new File (smallFolder, fileAndFolderName + "." + extension);
out = new PrintWriter(smallFile);
for (int i = 0; i < smallFileLines; i++ ) {
@@ -125,7 +119,7 @@ public class TestConstantFolding extends PlanTestBase {
test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
String query2 = "SELECT * " +
- "FROM cp.`/parquet/alltypes.json` " +
+ "FROM cp.`parquet/alltypes.json` " +
"WHERE 12 = extract(day from (to_timestamp('2014-02-12 03:18:31:07 AM', 'YYYY-MM-dd HH:mm:ss:SS a'))) " +
"AND cast( `int_col` AS int) = castint('1') " +
"AND cast( `bigint_col` AS bigint) = castbigint('100000000000') " +
@@ -144,10 +138,7 @@ public class TestConstantFolding extends PlanTestBase {
"AND cast( `intervalday_col` AS interval day) = castintervalday('P1D')" +
"AND cast( `bit_col` AS boolean) = castbit('false') " +
"AND `varchar_col` = concat('qwe','rty') " +
-
- "AND cast( `time_col` AS time) = casttime('01:00:00') " +
-
- "";
+ "AND cast( `time_col` AS time) = casttime('01:00:00')";
testBuilder()
@@ -164,8 +155,7 @@ public class TestConstantFolding extends PlanTestBase {
.baselineValues(
"1", "1", "1", "1", "01:00:00", "1.0", "100000000000", "1", "1", "1995-01-01", "1995-01-01 01:00:10.000",
"123456789.000000000", "P1Y", "P1D", "P1Y1M1DT1H1M", "123456789.000000000",
- "123456789.000000000", "qwerty", "qwerty","qwerty", "false"
- )
+ "123456789.000000000", "qwerty", "qwerty","qwerty", "false")
.go();
} finally {
test("alter session set `store.json.all_text_mode` = false;");
@@ -176,20 +166,18 @@ public class TestConstantFolding extends PlanTestBase {
@Ignore("DRILL-2553")
@Test
public void testConstExprFolding_withPartitionPrune_verySmallFiles() throws Exception {
- new SmallFileCreator(folder).createFiles(1, 8);
- String path = folder.getRoot().toPath().toString();
+ new SmallFileCreator(dirTestWatcher.getRootDir()).createFiles(1, 8);
testPlanOneExpectedPatternOneExcluded(
- "select * from dfs.`" + path + "/*/*.csv` where dir0 = concat('small','file')",
+ "select * from dfs.`*/*.csv` where dir0 = concat('small','file')",
"smallfile",
"bigfile");
}
@Test
public void testConstExprFolding_withPartitionPrune() throws Exception {
- new SmallFileCreator(folder).createFiles(1, 1000);
- String path = folder.getRoot().toPath().toString();
+ new SmallFileCreator(dirTestWatcher.getRootDir()).createFiles(1, 1000);
testPlanOneExpectedPatternOneExcluded(
- "select * from dfs.`" + path + "/*/*.csv` where dir0 = concat('small','file')",
+ "select * from dfs.`*/*.csv` where dir0 = concat('small','file')",
"smallfile",
"bigfile");
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java
index 7fc5cea..b2671be 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/BaseTestImpersonation.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.security.UserGroupInformation;
import java.io.File;
+import java.nio.file.Paths;
import java.util.Map;
import java.util.Properties;
@@ -49,7 +50,7 @@ public class BaseTestImpersonation extends PlanTestBase {
protected static MiniDFSCluster dfsCluster;
protected static Configuration dfsConf;
protected static FileSystem fs;
- protected static String miniDfsStoragePath;
+ protected static File miniDfsStoragePath;
// Test users and groups
protected static final String[] org1Users = { "user0_1", "user1_1", "user2_1", "user3_1", "user4_1", "user5_1" };
@@ -93,8 +94,8 @@ public class BaseTestImpersonation extends PlanTestBase {
// Set the MiniDfs base dir to be the temp directory of the test, so that all files created within the MiniDfs
// are properly cleanup when test exits.
- miniDfsStoragePath = System.getProperty("java.io.tmpdir") + Path.SEPARATOR + testClass;
- dfsConf.set("hdfs.minidfs.basedir", miniDfsStoragePath);
+ miniDfsStoragePath = dirTestWatcher.makeRootSubDir(Paths.get("miniDfs"));
+ dfsConf.set("hdfs.minidfs.basedir", miniDfsStoragePath.getCanonicalPath());
if (isImpersonationEnabled) {
// Set the proxyuser settings so that the user who is running the Drillbits/MiniDfs can impersonate other users.
@@ -126,7 +127,7 @@ public class BaseTestImpersonation extends PlanTestBase {
// Create a HDFS based storage plugin based on local storage plugin and add it to plugin registry (connection string
// for mini dfs is varies for each run).
final StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
- final FileSystemConfig lfsPluginConfig = (FileSystemConfig) pluginRegistry.getPlugin("dfs_test").getConfig();
+ final FileSystemConfig lfsPluginConfig = (FileSystemConfig) pluginRegistry.getPlugin("dfs").getConfig();
final FileSystemConfig miniDfsPluginConfig = new FileSystemConfig();
miniDfsPluginConfig.connection = dfsConf.get(FileSystem.FS_DEFAULT_NAME_KEY);
@@ -156,7 +157,7 @@ public class BaseTestImpersonation extends PlanTestBase {
}
if (miniDfsStoragePath != null) {
- FileUtils.deleteQuietly(new File(miniDfsStoragePath));
+ FileUtils.deleteQuietly(miniDfsStoragePath);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationDisabledWithMiniDFS.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationDisabledWithMiniDFS.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationDisabledWithMiniDFS.java
index 767c9ed..702d14c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationDisabledWithMiniDFS.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationDisabledWithMiniDFS.java
@@ -52,14 +52,14 @@ public class TestImpersonationDisabledWithMiniDFS extends BaseTestImpersonation
// page of data all at once, see notes above testReadLargeParquetFileFromDFS()
test(String.format(
"CREATE TABLE %s.tmp.large_employee AS " +
- "(SELECT employee_id, full_name FROM cp.`/employee.json`) " +
- "UNION ALL (SELECT employee_id, full_name FROM cp.`/employee.json`)" +
- "UNION ALL (SELECT employee_id, full_name FROM cp.`/employee.json`)" +
- "UNION ALL (SELECT employee_id, full_name FROM cp.`/employee.json`)" +
- "UNION ALL (SELECT employee_id, full_name FROM cp.`/employee.json`)" +
- "UNION ALL (SELECT employee_id, full_name FROM cp.`/employee.json`)" +
- "UNION ALL (SELECT employee_id, full_name FROM cp.`/employee.json`)" +
- "UNION ALL (SELECT employee_id, full_name FROM cp.`/employee.json`)",
+ "(SELECT employee_id, full_name FROM cp.`employee.json`) " +
+ "UNION ALL (SELECT employee_id, full_name FROM cp.`employee.json`)" +
+ "UNION ALL (SELECT employee_id, full_name FROM cp.`employee.json`)" +
+ "UNION ALL (SELECT employee_id, full_name FROM cp.`employee.json`)" +
+ "UNION ALL (SELECT employee_id, full_name FROM cp.`employee.json`)" +
+ "UNION ALL (SELECT employee_id, full_name FROM cp.`employee.json`)" +
+ "UNION ALL (SELECT employee_id, full_name FROM cp.`employee.json`)" +
+ "UNION ALL (SELECT employee_id, full_name FROM cp.`employee.json`)",
MINIDFS_STORAGE_PLUGIN_NAME));
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
index 60baed9..d023610 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationMetadata.java
@@ -105,17 +105,17 @@ public class TestImpersonationMetadata extends BaseTestImpersonation {
// create tables as user2
updateClient(user2);
- test(String.format("use `%s.user2_workspace1`", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("use `%s.user2_workspace1`", MINIDFS_STORAGE_PLUGIN_NAME);
// create a table that can be dropped by another user in a different group
test("create table parquet_table_775 as select * from cp.`employee.json`");
// create a table that cannot be dropped by another user
- test(String.format("use `%s.user2_workspace2`", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("use `%s.user2_workspace2`", MINIDFS_STORAGE_PLUGIN_NAME);
test("create table parquet_table_700 as select * from cp.`employee.json`");
// Drop tables as user1
updateClient(user1);
- test(String.format("use `%s.user2_workspace1`", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("use `%s.user2_workspace1`", MINIDFS_STORAGE_PLUGIN_NAME);
testBuilder()
.sqlQuery("drop table parquet_table_775")
.unOrdered()
@@ -123,7 +123,7 @@ public class TestImpersonationMetadata extends BaseTestImpersonation {
.baselineValues(true, String.format("Table [%s] dropped", "parquet_table_775"))
.go();
- test(String.format("use `%s.user2_workspace2`", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("use `%s.user2_workspace2`", MINIDFS_STORAGE_PLUGIN_NAME);
boolean dropFailed = false;
try {
test("drop table parquet_table_700");
@@ -154,11 +154,11 @@ public class TestImpersonationMetadata extends BaseTestImpersonation {
updateClient(user1);
// Try show tables in schema "drillTestGrp1_700" which is owned by "user1"
- test(String.format("SHOW FILES IN %s.drillTestGrp1_700", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("SHOW FILES IN %s.drillTestGrp1_700", MINIDFS_STORAGE_PLUGIN_NAME);
// Try show tables in schema "drillTestGrp0_750" which is owned by "processUser" and has group permissions for
// "user1"
- test(String.format("SHOW FILES IN %s.drillTestGrp0_750", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("SHOW FILES IN %s.drillTestGrp0_750", MINIDFS_STORAGE_PLUGIN_NAME);
}
@Test
@@ -166,7 +166,7 @@ public class TestImpersonationMetadata extends BaseTestImpersonation {
updateClient(user2);
// Try show tables in schema "drillTestGrp0_755" which is owned by "processUser" and group0. "user2" is not part
// of the "group0"
- test(String.format("SHOW FILES IN %s.drillTestGrp0_755", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("SHOW FILES IN %s.drillTestGrp0_755", MINIDFS_STORAGE_PLUGIN_NAME);
}
@Test
@@ -176,7 +176,7 @@ public class TestImpersonationMetadata extends BaseTestImpersonation {
updateClient(user2);
try {
// Try show tables in schema "drillTestGrp1_700" which is owned by "user1"
- test(String.format("SHOW FILES IN %s.drillTestGrp1_700", MINIDFS_STORAGE_PLUGIN_NAME));
+ test("SHOW FILES IN %s.drillTestGrp1_700", MINIDFS_STORAGE_PLUGIN_NAME);
} catch(UserRemoteException e) {
ex = e;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationQueries.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationQueries.java
index 88b2ad4..b3977e9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestImpersonationQueries.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.impersonation;
import com.google.common.collect.Maps;
import org.apache.drill.categories.SecurityTest;
import org.apache.drill.common.exceptions.UserRemoteException;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.store.avro.AvroTestUtil;
import org.apache.drill.exec.store.dfs.WorkspaceConfig;
import org.apache.drill.categories.SlowTest;
@@ -97,7 +97,7 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
private static void createTestTable(String user, String group, String tableName) throws Exception {
updateClient(user);
test("USE " + getWSSchema(user));
- test(String.format("CREATE TABLE %s as SELECT * FROM cp.`tpch/%s.parquet`;", tableName, tableName));
+ test("CREATE TABLE %s as SELECT * FROM cp.`tpch/%s.parquet`", tableName, tableName);
// Change the ownership and permissions manually. Currently there is no option to specify the default permissions
// and ownership for new tables.
@@ -162,7 +162,7 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
private static void createRecordReadersData(String user, String group) throws Exception {
// copy sequence file
updateClient(user);
- Path localFile = new Path(FileUtils.getResourceAsFile("/sequencefiles/simple.seq").toURI().toString());
+ Path localFile = new Path(DrillFileUtils.getResourceAsFile("/sequencefiles/simple.seq").toURI().toString());
Path dfsFile = new Path(getUserHome(user), "simple.seq");
fs.copyFromLocalFile(localFile, dfsFile);
fs.setOwner(dfsFile, user, group);
@@ -180,7 +180,7 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
// Table lineitem is owned by "user0_1:group0_1" with permissions 750. Try to read the table as "user0_1". We
// shouldn't expect any errors.
updateClient(org1Users[0]);
- test(String.format("SELECT * FROM %s.lineitem ORDER BY l_orderkey LIMIT 1", getWSSchema(org1Users[0])));
+ test("SELECT * FROM %s.lineitem ORDER BY l_orderkey LIMIT 1", getWSSchema(org1Users[0]));
}
@Test
@@ -188,7 +188,7 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
// Table lineitem is owned by "user0_1:group0_1" with permissions 750. Try to read the table as "user1_1". We
// shouldn't expect any errors as "user1_1" is part of the "group0_1"
updateClient(org1Users[1]);
- test(String.format("SELECT * FROM %s.lineitem ORDER BY l_orderkey LIMIT 1", getWSSchema(org1Users[0])));
+ test("SELECT * FROM %s.lineitem ORDER BY l_orderkey LIMIT 1", getWSSchema(org1Users[0]));
}
@Test
@@ -198,7 +198,7 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
// Table lineitem is owned by "user0_1:group0_1" with permissions 750. Now try to read the table as "user2_1". We
// should expect a permission denied error as "user2_1" is not part of the "group0_1"
updateClient(org1Users[2]);
- test(String.format("SELECT * FROM %s.lineitem ORDER BY l_orderkey LIMIT 1", getWSSchema(org1Users[0])));
+ test("SELECT * FROM %s.lineitem ORDER BY l_orderkey LIMIT 1", getWSSchema(org1Users[0]));
} catch(UserRemoteException e) {
ex = e;
}
@@ -212,7 +212,7 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
@Test
public void testMultiLevelImpersonationEqualToMaxUserHops() throws Exception {
updateClient(org1Users[4]);
- test(String.format("SELECT * from %s.u4_lineitem LIMIT 1;", getWSSchema(org1Users[4])));
+ test("SELECT * from %s.u4_lineitem LIMIT 1;", getWSSchema(org1Users[4]));
}
@Test
@@ -221,7 +221,7 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
try {
updateClient(org1Users[5]);
- test(String.format("SELECT * from %s.u4_lineitem LIMIT 1;", getWSSchema(org1Users[4])));
+ test("SELECT * from %s.u4_lineitem LIMIT 1;", getWSSchema(org1Users[4]));
} catch (UserRemoteException e) {
ex = e;
}
@@ -235,8 +235,8 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
@Test
public void testMultiLevelImpersonationJoinEachSideReachesMaxUserHops() throws Exception {
updateClient(org1Users[4]);
- test(String.format("SELECT * from %s.u4_lineitem l JOIN %s.u3_orders o ON l.l_orderkey = o.o_orderkey LIMIT 1;",
- getWSSchema(org1Users[4]), getWSSchema(org2Users[3])));
+ test("SELECT * from %s.u4_lineitem l JOIN %s.u3_orders o ON l.l_orderkey = o.o_orderkey LIMIT 1",
+ getWSSchema(org1Users[4]), getWSSchema(org2Users[3]));
}
@Test
@@ -245,8 +245,8 @@ public class TestImpersonationQueries extends BaseTestImpersonation {
try {
updateClient(org1Users[4]);
- test(String.format("SELECT * from %s.u4_lineitem l JOIN %s.u4_orders o ON l.l_orderkey = o.o_orderkey LIMIT 1;",
- getWSSchema(org1Users[4]), getWSSchema(org2Users[4])));
+ test("SELECT * from %s.u4_lineitem l JOIN %s.u4_orders o ON l.l_orderkey = o.o_orderkey LIMIT 1",
+ getWSSchema(org1Users[4]), getWSSchema(org2Users[4]));
} catch(UserRemoteException e) {
ex = e;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonation.java
index 7537431..ee57ad9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonation.java
@@ -90,7 +90,7 @@ public class TestInboundImpersonation extends BaseTestImpersonation {
final String tableName = "lineitem";
updateClient(OWNER, OWNER_PASSWORD);
test("USE " + getWSSchema(OWNER));
- test(String.format("CREATE TABLE %s as SELECT * FROM cp.`tpch/%s.parquet`;", tableName, tableName));
+ test("CREATE TABLE %s as SELECT * FROM cp.`tpch/%s.parquet`", tableName, tableName);
// Change the ownership and permissions manually.
// Currently there is no option to specify the default permissions and ownership for new tables.
@@ -101,9 +101,9 @@ public class TestInboundImpersonation extends BaseTestImpersonation {
// Create a view on top of lineitem table; allow IMPERSONATION_TARGET to read the view
// /user/user0_1 u0_lineitem 750 user0_1:group0_1
final String viewName = "u0_lineitem";
- test(String.format("ALTER SESSION SET `%s`='%o';", ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY, (short) 0750));
- test(String.format("CREATE VIEW %s.%s AS SELECT l_orderkey, l_partkey FROM %s.%s;",
- getWSSchema(OWNER), viewName, getWSSchema(OWNER), "lineitem"));
+ test("ALTER SESSION SET `%s`='%o';", ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY, (short) 0750);
+ test("CREATE VIEW %s.%s AS SELECT l_orderkey, l_partkey FROM %s.%s",
+ getWSSchema(OWNER), viewName, getWSSchema(OWNER), "lineitem");
// Verify the view file created has the expected permissions and ownership
final Path viewFilePath = new Path(getUserHome(OWNER), viewName + DotDrillType.VIEW.getEnding());
final FileStatus status = fs.getFileStatus(viewFilePath);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
index e1d2dfc..199b1e1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.impersonation;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.drill.categories.SecurityTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.rpc.user.InboundImpersonationManager;
import org.apache.drill.exec.server.options.OptionDefinition;
@@ -43,7 +43,7 @@ public class TestInboundImpersonationPrivileges extends BaseTestImpersonation {
static {
try {
- IMPERSONATION_POLICIES = Files.toString(FileUtils.getResourceAsFile("/inbound_impersonation_policies.json"),
+ IMPERSONATION_POLICIES = Files.toString(DrillFileUtils.getResourceAsFile("/inbound_impersonation_policies.json"),
Charsets.UTF_8);
} catch (final IOException e) {
throw new RuntimeException("Cannot load impersonation policies.", e);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
index 6fca84d..e903166 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java
@@ -30,7 +30,7 @@ import java.util.Properties;
import org.apache.drill.categories.MemoryTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -204,7 +204,7 @@ public class TestAllocators extends DrillTest {
// Get a few physical operators. Easiest way is to read a physical plan.
PhysicalPlanReader planReader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(bitContext,
storageRegistry);
- PhysicalPlan plan = planReader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(planFile),
+ PhysicalPlan plan = planReader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile(planFile),
Charsets.UTF_8));
List<PhysicalOperator> physicalOperators = plan.getSortedOperators();
Iterator<PhysicalOperator> physicalOperatorIterator = physicalOperators.iterator();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
index 8f6ab6e..6173bb3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.nested;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java
index d0d0016..1754fab 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.nested;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.UnlikelyTest;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java
index a5f7bdc..fca9cb4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.opt;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.LogicalPlanPersistence;
import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory;
import org.junit.Test;
@@ -31,7 +31,7 @@ public class BasicOptimizerTest extends ExecTest {
public void parseSimplePlan() throws Exception{
DrillConfig c = DrillConfig.create();
LogicalPlanPersistence lpp = PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c);
- LogicalPlan plan = LogicalPlan.parse(lpp, FileUtils.getResourceAsString("/scan_screen_logical.json"));
+ LogicalPlan plan = LogicalPlan.parse(lpp, DrillFileUtils.getResourceAsString("/scan_screen_logical.json"));
String unparse = plan.unparse(lpp);
// System.out.println(unparse);
//System.out.println( new BasicOptimizer(DrillConfig.create()).convert(plan).unparse(c.getMapper().writer()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java
index 2fadb6e..f4ae026 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java
@@ -22,7 +22,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.LogicalPlanPersistence;
import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.common.scanner.persistence.ScanResult;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.planner.PhysicalPlanReader;
@@ -49,7 +49,7 @@ public class TestParsePhysicalPlan extends ExecTest {
PhysicalPlanReader reader = new PhysicalPlanReader(c, scanResult, lpp, CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), null);
ObjectReader r = lpp.getMapper().reader(PhysicalPlan.class);
ObjectWriter writer = lpp.getMapper().writer();
- PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/physical_test1.json"), Charsets.UTF_8));
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/physical_test1.json"), Charsets.UTF_8));
String unparse = plan.unparse(writer);
// System.out.println(unparse);
}
[12/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java
index f8b0d47..8bbe914 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.physical.impl.limit;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.PlanTestBase;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.common.expression.SchemaPath;
@@ -48,8 +48,8 @@ public class TestEarlyLimit0Optimization extends BaseTestQuery {
@BeforeClass
public static void createView() throws Exception {
- test("USE dfs_test.tmp");
- test(String.format("CREATE OR REPLACE VIEW %s AS SELECT " +
+ test("USE dfs.tmp");
+ test("CREATE OR REPLACE VIEW %s AS SELECT " +
"CAST(employee_id AS INT) AS employee_id, " +
"CAST(full_name AS VARCHAR(25)) AS full_name, " +
"CAST(position_id AS INTEGER) AS position_id, " +
@@ -63,26 +63,22 @@ public class TestEarlyLimit0Optimization extends BaseTestQuery {
"CAST(gender AS CHAR(1)) AS gender " +
"FROM cp.`employee.json` " +
"ORDER BY employee_id " +
- "LIMIT 1;", viewName));
- // { "employee_id":1,"full_name":"Sheri Nowmer","first_name":"Sheri","last_name":"Nowmer","position_id":1,
- // "position_title":"President","store_id":0,"department_id":1,"birth_date":"1961-08-26",
- // "hire_date":"1994-12-01 00:00:00.0","end_date":null,"salary":80000.0000,"supervisor_id":0,
- // "education_level":"Graduate Degree","marital_status":"S","gender":"F","management_role":"Senior Management" }
+ "LIMIT 1", viewName);
}
@AfterClass
public static void tearDownView() throws Exception {
- test("DROP VIEW " + viewName + ";");
+ test("DROP VIEW " + viewName);
}
@Before
public void setOption() throws Exception {
- test("SET `%s` = true;", ExecConstants.EARLY_LIMIT0_OPT_KEY);
+ test("SET `%s` = true", ExecConstants.EARLY_LIMIT0_OPT_KEY);
}
@After
public void resetOption() throws Exception {
- test("RESET `%s`;", ExecConstants.EARLY_LIMIT0_OPT_KEY);
+ test("RESET `%s`", ExecConstants.EARLY_LIMIT0_OPT_KEY);
}
// -------------------- SIMPLE QUERIES --------------------
@@ -90,7 +86,7 @@ public class TestEarlyLimit0Optimization extends BaseTestQuery {
@Test
public void infoSchema() throws Exception {
testBuilder()
- .sqlQuery(String.format("DESCRIBE %s", viewName))
+ .sqlQuery("DESCRIBE %s", viewName)
.unOrdered()
.baselineColumns("COLUMN_NAME", "DATA_TYPE", "IS_NULLABLE")
.baselineValues("employee_id", "INTEGER", "YES")
@@ -110,7 +106,7 @@ public class TestEarlyLimit0Optimization extends BaseTestQuery {
@Test
public void simpleSelect() throws Exception {
testBuilder()
- .sqlQuery(String.format("SELECT * FROM %s", viewName))
+ .sqlQuery("SELECT * FROM %s", viewName)
.ordered()
.baselineColumns("employee_id", "full_name", "position_id", "department_id", "birth_date", "hire_date",
"salary", "fsalary", "single", "education_level", "gender")
@@ -303,7 +299,7 @@ public class TestEarlyLimit0Optimization extends BaseTestQuery {
.ordered()
.baselineColumns("s", "p", "a", "c")
.baselineValues(null, 0.0D, 1.0D, 1L)
- .go();
+ .go();
testBuilder()
.sqlQuery(wrapLimit0(query))
@@ -339,10 +335,10 @@ public class TestEarlyLimit0Optimization extends BaseTestQuery {
@Test
public void nullableSumAndCount() throws Exception {
- final String query = "SELECT " +
+ final String query = String.format("SELECT " +
"COUNT(position_id) AS c, " +
"SUM(CAST((CASE WHEN position_id = 1 THEN NULL ELSE position_id END) AS INT)) AS p " +
- "FROM " + viewName;
+ "FROM %s", viewName);
@SuppressWarnings("unchecked")
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList(
@@ -412,13 +408,13 @@ public class TestEarlyLimit0Optimization extends BaseTestQuery {
@Test
public void sumsAndCounts1() throws Exception {
- final String query = "SELECT " +
+ final String query = String.format("SELECT " +
"COUNT(*) as cs, " +
"COUNT(1) as c1, " +
"COUNT(employee_id) as cc, " +
"SUM(1) as s1," +
"department_id " +
- " FROM " + viewName + " GROUP BY department_id";
+ " FROM %s GROUP BY department_id", viewName);
@SuppressWarnings("unchecked")
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList(
@@ -507,8 +503,8 @@ public class TestEarlyLimit0Optimization extends BaseTestQuery {
@Test
public void cast() throws Exception {
- final String query = "SELECT CAST(fsalary AS DOUBLE) AS d," +
- "CAST(employee_id AS BIGINT) AS e FROM " + viewName;
+ final String query = String.format("SELECT CAST(fsalary AS DOUBLE) AS d," +
+ "CAST(employee_id AS BIGINT) AS e FROM %s", viewName);
@SuppressWarnings("unchecked")
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList(
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java
index b49a12e..7a2ee07 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java
@@ -17,21 +17,27 @@
*/
package org.apache.drill.exec.physical.impl.limit;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.PlanTestBase;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.test.OperatorFixture;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
+
import static org.junit.Assert.assertEquals;
@Category(OperatorTest.class)
public class TestLimitWithExchanges extends BaseTestQuery {
- final String WORKING_PATH = TestTools.getWorkingPath();
- final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel", "json"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("tpchmulti", "region"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("tpchmulti", "nation"));
+ }
@Test
public void testLimitWithExchanges() throws Exception{
@@ -46,30 +52,28 @@ public class TestLimitWithExchanges extends BaseTestQuery {
final String[] excludedPlan = {};
// case 1. single table query.
- final String sql = String.format("select * from dfs_test.`%s/multilevel/json` limit 1 offset 2", TEST_RES_PATH);
+ final String sql = "select * from dfs.`multilevel/json` limit 1 offset 2";
final String[] expectedPlan ={"(?s)Limit\\(offset=\\[2\\], fetch=\\[1\\].*UnionExchange.*Limit\\(fetch=\\[3\\]\\).*Scan"};
testLimitHelper(sql, expectedPlan, excludedPlan, 1);
- final String sql2 = String.format("select * from dfs_test.`%s/multilevel/json` limit 1 offset 0", TEST_RES_PATH);
+ final String sql2 = "select * from dfs.`multilevel/json` limit 1 offset 0";
final String[] expectedPlan2 = {"(?s)Limit\\(offset=\\[0\\], fetch=\\[1\\].*UnionExchange.*Limit\\(fetch=\\[1\\]\\).*Scan"};
testLimitHelper(sql2, expectedPlan2, excludedPlan, 1);
- final String sql3 = String.format("select * from dfs_test.`%s/multilevel/json` limit 1", TEST_RES_PATH);
+ final String sql3 = "select * from dfs.`multilevel/json` limit 1";
final String[] expectedPlan3 = {"(?s)Limit\\(fetch=\\[1\\].*UnionExchange.*Limit\\(fetch=\\[1\\]\\).*Scan"};
testLimitHelper(sql3, expectedPlan3, excludedPlan, 1);
// case 2: join query.
- final String sql4 = String.format(
- "select * from dfs_test.`%s/tpchmulti/region` r, dfs_test.`%s/tpchmulti/nation` n " +
- "where r.r_regionkey = n.n_regionkey limit 1 offset 2", TEST_RES_PATH, TEST_RES_PATH );
+ final String sql4 = "select * from dfs.`tpchmulti/region` r, dfs.`tpchmulti/nation` n " +
+ "where r.r_regionkey = n.n_regionkey limit 1 offset 2";
final String[] expectedPlan4 = {"(?s)Limit\\(offset=\\[2\\], fetch=\\[1\\].*UnionExchange.*Limit\\(fetch=\\[3\\]\\).*Join"};
testLimitHelper(sql4, expectedPlan4, excludedPlan, 1);
- final String sql5 = String.format(
- "select * from dfs_test.`%s/tpchmulti/region` r, dfs_test.`%s/tpchmulti/nation` n " +
- "where r.r_regionkey = n.n_regionkey limit 1", TEST_RES_PATH, TEST_RES_PATH );
+ final String sql5 = "select * from dfs.`tpchmulti/region` r, dfs.`tpchmulti/nation` n " +
+ "where r.r_regionkey = n.n_regionkey limit 1";
final String[] expectedPlan5 = {"(?s)Limit\\(fetch=\\[1\\].*UnionExchange.*Limit\\(fetch=\\[1\\]\\).*Join"};
testLimitHelper(sql5, expectedPlan5, excludedPlan, 1);
@@ -86,11 +90,8 @@ public class TestLimitWithExchanges extends BaseTestQuery {
final String[] expectedPlan ={};
// case 1. Only "offset", but no "limit" : should not push "limit" down.
- final String sql = String.format("select * from dfs_test.`%s/tpchmulti/region` offset 2", TEST_RES_PATH);
- final String[] excludedPlan = {"(?s)Limit\\(offset=\\[2\\].*UnionExchange.*Limit.*Scan"};
-
// case 2. "limit" is higher than # of rowcount in table : should not push "limit" down.
- final String sql2 = String.format("select * from dfs_test.`%s/tpchmulti/region` limit 100", TEST_RES_PATH);
+ final String sql2 = "select * from dfs.`tpchmulti/region` limit 100";
final String[] excludedPlan2 = {"(?s)Limit\\(fetch=\\[100\\].*UnionExchange.*Limit.*Scan"};
testLimitHelper(sql2, expectedPlan, excludedPlan2, 5);
@@ -107,16 +108,16 @@ public class TestLimitWithExchanges extends BaseTestQuery {
// nation has 3 files, total 25 rows.
// Given slice_target = 5, if # of rows to fetch is < 5 : do NOT insert Exchange, and the query should run in single fragment.
// if # of row to fetch is >= 5: do insert exchange, and query should run in multiple fragments.
- final String sql = String.format("select * from dfs_test.`%s/tpchmulti/nation` limit 2", TEST_RES_PATH); // Test Limit_On_Scan rule.
- final String sql2 = String.format("select n_nationkey + 1000 from dfs_test.`%s/tpchmulti/nation` limit 2", TEST_RES_PATH); // Test Limit_On_Project rule.
+ final String sql = "select * from dfs.`tpchmulti/nation` limit 2"; // Test Limit_On_Scan rule.
+ final String sql2 = "select n_nationkey + 1000 from dfs.`tpchmulti/nation` limit 2"; // Test Limit_On_Project rule.
final String [] expectedPlan = {};
final String [] excludedPlan = {"UnionExchange"};
testLimitHelper(sql, expectedPlan, excludedPlan, 2);
testLimitHelper(sql2, expectedPlan, excludedPlan, 2);
- final String sql3 = String.format("select * from dfs_test.`%s/tpchmulti/nation` limit 10", TEST_RES_PATH); // Test Limit_On_Scan rule.
- final String sql4 = String.format("select n_nationkey + 1000 from dfs_test.`%s/tpchmulti/nation` limit 10", TEST_RES_PATH); // Test Limit_On_Project rule.
+ final String sql3 = "select * from dfs.`tpchmulti/nation` limit 10"; // Test Limit_On_Scan rule.
+ final String sql4 = "select n_nationkey + 1000 from dfs.`tpchmulti/nation` limit 10"; // Test Limit_On_Project rule.
final String [] expectedPlan2 = {"UnionExchange"};
final String [] excludedPlan2 = {};
@@ -130,7 +131,7 @@ public class TestLimitWithExchanges extends BaseTestQuery {
@Test
public void TestLimitAllOnParquet() throws Exception {
- final String query = String.format("select t.n_nationkey from cp.`tpch/nation.parquet` t limit all offset 5", TEST_RES_PATH);
+ final String query = "select t.n_nationkey from cp.`tpch/nation.parquet` t limit all offset 5";
final String [] expectedPlan = {};
final String [] excludedPlan = {"UnionExchange"};
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
index 042d38e..2032afc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertTrue;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -83,7 +83,7 @@ public class TestSimpleLimit extends ExecTest {
private void verifyLimitCount(DrillbitContext bitContext, UserClientConnection connection, String testPlan, int expectedCount) throws Throwable {
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -103,7 +103,7 @@ public class TestSimpleLimit extends ExecTest {
private void verifySum(DrillbitContext bitContext, UserClientConnection connection, String testPlan, int expectedCount, long expectedSum) throws Throwable {
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
index f0aa1b7..46afb42 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java
@@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.proto.UserBitShared.QueryData;
@@ -57,7 +57,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
bit2.run();
client.connect();
final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/mergerecv/merging_receiver.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/mergerecv/merging_receiver.json"),
Charsets.UTF_8));
int count = 0;
final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
@@ -88,7 +88,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
client.connect();
final List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/mergerecv/multiple_providers.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/mergerecv/multiple_providers.json"),
Charsets.UTF_8));
int count = 0;
final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
@@ -137,7 +137,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
client.connect();
final List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/mergerecv/empty_batch.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/mergerecv/empty_batch.json"),
Charsets.UTF_8));
int count = 0;
final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
@@ -167,7 +167,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
client.connect();
final List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/mergerecv/empty_batch_noschema.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/mergerecv/empty_batch_noschema.json"),
Charsets.UTF_8));
int count = 0;
final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
@@ -197,7 +197,7 @@ public class TestMergingReceiver extends PopUnitTestBase {
client.connect();
final List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/mergerecv/multiple_providers_empty_batches.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/mergerecv/multiple_providers_empty_batches.json"),
Charsets.UTF_8));
int count = 0;
final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java
index 4b00821..5a21ef1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java
@@ -28,7 +28,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.scanner.ClassPathScanner;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.record.RecordBatchLoader;
@@ -77,7 +77,7 @@ public class TestOrderedPartitionExchange extends PopUnitTestBase {
bit2.run();
client.connect();
List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
- Files.toString(FileUtils.getResourceAsFile("/sender/ordered_exchange.json"),
+ Files.toString(DrillFileUtils.getResourceAsFile("/sender/ordered_exchange.json"),
Charsets.UTF_8));
int count = 0;
List<Integer> partitionRecordCounts = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
index e35fba7..544ed49 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Random;
@@ -67,11 +68,9 @@ import org.apache.drill.exec.server.options.OptionValue.AccessibleScopes;
import org.apache.drill.exec.server.options.OptionValue.OptionScope;
import org.apache.drill.exec.util.Utilities;
import org.apache.drill.exec.work.QueryWorkUnit;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import com.google.common.collect.Lists;
@@ -94,31 +93,22 @@ public class TestPartitionSender extends PlanTestBase {
.withCredentials(UserBitShared.UserCredentials.newBuilder().setUserName("foo").build())
.build();
-
- public static TemporaryFolder testTempFolder = new TemporaryFolder();
-
- private final static int NUM_DEPTS = 40;
- private final static int NUM_EMPLOYEES = 1000;
- private final static int DRILLBITS_COUNT = 3;
-
- private static String empTableLocation;
+ private static final int NUM_DEPTS = 40;
+ private static final int NUM_EMPLOYEES = 1000;
+ private static final int DRILLBITS_COUNT = 3;
+ private static final String TABLE = "table";
private static String groupByQuery;
@BeforeClass
- public static void setupTempFolder() throws IOException {
- testTempFolder.create();
- }
-
- @BeforeClass
public static void generateTestDataAndQueries() throws Exception {
// Table consists of two columns "emp_id", "emp_name" and "dept_id"
- empTableLocation = testTempFolder.newFolder().getAbsolutePath();
+ final File empTableLocation = dirTestWatcher.makeRootSubDir(Paths.get(TABLE));
// Write 100 records for each new file
final int empNumRecsPerFile = 100;
for(int fileIndex=0; fileIndex<NUM_EMPLOYEES/empNumRecsPerFile; fileIndex++) {
- File file = new File(empTableLocation + File.separator + fileIndex + ".json");
+ File file = new File(empTableLocation, fileIndex + ".json");
PrintWriter printWriter = new PrintWriter(file);
for (int recordIndex = fileIndex*empNumRecsPerFile; recordIndex < (fileIndex+1)*empNumRecsPerFile; recordIndex++) {
String record = String.format("{ \"emp_id\" : %d, \"emp_name\" : \"Employee %d\", \"dept_id\" : %d }",
@@ -129,12 +119,7 @@ public class TestPartitionSender extends PlanTestBase {
}
// Initialize test queries
- groupByQuery = String.format("SELECT dept_id, count(*) as numEmployees FROM dfs.`%s` GROUP BY dept_id", empTableLocation);
- }
-
- @AfterClass
- public static void cleanupTempFolder() throws IOException {
- testTempFolder.delete();
+ groupByQuery = String.format("SELECT dept_id, count(*) as numEmployees FROM dfs.`%s` GROUP BY dept_id", TABLE);
}
@Test
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
index f62d133..022d153 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java
@@ -24,7 +24,7 @@ import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -57,7 +57,7 @@ public class TestSimpleProjection extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/project/test1.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/project/test1.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
index 77b03e3..98c997f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java
@@ -24,7 +24,7 @@ import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -59,7 +59,7 @@ public class TestSimpleSort extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/sort/one_key_sort.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/sort/one_key_sort.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
@@ -97,7 +97,7 @@ public class TestSimpleSort extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/sort/two_key_sort.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/sort/two_key_sort.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java
index 19f4f61..7a62706 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSort.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.physical.impl.sort;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.util.JsonStringArrayList;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
index 15b13d8..50b18ea 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/svremover/TestSVRemover.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.physical.impl.svremover;
import static org.junit.Assert.assertEquals;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
public class TestSVRemover extends BaseTestQuery {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
index b00e143..51d1e7b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -61,7 +61,7 @@ public class TestTraceMultiRecordBatch extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/multi_record_batch_trace.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/trace/multi_record_batch_trace.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
index 3e08ed1..780c83d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.cache.VectorAccessibleSerializable;
@@ -75,7 +75,7 @@ public class TestTraceOutputDump extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
index 9002936..4f49355 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertTrue;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.ops.FragmentContext;
@@ -54,7 +54,7 @@ public class TestSimpleUnion extends ExecTest {
mockDrillbitContext(bitContext);
final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
- final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/union/test1.json"), Charsets.UTF_8));
+ final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/union/test1.json"), Charsets.UTF_8));
final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java
index 4b3cbff..2fd9d6f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java
@@ -21,14 +21,17 @@ import static org.junit.Assert.assertFalse;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.store.easy.text.compliant.CompliantTextRecordReader;
+import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.DrillTest;
import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.test.LogFixture;
+import org.apache.drill.test.DirTestWatcher;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import ch.qos.logback.classic.Level;
@@ -38,6 +41,9 @@ public class TestValidationOptions extends DrillTest {
protected static LogFixture logFixture;
+ @Rule
+ public final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
logFixture = LogFixture.builder()
@@ -59,13 +65,12 @@ public class TestValidationOptions extends DrillTest {
@Test
public void testOptions() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
.maxParallelization(1)
.configProperty(ExecConstants.ENABLE_ITERATOR_VALIDATION, false)
.configProperty(ExecConstants.ENABLE_VECTOR_VALIDATION, false)
.sessionOption(ExecConstants.ENABLE_ITERATOR_VALIDATION_OPTION, true)
- .sessionOption(ExecConstants.ENABLE_VECTOR_VALIDATION_OPTION, true)
- ;
+ .sessionOption(ExecConstants.ENABLE_VECTOR_VALIDATION_OPTION, true);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
@@ -93,13 +98,12 @@ public class TestValidationOptions extends DrillTest {
@Test
public void testConfig() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
.maxParallelization(1)
.configProperty(ExecConstants.ENABLE_ITERATOR_VALIDATION, true)
.configProperty(ExecConstants.ENABLE_VECTOR_VALIDATION, true)
.sessionOption(ExecConstants.ENABLE_ITERATOR_VALIDATION_OPTION, false)
- .sessionOption(ExecConstants.ENABLE_VECTOR_VALIDATION_OPTION, false)
- ;
+ .sessionOption(ExecConstants.ENABLE_VECTOR_VALIDATION_OPTION, false);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
@@ -119,9 +123,8 @@ public class TestValidationOptions extends DrillTest {
@Test
public void testDefaults() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
- .maxParallelization(1)
- ;
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .maxParallelization(1);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
index c68cbe5..15b33c9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
@@ -17,11 +17,13 @@
*/
package org.apache.drill.exec.physical.impl.window;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -268,7 +270,7 @@ public class GenerateTestData {
}
}
- private static void writeData(final String path, final Partition[] partitions, final boolean addLineNo)
+ private static void writeData(final Path path, final Partition[] partitions, final boolean addLineNo)
throws FileNotFoundException {
// total number of rows
@@ -283,7 +285,7 @@ public class GenerateTestData {
// data file(s)
int fileId = 0;
- PrintStream dataStream = new PrintStream(path + "/" + fileId + ".data.json");
+ PrintStream dataStream = new PrintStream(path.resolve(fileId + ".data.json").toFile());
int emp_idx = 0;
int lineNo = 0;
@@ -316,11 +318,11 @@ public class GenerateTestData {
dataStream.close();
}
- private static void writeResults(final String path, final String prefix, final Partition[] partitions) throws FileNotFoundException {
+ private static void writeResults(final Path path, final String prefix, final Partition[] partitions) throws FileNotFoundException {
// expected results for query without order by clause
- final PrintStream resultStream = new PrintStream(path + prefix + ".tsv");
+ final PrintStream resultStream = new PrintStream(path.toString() + prefix + ".tsv");
// expected results for query with order by clause
- final PrintStream resultOrderStream = new PrintStream(path + prefix + ".oby.tsv");
+ final PrintStream resultOrderStream = new PrintStream(path.toString() + prefix + ".oby.tsv");
int idx = 0;
for (final Partition partition : partitions) {
@@ -354,11 +356,12 @@ public class GenerateTestData {
private static void generateData(final String tableName, final Partition[] pby_data, final Partition[] nopby_data,
final boolean addLineNo) throws FileNotFoundException {
- final String WORKING_PATH = TestTools.getWorkingPath();
- final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
- final String path = TEST_RES_PATH+"/window/" + tableName;
+ final Path path = TestTools.WORKING_PATH
+ .resolve(TestTools.TEST_RESOURCES)
+ .resolve(Paths.get("window", tableName));
+
+ final File pathFolder = path.toFile();
- final File pathFolder = new File(path);
if (!pathFolder.exists()) {
if (!pathFolder.mkdirs()) {
System.err.printf("Couldn't create folder %s, exiting%n", path);
@@ -366,7 +369,6 @@ public class GenerateTestData {
}
writeData(path, pby_data, addLineNo);
-
writeResults(path, "", nopby_data);
writeResults(path, ".pby", pby_data);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
index ab1ca31..90183d9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/TestWindowFrame.java
@@ -17,14 +17,15 @@
******************************************************************************/
package org.apache.drill.exec.physical.impl.window;
+import java.io.IOException;
+import java.nio.file.Paths;
import java.util.Properties;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.DrillTestWrapper;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.DrillTestWrapper;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserRemoteException;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecConstants;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -35,22 +36,20 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
public class TestWindowFrame extends BaseTestQuery {
-
- private static final String TEST_RES_PATH = TestTools.getWorkingPath() + "/src/test/resources";
-
@BeforeClass
- public static void setupMSortBatchSize() {
+ public static void setupMSortBatchSize() throws IOException {
// make sure memory sorter outputs 20 rows per batch
final Properties props = cloneDefaultTestConfigProperties();
props.put(ExecConstants.EXTERNAL_SORT_MSORT_MAX_BATCHSIZE, Integer.toString(20));
updateTestCluster(1, DrillConfig.create(props));
+ dirTestWatcher.copyResourceToRoot(Paths.get("window"));
}
private DrillTestWrapper buildWindowQuery(final String tableName, final boolean withPartitionBy, final int numBatches)
throws Exception {
return testBuilder()
- .sqlQuery(getFile("window/q1.sql"), TEST_RES_PATH, tableName, withPartitionBy ? "(partition by position_id)":"()")
+ .sqlQuery(getFile("window/q1.sql"), tableName, withPartitionBy ? "(partition by position_id)":"()")
.ordered()
.csvBaselineFile("window/" + tableName + (withPartitionBy ? ".pby" : "") + ".tsv")
.baselineColumns("count", "sum")
@@ -61,7 +60,7 @@ public class TestWindowFrame extends BaseTestQuery {
private DrillTestWrapper buildWindowWithOrderByQuery(final String tableName, final boolean withPartitionBy,
final int numBatches) throws Exception {
return testBuilder()
- .sqlQuery(getFile("window/q2.sql"), TEST_RES_PATH, tableName, withPartitionBy ? "(partition by position_id order by sub)" : "(order by sub)")
+ .sqlQuery(getFile("window/q2.sql"), tableName, withPartitionBy ? "(partition by position_id order by sub)" : "(order by sub)")
.ordered()
.csvBaselineFile("window/" + tableName + (withPartitionBy ? ".pby" : "") + ".oby.tsv")
.baselineColumns("count", "sum", "row_number", "rank", "dense_rank", "cume_dist", "percent_rank")
@@ -104,23 +103,23 @@ public class TestWindowFrame extends BaseTestQuery {
public void testMultipleFramers() throws Exception {
final String window = " OVER(PARTITION BY position_id ORDER by sub)";
test("SELECT COUNT(*)"+window+", SUM(salary)"+window+", ROW_NUMBER()"+window+", RANK()"+window+" " +
- "FROM dfs_test.`"+TEST_RES_PATH+"/window/b1.p1`"
+ "FROM dfs.`window/b1.p1`"
);
}
@Test
public void testUnboundedFollowing() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/q3.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/q3.sql"))
.ordered()
- .sqlBaselineQuery(getFile("window/q4.sql"), TEST_RES_PATH)
+ .sqlBaselineQuery(getFile("window/q4.sql"))
.build()
.run();
}
@Test
public void testAggregateRowsUnboundedAndCurrentRow() throws Exception {
- final String table = "dfs_test.`"+TEST_RES_PATH+"/window/b4.p4`";
+ final String table = "dfs.`window/b4.p4`";
testBuilder()
.sqlQuery(getFile("window/aggregate_rows_unbounded_current.sql"), table)
.ordered()
@@ -131,7 +130,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testLastValueRowsUnboundedAndCurrentRow() throws Exception {
- final String table = "dfs_test.`"+TEST_RES_PATH+"/window/b4.p4`";
+ final String table = "dfs.`window/b4.p4`";
testBuilder()
.sqlQuery(getFile("window/last_value_rows_unbounded_current.sql"), table)
.unOrdered()
@@ -142,7 +141,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testAggregateRangeCurrentAndCurrent() throws Exception {
- final String table = "dfs_test.`"+TEST_RES_PATH+"/window/b4.p4`";
+ final String table = "dfs.`window/b4.p4`";
testBuilder()
.sqlQuery(getFile("window/aggregate_range_current_current.sql"), table)
.unOrdered()
@@ -153,7 +152,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testFirstValueRangeCurrentAndCurrent() throws Exception {
- final String table = "dfs_test.`"+TEST_RES_PATH+"/window/b4.p4`";
+ final String table = "dfs.`window/b4.p4`";
testBuilder()
.sqlQuery(getFile("window/first_value_range_current_current.sql"), table)
.unOrdered()
@@ -210,27 +209,27 @@ public class TestWindowFrame extends BaseTestQuery {
@Test // DRILL-3218
@Category(UnlikelyTest.class)
public void testMaxVarChar() throws Exception {
- test(getFile("window/q3218.sql"), TEST_RES_PATH);
+ test(getFile("window/q3218.sql"));
}
@Test // DRILL-3220
@Category(UnlikelyTest.class)
public void testCountConst() throws Exception {
- test(getFile("window/q3220.sql"), TEST_RES_PATH);
+ test(getFile("window/q3220.sql"));
}
@Test // DRILL-3604
@Category(UnlikelyTest.class)
public void testFix3604() throws Exception {
// make sure the query doesn't fail
- test(getFile("window/3604.sql"), TEST_RES_PATH);
+ test(getFile("window/3604.sql"));
}
@Test // DRILL-3605
@Category(UnlikelyTest.class)
public void testFix3605() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/3605.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/3605.sql"))
.ordered()
.csvBaselineFile("window/3605.tsv")
.baselineColumns("col2", "lead_col2")
@@ -242,7 +241,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void testFix3606() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/3606.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/3606.sql"))
.ordered()
.csvBaselineFile("window/3606.tsv")
.baselineColumns("col2", "lead_col2")
@@ -253,7 +252,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testLead() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/lead.oby.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/lead.oby.sql"))
.ordered()
.csvBaselineFile("window/b4.p4.lead.oby.tsv")
.baselineColumns("lead")
@@ -264,7 +263,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testLagWithPby() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/lag.pby.oby.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/lag.pby.oby.sql"))
.ordered()
.csvBaselineFile("window/b4.p4.lag.pby.oby.tsv")
.baselineColumns("lag")
@@ -276,7 +275,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testLag() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/lag.oby.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/lag.oby.sql"))
.ordered()
.csvBaselineFile("window/b4.p4.lag.oby.tsv")
.baselineColumns("lag")
@@ -287,7 +286,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testLeadWithPby() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/lead.pby.oby.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/lead.pby.oby.sql"))
.ordered()
.csvBaselineFile("window/b4.p4.lead.pby.oby.tsv")
.baselineColumns("lead")
@@ -298,7 +297,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testFirstValue() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/fval.pby.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/fval.pby.sql"))
.ordered()
.csvBaselineFile("window/b4.p4.fval.pby.tsv")
.baselineColumns("first_value")
@@ -309,7 +308,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testLastValue() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/lval.pby.oby.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/lval.pby.oby.sql"))
.ordered()
.csvBaselineFile("window/b4.p4.lval.pby.oby.tsv")
.baselineColumns("last_value")
@@ -320,19 +319,19 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testFirstValueAllTypes() throws Exception {
// make sure all types are handled properly
- test(getFile("window/fval.alltypes.sql"), TEST_RES_PATH);
+ test(getFile("window/fval.alltypes.sql"));
}
@Test
public void testLastValueAllTypes() throws Exception {
// make sure all types are handled properly
- test(getFile("window/fval.alltypes.sql"), TEST_RES_PATH);
+ test(getFile("window/fval.alltypes.sql"));
}
@Test
public void testNtile() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/ntile.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/ntile.sql"))
.ordered()
.csvBaselineFile("window/b2.p4.ntile.tsv")
.baselineColumns("ntile")
@@ -343,7 +342,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void test3648Fix() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/3648.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/3648.sql"))
.ordered()
.csvBaselineFile("window/3648.tsv")
.baselineColumns("ntile")
@@ -353,13 +352,13 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void test3654Fix() throws Exception {
- test("SELECT FIRST_VALUE(col8) OVER(PARTITION BY col7 ORDER BY col8) FROM dfs_test.`%s/window/3648.parquet`", TEST_RES_PATH);
+ test("SELECT FIRST_VALUE(col8) OVER(PARTITION BY col7 ORDER BY col8) FROM dfs.`window/3648.parquet`");
}
@Test
public void test3643Fix() throws Exception {
try {
- test("SELECT NTILE(0) OVER(PARTITION BY col7 ORDER BY col8) FROM dfs_test.`%s/window/3648.parquet`", TEST_RES_PATH);
+ test("SELECT NTILE(0) OVER(PARTITION BY col7 ORDER BY col8) FROM dfs.`window/3648.parquet`");
fail("Query should have failed");
} catch (UserRemoteException e) {
assertEquals(ErrorType.FUNCTION, e.getErrorType());
@@ -369,7 +368,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void test3668Fix() throws Exception {
testBuilder()
- .sqlQuery(getFile("window/3668.sql"), TEST_RES_PATH)
+ .sqlQuery(getFile("window/3668.sql"))
.ordered()
.baselineColumns("cnt").baselineValues(2L)
.build()
@@ -379,20 +378,20 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void testLeadParams() throws Exception {
// make sure we only support default arguments for LEAD/LAG functions
- final String query = "SELECT %s OVER(PARTITION BY col7 ORDER BY col8) FROM dfs_test.`%s/window/3648.parquet`";
+ final String query = "SELECT %s OVER(PARTITION BY col7 ORDER BY col8) FROM dfs.`window/3648.parquet`";
- test(query, "LEAD(col8, 1)", TEST_RES_PATH);
- test(query, "LAG(col8, 1)", TEST_RES_PATH);
+ test(query, "LEAD(col8, 1)");
+ test(query, "LAG(col8, 1)");
try {
- test(query, "LEAD(col8, 2)", TEST_RES_PATH);
+ test(query, "LEAD(col8, 2)");
fail("query should fail");
} catch (UserRemoteException e) {
assertEquals(ErrorType.UNSUPPORTED_OPERATION, e.getErrorType());
}
try {
- test(query, "LAG(col8, 2)", TEST_RES_PATH);
+ test(query, "LAG(col8, 2)");
fail("query should fail");
} catch (UserRemoteException e) {
assertEquals(ErrorType.UNSUPPORTED_OPERATION, e.getErrorType());
@@ -435,12 +434,12 @@ public class TestWindowFrame extends BaseTestQuery {
@Test
public void test4457() throws Exception {
- runSQL(String.format("CREATE TABLE dfs_test.tmp.`4457` AS " +
+ runSQL("CREATE TABLE dfs.tmp.`4457` AS " +
"SELECT columns[0] AS c0, NULLIF(columns[1], 'null') AS c1 " +
- "FROM dfs_test.`%s/window/4457.csv`", TEST_RES_PATH));
+ "FROM dfs.`window/4457.csv`");
testBuilder()
- .sqlQuery("SELECT COALESCE(FIRST_VALUE(c1) OVER(ORDER BY c0 RANGE BETWEEN CURRENT ROW AND CURRENT ROW), 'EMPTY') AS fv FROM dfs_test.tmp.`4457`")
+ .sqlQuery("SELECT COALESCE(FIRST_VALUE(c1) OVER(ORDER BY c0 RANGE BETWEEN CURRENT ROW AND CURRENT ROW), 'EMPTY') AS fv FROM dfs.tmp.`4457`")
.ordered()
.baselineColumns("fv")
.baselineValues("a")
@@ -458,7 +457,7 @@ public class TestWindowFrame extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void test4657() throws Exception {
testBuilder()
- .sqlQuery("select row_number() over(order by position_id) rn, rank() over(order by position_id) rnk from dfs_test.`%s/window/b3.p2`", TEST_RES_PATH)
+ .sqlQuery("select row_number() over(order by position_id) rn, rank() over(order by position_id) rnk from dfs.`window/b3.p2`")
.ordered()
.csvBaselineFile("window/4657.tsv")
.baselineColumns("rn", "rnk")
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
index 4f139fc..1013511 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
@@ -21,16 +21,19 @@ import static java.lang.String.format;
import org.apache.drill.categories.ParquetTest;
import org.apache.drill.PlanTestBase;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.TestBuilder;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.store.parquet.Metadata;
-import org.apache.hadoop.fs.Path;
import org.joda.time.DateTime;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
/**
* Tests for compatibility reading old parquet files after date corruption
* issue was fixed in DRILL-4203.
@@ -64,7 +67,7 @@ import org.junit.experimental.categories.Category;
*/
@Category({ParquetTest.class, UnlikelyTest.class})
public class TestCorruptParquetDateCorrection extends PlanTestBase {
-
+ private static final Path PARQUET_4203 = Paths.get("parquet", "4203_corrupt_dates");
// 4 files are in the directory:
// - one created with the parquet-writer version number of "2"
// - files have extra meta field: parquet-writer.version = 2
@@ -75,44 +78,39 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
// - one from the 0.6 version of Drill, before files had min/max statistics
// - detecting corrupt values must be deferred to actual data page reading
// - one from 1.4, where there is a proper created-by, but the corruption is present
- private static final String MIXED_CORRUPTED_AND_CORRECT_DATES_PATH =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions";
+ private static final Path MIXED_CORRUPTED_AND_CORRECT_DATES_PATH = PARQUET_4203.resolve("mixed_drill_versions");
// partitioned with 1.2.0, no certain metadata that these were written with Drill
// the value will be checked to see that they look corrupt and they will be corrected
// by default. Users can use the format plugin option autoCorrectCorruptDates to disable
// this behavior if they have foreign parquet files with valid rare date values that are
// in the similar range as Drill's corrupt values
- private static final String CORRUPTED_PARTITIONED_DATES_1_2_PATH =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2";
+ private static final Path PARTITIONED_1_2_FOLDER = Paths.get("partitioned_with_corruption_4203_1_2");
+ private static final Path CORRUPTED_PARTITIONED_DATES_1_2_PATH = PARQUET_4203.resolve(PARTITIONED_1_2_FOLDER);
// partitioned with 1.4.0, no certain metadata regarding the date corruption status.
// The same detection approach of the corrupt date values as for the files partitioned with 1.2.0
- private static final String CORRUPTED_PARTITIONED_DATES_1_4_0_PATH =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203";
- private static final String PARQUET_DATE_FILE_WITH_NULL_FILLED_COLS =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/null_date_cols_with_corruption_4203.parquet";
- private static final String CORRECT_PARTITIONED_DATES_1_9_PATH =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption";
- private static final String VARCHAR_PARTITIONED =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition";
- private static final String DATE_PARTITIONED =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition";
- private static final String EXCEPTION_WHILE_PARSING_CREATED_BY_META =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/hive1dot2_fewtypes_null";
- private static final String CORRECT_DATES_1_6_0_PATH =
- "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/correct_dates_and_old_drill_parquet_writer.parquet";
- private static final String PARTITIONED_1_2_FOLDER = "partitioned_with_corruption_4203_1_2";
- private static final String MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER = "mixed_partitioned";
-
+ private static final Path PARTITIONED_1_4_FOLDER = Paths.get("partitioned_with_corruption_4203");
+ private static final Path CORRUPTED_PARTITIONED_DATES_1_4_0_PATH = PARQUET_4203.resolve(PARTITIONED_1_4_FOLDER);
+ private static final Path PARQUET_DATE_FILE_WITH_NULL_FILLED_COLS = PARQUET_4203.resolve("null_date_cols_with_corruption_4203.parquet");
+ private static final Path PARTITIONED_1_9_FOLDER = Paths.get("1_9_0_partitioned_no_corruption");
+ private static final Path CORRECT_PARTITIONED_DATES_1_9_PATH = PARQUET_4203.resolve(PARTITIONED_1_9_FOLDER);
+ private static final Path VARCHAR_PARTITIONED = PARQUET_4203.resolve("fewtypes_varcharpartition");
+ private static final Path DATE_PARTITIONED = PARQUET_4203.resolve("fewtypes_datepartition");
+ private static final Path EXCEPTION_WHILE_PARSING_CREATED_BY_META = PARQUET_4203.resolve("hive1dot2_fewtypes_null");
+ private static final Path CORRECT_DATES_1_6_0_PATH = PARQUET_4203.resolve("correct_dates_and_old_drill_parquet_writer.parquet");
+ private static final Path MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER = Paths.get("mixed_partitioned");
@BeforeClass
public static void initFs() throws Exception {
// Move files into temp directory, rewrite the metadata cache file to contain the appropriate absolute path
- copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_2_PATH);
- copyMetaDataCacheToTempReplacingInternalPaths("parquet/4203_corrupt_dates/drill.parquet.metadata_1_2.requires_replace.txt",
- PARTITIONED_1_2_FOLDER, Metadata.METADATA_FILENAME);
- copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_2_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
- copyDirectoryIntoTempSpace(CORRECT_PARTITIONED_DATES_1_9_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
- copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_4_0_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
+ dirTestWatcher.copyResourceToRoot(PARQUET_4203);
+ dirTestWatcher.copyResourceToRoot(CORRUPTED_PARTITIONED_DATES_1_2_PATH, PARTITIONED_1_2_FOLDER);
+ dirTestWatcher.copyResourceToRoot(CORRUPTED_PARTITIONED_DATES_1_4_0_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
+ dirTestWatcher.copyResourceToRoot(CORRUPTED_PARTITIONED_DATES_1_2_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER.resolve(PARTITIONED_1_2_FOLDER));
+ dirTestWatcher.copyResourceToRoot(CORRECT_PARTITIONED_DATES_1_9_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER.resolve(PARTITIONED_1_9_FOLDER));
+ dirTestWatcher.copyResourceToRoot(CORRUPTED_PARTITIONED_DATES_1_4_0_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER.resolve(PARTITIONED_1_4_FOLDER));
+ File metaData = dirTestWatcher.copyResourceToRoot(PARQUET_4203.resolve("drill.parquet.metadata_1_2.requires_replace.txt"),
+ PARTITIONED_1_2_FOLDER.resolve(Metadata.METADATA_FILENAME));
+ dirTestWatcher.replaceMetaDataContents(metaData, dirTestWatcher.getRootDir(), null);
}
/**
@@ -190,12 +188,11 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
.go();
}
-
@Test
public void testReadPartitionedOnCorruptedDates_UserDisabledCorrection() throws Exception {
try {
for (String selection : new String[]{"*", "date_col"}) {
- for (String table : new String[]{CORRUPTED_PARTITIONED_DATES_1_2_PATH, CORRUPTED_PARTITIONED_DATES_1_4_0_PATH}) {
+ for (Path table : new Path[]{CORRUPTED_PARTITIONED_DATES_1_2_PATH, CORRUPTED_PARTITIONED_DATES_1_4_0_PATH}) {
// for sanity, try reading all partitions without a filter
TestBuilder builder = testBuilder()
.sqlQuery("select %s from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))",
@@ -228,7 +225,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
public void testCorruptValueDetectionDuringPruning() throws Exception {
try {
for (String selection : new String[]{"*", "date_col"}) {
- for (String table : new String[]{CORRUPTED_PARTITIONED_DATES_1_2_PATH, CORRUPTED_PARTITIONED_DATES_1_4_0_PATH}) {
+ for (Path table : new Path[]{CORRUPTED_PARTITIONED_DATES_1_2_PATH, CORRUPTED_PARTITIONED_DATES_1_4_0_PATH}) {
// for sanity, try reading all partitions without a filter
TestBuilder builder = testBuilder()
.sqlQuery("select %s from dfs.`%s`", selection, table)
@@ -323,7 +320,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
@Test
public void testReadOldMetadataCacheFile() throws Exception {
// for sanity, try reading all partitions without a filter
- String query = format("select date_col from dfs.`%s`", new Path(getDfsTestTmpSchemaLocation(), PARTITIONED_1_2_FOLDER));
+ String query = format("select date_col from dfs.`%s`", PARTITIONED_1_2_FOLDER);
TestBuilder builder = testBuilder()
.sqlQuery(query)
.unOrdered()
@@ -336,7 +333,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
@Test
public void testReadOldMetadataCacheFileWithPruning() throws Exception {
String query = format("select date_col from dfs.`%s` where date_col = date '1970-01-01'",
- new Path(getDfsTestTmpSchemaLocation(), PARTITIONED_1_2_FOLDER));
+ PARTITIONED_1_2_FOLDER);
// verify that pruning is actually taking place
testPlanMatchingPatterns(query, new String[]{"numFiles=1", "usedMetadataFile=true"}, null);
@@ -354,7 +351,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
// for sanity, try reading all partitions without a filter
TestBuilder builder = testBuilder()
.sqlQuery("select date_col from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))",
- new Path(getDfsTestTmpSchemaLocation(), PARTITIONED_1_2_FOLDER))
+ PARTITIONED_1_2_FOLDER)
.unOrdered()
.baselineColumns("date_col");
addCorruptedDateBaselineValues(builder);
@@ -362,7 +359,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
String query = format("select date_col from table(dfs.`%s` (type => 'parquet', " +
"autoCorrectCorruptDates => false)) where date_col = cast('15334-03-17' as date)",
- new Path(getDfsTestTmpSchemaLocation(), PARTITIONED_1_2_FOLDER));
+ PARTITIONED_1_2_FOLDER);
// verify that pruning is actually taking place
testPlanMatchingPatterns(query, new String[]{"numFiles=1", "usedMetadataFile=true"}, null);
@@ -377,13 +374,13 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
@Test
public void testReadNewMetadataCacheFileOverOldAndNewFiles() throws Exception {
- String table = format("dfs.`%s`", new Path(getDfsTestTmpSchemaLocation(), MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER));
- copyMetaDataCacheToTempReplacingInternalPaths(
- "parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt",
- MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER, Metadata.METADATA_FILENAME);
+ File meta = dirTestWatcher.copyResourceToRoot(
+ PARQUET_4203.resolve("mixed_version_partitioned_metadata.requires_replace.txt"),
+ MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER.resolve(Metadata.METADATA_FILENAME));
+ dirTestWatcher.replaceMetaDataContents(meta, dirTestWatcher.getRootDir(), null);
// for sanity, try reading all partitions without a filter
TestBuilder builder = testBuilder()
- .sqlQuery("select date_col from " + table)
+ .sqlQuery("select date_col from dfs.`%s`", MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER)
.unOrdered()
.baselineColumns("date_col");
addDateBaselineValues(builder);
@@ -391,7 +388,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
addDateBaselineValues(builder);
builder.go();
- String query = format("select date_col from %s where date_col = date '1970-01-01'", table);
+ String query = format("select date_col from dfs.`%s` where date_col = date '1970-01-01'", MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
// verify that pruning is actually taking place
testPlanMatchingPatterns(query, new String[]{"numFiles=3", "usedMetadataFile=true"}, null);
@@ -436,7 +433,6 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
}
}
-
private void addDateBaselineValues(TestBuilder builder) {
builder
.baselineValues(new DateTime(1970, 1, 1, 0, 0))
@@ -477,5 +473,4 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
builder.go();
}
}
-
}
[19/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
deleted file mode 100644
index acde8ed..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
+++ /dev/null
@@ -1,689 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.RecognitionException;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.drill.DrillTestWrapper.TestServices;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.proto.UserBitShared.QueryType;
-import org.apache.drill.exec.proto.UserProtos.PreparedStatementHandle;
-import org.apache.drill.exec.record.BatchSchema;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.drill.exec.util.JsonStringHashMap;
-import org.apache.drill.exec.util.Text;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import org.joda.time.DateTimeZone;
-
-public class TestBuilder {
-
- /**
- * Test query to rung. Type of object depends on the {@link #queryType}
- */
- private Object query;
- // the type of query for the test
- private UserBitShared.QueryType queryType;
- // should the validation enforce ordering
- private Boolean ordered;
- private boolean approximateEquality;
- private TestServices services;
- // Used to pass the type information associated with particular column names rather than relying on the
- // ordering of the columns in the CSV file, or the default type inferences when reading JSON, this is used for the
- // case where results of the test query are adding type casts to the baseline queries, this saves a little bit of
- // setup in cases where strict type enforcement is not necessary for a given test
- protected Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap;
- // queries to run before the baseline or test queries, can be used to set options
- private String baselineOptionSettingQueries;
- private String testOptionSettingQueries;
- // two different methods are available for comparing ordered results, the default reads all of the records
- // into giant lists of objects, like one giant on-heap batch of 'vectors'
- // this flag enables the other approach which iterates through a hyper batch for the test query results and baseline
- // while this does work faster and use less memory, it can be harder to debug as all of the elements are not in a
- // single list
- private boolean highPerformanceComparison;
- // column names for use with the baseline values
- protected String[] baselineColumns;
- // In cases where we need to verify larger datasets without the risk of running the baseline data through
- // the drill engine, results can be provided in a list of maps. While this model does make a lot of sense, there is a
- // lot of work to make the type handling/casting work correctly, and making robust complex type handling work completely outside
- // of the drill engine for generating baselines would likely be more work than it would be worth. For now we will be
- // going with an approach of using this facility to validate the parts of the drill engine that could break in ways
- // that would affect the reading of baseline files (i.e. we need robust test for storage engines, project and casting that
- // use this interface) and then rely on the engine for the rest of the tests that will use the baseline queries.
- private List<Map<String, Object>> baselineRecords;
-
- private int expectedNumBatches = DrillTestWrapper.EXPECTED_BATCH_COUNT_NOT_SET;
-
- public TestBuilder(TestServices services) {
- this.services = services;
- reset();
- }
-
- public TestBuilder(TestServices services, Object query, UserBitShared.QueryType queryType, Boolean ordered,
- boolean approximateEquality, Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap,
- String baselineOptionSettingQueries, String testOptionSettingQueries, boolean highPerformanceComparison,
- int expectedNumBatches) {
- this(services);
- if (ordered == null) {
- throw new RuntimeException("Ordering not set, when using a baseline file or query you must explicitly call the ordered() or unOrdered() method on the " + this.getClass().getSimpleName());
- }
- this.query = query;
- this.queryType = queryType;
- this.ordered = ordered;
- this.approximateEquality = approximateEquality;
- this.baselineTypeMap = baselineTypeMap;
- this.baselineOptionSettingQueries = baselineOptionSettingQueries;
- this.testOptionSettingQueries = testOptionSettingQueries;
- this.highPerformanceComparison = highPerformanceComparison;
- this.expectedNumBatches = expectedNumBatches;
- }
-
- protected TestBuilder reset() {
- query = "";
- ordered = null;
- approximateEquality = false;
- highPerformanceComparison = false;
- testOptionSettingQueries = "";
- baselineOptionSettingQueries = "";
- baselineRecords = null;
- return this;
- }
-
- public DrillTestWrapper build() throws Exception {
- if ( ! ordered && highPerformanceComparison ) {
- throw new Exception("High performance comparison only available for ordered checks, to enforce this restriction, ordered() must be called first.");
- }
- return new DrillTestWrapper(this, services, query, queryType, baselineOptionSettingQueries, testOptionSettingQueries,
- getValidationQueryType(), ordered, highPerformanceComparison, baselineRecords, expectedNumBatches);
- }
-
- public List<Pair<SchemaPath, TypeProtos.MajorType>> getExpectedSchema() {
- return null;
- }
-
- public void go() throws Exception {
- build().run();
- }
-
- public TestBuilder sqlQuery(String query) {
- this.query = QueryTestUtil.normalizeQuery(query);
- this.queryType = UserBitShared.QueryType.SQL;
- return this;
- }
-
- public TestBuilder sqlQuery(String query, Object... replacements) {
- return sqlQuery(String.format(query, replacements));
- }
-
- public TestBuilder preparedStatement(PreparedStatementHandle preparedStatementHandle) {
- queryType = QueryType.PREPARED_STATEMENT;
- query = preparedStatementHandle;
- return this;
- }
-
- public TestBuilder sqlQueryFromFile(String queryFile) throws IOException {
- String query = BaseTestQuery.getFile(queryFile);
- this.query = query;
- queryType = UserBitShared.QueryType.SQL;
- return this;
- }
-
- public TestBuilder physicalPlanFromFile(String queryFile) throws IOException {
- String query = BaseTestQuery.getFile(queryFile);
- this.query = query;
- queryType = UserBitShared.QueryType.PHYSICAL;
- return this;
- }
-
- public TestBuilder ordered() {
- ordered = true;
- return this;
- }
-
- public TestBuilder unOrdered() {
- ordered = false;
- return this;
- }
-
- // this can only be used with ordered verifications, it does run faster and use less memory but may be
- // a little harder to debug as it iterates over a hyper batch rather than reading all of the values into
- // large on-heap lists
- public TestBuilder highPerformanceComparison() throws Exception {
- highPerformanceComparison = true;
- return this;
- }
-
- // list of queries to run before the baseline query, can be used to set several options
- // list takes the form of a semi-colon separated list
- public TestBuilder optionSettingQueriesForBaseline(String queries) {
- baselineOptionSettingQueries = queries;
- return this;
- }
-
- public TestBuilder optionSettingQueriesForBaseline(String queries, Object... args) {
- baselineOptionSettingQueries = String.format(queries, args);
- return this;
- }
-
- /**
- * list of queries to run before the test query, can be used to set several options
- * list takes the form of a semi-colon separated list.
- * @param queries queries that set session and system options
- * @return this test builder
- */
-
- public TestBuilder optionSettingQueriesForTestQuery(String queries) {
- testOptionSettingQueries = queries;
- return this;
- }
-
- public TestBuilder optionSettingQueriesForTestQuery(String query, Object... args) throws Exception {
- testOptionSettingQueries = String.format(query, args);
- return this;
- }
-
- public TestBuilder approximateEquality() {
- approximateEquality = true;
- return this;
- }
-
- // modified code from SchemaPath.De class. This should be used sparingly and only in tests if absolutely needed.
- public static SchemaPath parsePath(String path) {
- try {
- ExprLexer lexer = new ExprLexer(new ANTLRStringStream(path));
- CommonTokenStream tokens = new CommonTokenStream(lexer);
- ExprParser parser = new ExprParser(tokens);
-
- ExprParser.parse_return ret = parser.parse();
-
- if (ret.e instanceof SchemaPath) {
- return (SchemaPath) ret.e;
- } else {
- throw new IllegalStateException("Schema path is not a valid format.");
- }
- } catch (RecognitionException e) {
- throw new RuntimeException(e);
- }
- }
-
- Object getValidationQuery() throws Exception {
- throw new RuntimeException("Must provide some kind of baseline, either a baseline file or another query");
- }
-
- protected UserBitShared.QueryType getValidationQueryType() throws Exception {
- if (singleExplicitBaselineRecord()) {
- return null;
- }
- throw new RuntimeException("Must provide some kind of baseline, either a baseline file or another query");
- }
-
- public JSONTestBuilder jsonBaselineFile(String filePath) {
- return new JSONTestBuilder(filePath, services, query, queryType, ordered, approximateEquality,
- baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison,
- expectedNumBatches);
- }
-
- public CSVTestBuilder csvBaselineFile(String filePath) {
- return new CSVTestBuilder(filePath, services, query, queryType, ordered, approximateEquality,
- baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison,
- expectedNumBatches);
- }
-
- public SchemaTestBuilder schemaBaseLine(BatchSchema batchSchema) {
- List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = new ArrayList<>();
- for (final MaterializedField field : batchSchema) {
- expectedSchema.add(Pair.of(SchemaPath.getSimplePath(field.getName()), field.getType()));
- }
- return schemaBaseLine(expectedSchema);
- }
-
- public SchemaTestBuilder schemaBaseLine(List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema) {
- assert expectedSchema != null : "The expected schema can be provided once";
- assert baselineColumns == null : "The column information should be captured in expected schema, not baselineColumns";
-
- return new SchemaTestBuilder(
- services,
- query,
- queryType,
- baselineOptionSettingQueries,
- testOptionSettingQueries,
- expectedSchema);
- }
-
- public TestBuilder baselineTypes(Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap) {
- this.baselineTypeMap = baselineTypeMap;
- return this;
- }
-
- boolean typeInfoSet() {
- if (baselineTypeMap != null) {
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Indicate that the tests query should be checked for an empty result set.
- * @return the test builder
- */
- public TestBuilder expectsEmptyResultSet() {
- unOrdered();
- baselineRecords = new ArrayList<>();
- return this;
- }
-
- /**
- * Sets the number of expected batch count for this query. The test will fail if the query returns a different number
- * of batches
- *
- * @param expectedNumBatches expected batch count
- * @return this test builder
- */
- public TestBuilder expectsNumBatches(int expectedNumBatches) {
- this.expectedNumBatches = expectedNumBatches;
- return this;
- }
-
- /**
- * This method is used to pass in a simple list of values for a single record verification without
- * the need to create a CSV or JSON file to store the baseline.
- *
- * This can be called repeatedly to pass a list of records to verify. It works for both ordered and unordered
- * checks.
- *
- * @param baselineValues - the baseline values to validate
- * @return the test builder
- */
- public TestBuilder baselineValues(Object ... baselineValues) {
- assert getExpectedSchema() == null : "The expected schema is not needed when baselineValues are provided ";
- if (ordered == null) {
- throw new RuntimeException("Ordering not set, before specifying baseline data you must explicitly call the ordered() or unOrdered() method on the " + this.getClass().getSimpleName());
- }
- if (baselineRecords == null) {
- baselineRecords = new ArrayList<>();
- }
- Map<String, Object> ret = new HashMap<>();
- int i = 0;
- assertEquals("Must supply the same number of baseline values as columns.", baselineValues.length, baselineColumns.length);
- for (String s : baselineColumns) {
- ret.put(s, baselineValues[i]);
- i++;
- }
- this.baselineRecords.add(ret);
- return this;
- }
-
- /**
- * This can be used in cases where we want to avoid issues with the assumptions made by the test framework.
- * Most of the methods for verification in the framework run drill queries to generate the read baseline files or
- * execute alternative baseline queries. This model relies on basic functionality of reading files with storage
- * plugins and applying casts/projects to be stable.
- *
- * This method can be used to verify the engine for these cases and any other future execution paths that would
- * be used by both the test query and baseline. Without tests like this it is possible that some tests
- * could falsely report as passing, as both the test query and baseline query could run into the same problem
- * with an assumed stable code path and produce the same erroneous result.
- *
- * @param materializedRecords - a list of maps representing materialized results
- * @return the test builder
- */
- public TestBuilder baselineRecords(List<Map<String, Object>> materializedRecords) {
- this.baselineRecords = materializedRecords;
- return this;
- }
-
- /**
- * This setting has a slightly different impact on the test depending on some of the other
- * configuration options are set.
- *
- * If a JSON baseline file is given, this list will act as a project list to verify the
- * test query against a subset of the columns in the file.
- *
- * For a CSV baseline file, these will act as aliases for columns [0 .. n] in the repeated
- * varchar column that is read out of CSV.
- *
- * For a baseline sql query, this currently has no effect.
- *
- * For explicit baseline values given in java code with the baselineValues() method, these will
- * be used to create a map for the one record verification.
- */
- public TestBuilder baselineColumns(String... columns) {
- assert getExpectedSchema() == null : "The expected schema is not needed when baselineColumns are provided ";
- for (int i = 0; i < columns.length; i++) {
- columns[i] = parsePath(columns[i]).toExpr();
- }
- this.baselineColumns = columns;
- return this;
- }
-
- private boolean singleExplicitBaselineRecord() {
- return baselineRecords != null;
- }
-
- /**
- * Provide a SQL query to validate against.
- * @param baselineQuery
- * @return the test builder
- */
- public BaselineQueryTestBuilder sqlBaselineQuery(Object baselineQuery) {
- return new BaselineQueryTestBuilder(baselineQuery, UserBitShared.QueryType.SQL, services, query, queryType, ordered, approximateEquality,
- baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches);
- }
-
- public BaselineQueryTestBuilder sqlBaselineQuery(String query, String ...replacements) {
- return sqlBaselineQuery(String.format(query, (Object[]) replacements));
- }
-
- // provide a path to a file containing a SQL query to use as a baseline
- public BaselineQueryTestBuilder sqlBaselineQueryFromFile(String baselineQueryFilename) throws IOException {
- String baselineQuery = BaseTestQuery.getFile(baselineQueryFilename);
- return new BaselineQueryTestBuilder(baselineQuery, UserBitShared.QueryType.SQL, services, query, queryType, ordered, approximateEquality,
- baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches);
- }
-
- // as physical plans are verbose, this is the only option provided for specifying them, we should enforce
- // that physical plans, or any large JSON strings do not live in the Java source as literals
- public BaselineQueryTestBuilder physicalPlanBaselineQueryFromFile(String baselinePhysicalPlanPath) throws IOException {
- String baselineQuery = BaseTestQuery.getFile(baselinePhysicalPlanPath);
- return new BaselineQueryTestBuilder(baselineQuery, UserBitShared.QueryType.PHYSICAL, services, query, queryType, ordered, approximateEquality,
- baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries, highPerformanceComparison, expectedNumBatches);
- }
-
- private String getDecimalPrecisionScaleInfo(TypeProtos.MajorType type) {
- String precision = "";
- switch(type.getMinorType()) {
- case DECIMAL18:
- case DECIMAL28SPARSE:
- case DECIMAL38SPARSE:
- case DECIMAL38DENSE:
- case DECIMAL28DENSE:
- case DECIMAL9:
- precision = String.format("(%d,%d)", type.getPrecision(), type.getScale());
- break;
- default:
- ; // do nothing empty string set above
- }
- return precision;
- }
-
- public class CSVTestBuilder extends TestBuilder {
-
- // path to the baseline file that will be inserted into the validation query
- private String baselineFilePath;
- // use to cast the baseline file columns, if not set the types
- // that come out of the test query drive interpretation of baseline
- private TypeProtos.MajorType[] baselineTypes;
-
- CSVTestBuilder(String baselineFile, TestServices services, Object query, UserBitShared.QueryType queryType, Boolean ordered,
- boolean approximateEquality, Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap,
- String baselineOptionSettingQueries, String testOptionSettingQueries, boolean highPerformanceComparison,
- int expectedNumBatches) {
- super(services, query, queryType, ordered, approximateEquality, baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries,
- highPerformanceComparison, expectedNumBatches);
- this.baselineFilePath = baselineFile;
- }
-
- public CSVTestBuilder baselineTypes(TypeProtos.MajorType... baselineTypes) {
- this.baselineTypes = baselineTypes;
- this.baselineTypeMap = null;
- return this;
- }
-
- // convenience method to convert minor types to major types if no decimals with precisions are needed
- public CSVTestBuilder baselineTypes(TypeProtos.MinorType ... baselineTypes) {
- TypeProtos.MajorType[] majorTypes = new TypeProtos.MajorType[baselineTypes.length];
- int i = 0;
- for(TypeProtos.MinorType minorType : baselineTypes) {
- majorTypes[i] = Types.required(minorType);
- i++;
- }
- this.baselineTypes = majorTypes;
- this.baselineTypeMap = null;
- return this;
- }
-
- @Override
- protected TestBuilder reset() {
- super.reset();
- baselineTypeMap = null;
- baselineTypes = null;
- baselineFilePath = null;
- return this;
- }
-
- @Override
- boolean typeInfoSet() {
- if (super.typeInfoSet() || baselineTypes != null) {
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- String getValidationQuery() throws Exception {
- if (baselineColumns.length == 0) {
- throw new Exception("Baseline CSV files require passing column names, please call the baselineColumns() method on the test builder.");
- }
-
- if (baselineTypes != null) {
- assertEquals("Must pass the same number of types as column names if types are provided.", baselineTypes.length, baselineColumns.length);
- }
-
- String[] aliasedExpectedColumns = new String[baselineColumns.length];
- for (int i = 0; i < baselineColumns.length; i++) {
- aliasedExpectedColumns[i] = "columns[" + i + "] ";
- TypeProtos.MajorType majorType;
- if (baselineTypes != null) {
- majorType = baselineTypes[i];
- } else if (baselineTypeMap != null) {
- majorType = baselineTypeMap.get(parsePath(baselineColumns[i]));
- } else {
- throw new Exception("Type information not set for interpreting csv baseline file.");
- }
- String precision = getDecimalPrecisionScaleInfo(majorType);
- // TODO - determine if there is a better behavior here, if we do not specify a length the default behavior is
- // to cast to varchar with length 1
- // set default cast size for varchar, the cast function will take the lesser of this passed value and the
- // length of the incoming data when choosing the length for the outgoing data
- if (majorType.getMinorType() == TypeProtos.MinorType.VARCHAR ||
- majorType.getMinorType() == TypeProtos.MinorType.VARBINARY) {
- precision = "(65000)";
- }
- aliasedExpectedColumns[i] = "cast(" + aliasedExpectedColumns[i] + " as " +
- Types.getNameOfMinorType(majorType.getMinorType()) + precision + " ) " + baselineColumns[i];
- }
- String query = "select " + Joiner.on(", ").join(aliasedExpectedColumns) + " from cp.`" + baselineFilePath + "`";
- return query;
- }
-
- @Override
- protected UserBitShared.QueryType getValidationQueryType() throws Exception {
- return UserBitShared.QueryType.SQL;
- }
- }
-
- public class SchemaTestBuilder extends TestBuilder {
- private List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema;
- SchemaTestBuilder(TestServices services, Object query, UserBitShared.QueryType queryType,
- String baselineOptionSettingQueries, String testOptionSettingQueries, List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema) {
- super(services, query, queryType, false, false, null, baselineOptionSettingQueries, testOptionSettingQueries, false, -1);
- expectsEmptyResultSet();
- this.expectedSchema = expectedSchema;
- }
-
- @Override
- public TestBuilder baselineColumns(String... columns) {
- assert false : "The column information should be captured in expected scheme, not baselineColumns";
- return this;
- }
-
- @Override
- public TestBuilder baselineRecords(List<Map<String, Object>> materializedRecords) {
- assert false : "Since only schema will be compared in this test, no record is expected";
- return this;
- }
-
- @Override
- public TestBuilder baselineValues(Object... objects) {
- assert false : "Since only schema will be compared in this test, no record is expected";
- return this;
- }
-
- @Override
- protected UserBitShared.QueryType getValidationQueryType() throws Exception {
- return null;
- }
-
- @Override
- public List<Pair<SchemaPath, TypeProtos.MajorType>> getExpectedSchema() {
- return expectedSchema;
- }
- }
-
- public class JSONTestBuilder extends TestBuilder {
-
- // path to the baseline file that will be inserted into the validation query
- private String baselineFilePath;
-
- JSONTestBuilder(String baselineFile, TestServices services, Object query, UserBitShared.QueryType queryType, Boolean ordered,
- boolean approximateEquality, Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap,
- String baselineOptionSettingQueries, String testOptionSettingQueries, boolean highPerformanceComparison,
- int expectedNumBatches) {
- super(services, query, queryType, ordered, approximateEquality, baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries,
- highPerformanceComparison, expectedNumBatches);
- this.baselineFilePath = baselineFile;
- this.baselineColumns = new String[] {"*"};
- }
-
- @Override
- String getValidationQuery() {
- return "select " + Joiner.on(", ").join(baselineColumns) + " from cp.`" + baselineFilePath + "`";
- }
-
- @Override
- protected UserBitShared.QueryType getValidationQueryType() throws Exception {
- return UserBitShared.QueryType.SQL;
- }
-
- }
-
- public class BaselineQueryTestBuilder extends TestBuilder {
-
- /**
- * Baseline query. Type of object depends on {@link #baselineQueryType}
- */
- private Object baselineQuery;
- private UserBitShared.QueryType baselineQueryType;
-
- BaselineQueryTestBuilder(Object baselineQuery, UserBitShared.QueryType baselineQueryType, TestServices services,
- Object query, UserBitShared.QueryType queryType, Boolean ordered,
- boolean approximateEquality, Map<SchemaPath, TypeProtos.MajorType> baselineTypeMap,
- String baselineOptionSettingQueries, String testOptionSettingQueries, boolean highPerformanceComparison,
- int expectedNumBatches) {
- super(services, query, queryType, ordered, approximateEquality, baselineTypeMap, baselineOptionSettingQueries, testOptionSettingQueries,
- highPerformanceComparison, expectedNumBatches);
- this.baselineQuery = baselineQuery;
- this.baselineQueryType = baselineQueryType;
- }
-
- @Override
- Object getValidationQuery() {
- return baselineQuery;
- }
-
- @Override
- protected UserBitShared.QueryType getValidationQueryType() throws Exception {
- return baselineQueryType;
- }
-
- // This currently assumes that all explicit baseline queries will have fully qualified type information
- // if this changes, the baseline query can be run in a sub query with the implicit or explicit type passing
- // added on top of it, as is currently when done when reading a baseline file
- @Override
- boolean typeInfoSet() {
- return true;
- }
-
- }
-
- /**
- * Convenience method to create a {@link JsonStringArrayList list} from the given values.
- */
- public static JsonStringArrayList<Object> listOf(Object... values) {
- final JsonStringArrayList<Object> list = new JsonStringArrayList<>();
- for (Object value:values) {
- if (value instanceof CharSequence) {
- list.add(new Text(value.toString()));
- } else {
- list.add(value);
- }
- }
- return list;
- }
-
- /**
- * Convenience method to create a {@link JsonStringHashMap<String, Object> map} instance with the given key value sequence.
- *
- * Key value sequence consists of key - value pairs such that a key precedes its value. For instance:
- *
- * mapOf("name", "Adam", "age", 41) corresponds to {"name": "Adam", "age": 41} in JSON.
- */
- public static JsonStringHashMap<String, Object> mapOf(Object... keyValueSequence) {
- Preconditions.checkArgument(keyValueSequence.length%2==0, "Length of key value sequence must be even");
- final JsonStringHashMap<String, Object> map = new JsonStringHashMap<>();
- for (int i=0; i<keyValueSequence.length; i+=2) {
- Object value = keyValueSequence[i+1];
- if (value instanceof CharSequence) {
- value = new Text(value.toString());
- }
- map.put(String.class.cast(keyValueSequence[i]), value);
- }
- return map;
- }
-
- /**
- * Helper method for the timestamp values that depend on the local timezone
- * @param value expected timestamp value in UTC
- * @return timestamp value for the local timezone
- */
- public static Timestamp convertToLocalTimestamp(String value) {
- long UTCTimestamp = Timestamp.valueOf(value).getTime();
- return new Timestamp(DateTimeZone.getDefault().convertUTCToLocal(UTCTimestamp));
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestCTASJson.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestCTASJson.java b/exec/java-exec/src/test/java/org/apache/drill/TestCTASJson.java
index c76892d..d074397 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestCTASJson.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestCTASJson.java
@@ -17,19 +17,11 @@
*/
package org.apache.drill;
-
-import org.apache.drill.common.util.TestTools;
-import org.apache.drill.exec.ExecConstants;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
public class TestCTASJson extends PlanTestBase {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestCTASJson.class);
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
-
@Test
/**
* Test a source json file that contains records that are maps with fields of all types.
@@ -37,7 +29,7 @@ public class TestCTASJson extends PlanTestBase {
*/
public void testctas_alltypes_map() throws Exception {
String testName = "ctas_alltypes_map";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create table " + testName + "_json as select * from cp.`json/" + testName + ".json`");
final String query = "select * from `" + testName + "_json` t1 ";
@@ -65,7 +57,7 @@ public class TestCTASJson extends PlanTestBase {
*/
public void testctas_alltypes_map_noskip() throws Exception {
String testName = "ctas_alltypes_map";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create table " + testName + "_json as select * from cp.`json/" + testName + ".json`");
final String query = "select * from `" + testName + "_json` t1 ";
@@ -94,7 +86,7 @@ public class TestCTASJson extends PlanTestBase {
*/
public void testctas_alltypes_repeatedmap() throws Exception {
String testName = "ctas_alltypes_repeated_map";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create table " + testName + "_json as select * from cp.`json/" + testName + ".json`");
final String query = "select * from `" + testName + "_json` t1 ";
@@ -124,7 +116,7 @@ public class TestCTASJson extends PlanTestBase {
*/
public void testctas_alltypes_repeated_map_noskip() throws Exception {
String testName = "ctas_alltypes_repeated_map";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create table " + testName + "_json as select * from cp.`json/" + testName + ".json`");
final String query = "select * from `" + testName + "_json` t1 ";
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java b/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java
index b0238e2..fee29e0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestCTASPartitionFilter.java
@@ -17,20 +17,16 @@
*/
package org.apache.drill;
-
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.common.util.TestTools;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.nio.file.Paths;
+
import static org.junit.Assert.assertEquals;
public class TestCTASPartitionFilter extends PlanTestBase {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestCTASPartitionFilter.class);
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
-
private static void testExcludeFilter(String query, int expectedNumFiles,
String excludedFilterPattern, int expectedRowCount) throws Exception {
int actualRowCount = testSql(query);
@@ -47,9 +43,14 @@ public class TestCTASPartitionFilter extends PlanTestBase {
testPlanMatchingPatterns(query, new String[]{numFilesPattern, includedFilterPattern}, new String[]{});
}
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("multilevel"));
+ }
+
@Test
public void testDrill3965() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create table orders_auto_partition partition by(o_orderpriority) as select * from cp.`tpch/orders.parquet`");
test("explain plan for select count(*) from `orders_auto_partition/1_0_1.parquet` where o_orderpriority = '5-LOW'");
}
@@ -58,49 +59,49 @@ public class TestCTASPartitionFilter extends PlanTestBase {
public void withDistribution() throws Exception {
test("alter session set `planner.slice_target` = 1");
test("alter session set `store.partition.hash_distribute` = true");
- test("use dfs_test.tmp");
- test(String.format("create table orders_distribution partition by (o_orderpriority) as select * from dfs_test.`%s/multilevel/parquet`", TEST_RES_PATH));
+ test("use dfs.tmp");
+ test("create table orders_distribution partition by (o_orderpriority) as select * from dfs.`multilevel/parquet`");
String query = "select * from orders_distribution where o_orderpriority = '1-URGENT'";
- testExcludeFilter(query, 1, "Filter", 24);
+ testExcludeFilter(query, 1, "Filter\\(", 24);
}
@Test
public void withoutDistribution() throws Exception {
test("alter session set `planner.slice_target` = 1");
test("alter session set `store.partition.hash_distribute` = false");
- test("use dfs_test.tmp");
- test(String.format("create table orders_no_distribution partition by (o_orderpriority) as select * from dfs_test.`%s/multilevel/parquet`", TEST_RES_PATH));
+ test("use dfs.tmp");
+ test("create table orders_no_distribution partition by (o_orderpriority) as select * from dfs.`multilevel/parquet`");
String query = "select * from orders_no_distribution where o_orderpriority = '1-URGENT'";
- testExcludeFilter(query, 2, "Filter", 24);
+ testExcludeFilter(query, 2, "Filter\\(", 24);
}
@Test
public void testDRILL3410() throws Exception {
test("alter session set `planner.slice_target` = 1");
test("alter session set `store.partition.hash_distribute` = true");
- test("use dfs_test.tmp");
- test(String.format("create table drill_3410 partition by (o_orderpriority) as select * from dfs_test.`%s/multilevel/parquet`", TEST_RES_PATH));
+ test("use dfs.tmp");
+ test("create table drill_3410 partition by (o_orderpriority) as select * from dfs.`multilevel/parquet`");
String query = "select * from drill_3410 where (o_orderpriority = '1-URGENT' and o_orderkey = 10) or (o_orderpriority = '2-HIGH' or o_orderkey = 11)";
- testIncludeFilter(query, 1, "Filter", 34);
+ testIncludeFilter(query, 1, "Filter\\(", 34);
}
@Test
public void testDRILL3414() throws Exception {
test("alter session set `planner.slice_target` = 1");
test("alter session set `store.partition.hash_distribute` = true");
- test("use dfs_test.tmp");
- test(String.format("create table drill_3414 partition by (x, y) as select dir0 as x, dir1 as y, columns from dfs_test.`%s/multilevel/csv`", TEST_RES_PATH));
+ test("use dfs.tmp");
+ test("create table drill_3414 partition by (x, y) as select dir0 as x, dir1 as y, columns from dfs.`multilevel/csv`");
String query = ("select * from drill_3414 where (x=1994 or y='Q1') and (x=1995 or y='Q2' or columns[0] > 5000)");
- testIncludeFilter(query, 6, "Filter", 20);
+ testIncludeFilter(query, 6, "Filter\\(", 20);
}
@Test
public void testDRILL3414_2() throws Exception {
test("alter session set `planner.slice_target` = 1");
test("alter session set `store.partition.hash_distribute` = true");
- test("use dfs_test.tmp");
- test(String.format("create table drill_3414_2 partition by (x, y) as select dir0 as x, dir1 as y, columns from dfs_test.`%s/multilevel/csv`", TEST_RES_PATH));
+ test("use dfs.tmp");
+ test("create table drill_3414_2 partition by (x, y) as select dir0 as x, dir1 as y, columns from dfs.`multilevel/csv`");
String query = ("select * from drill_3414_2 where (x=1994 or y='Q1') and (x=1995 or y='Q2' or columns[0] > 5000) or columns[0] < 3000");
- testIncludeFilter(query, 1, "Filter", 120);
+ testIncludeFilter(query, 1, "Filter\\(", 120);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestCaseSensitivity.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestCaseSensitivity.java b/exec/java-exec/src/test/java/org/apache/drill/TestCaseSensitivity.java
index 0a157eb..1779078 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestCaseSensitivity.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestCaseSensitivity.java
@@ -20,6 +20,7 @@ package org.apache.drill;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.categories.UnlikelyTest;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
import org.junit.experimental.categories.Category;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java b/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
index cfd8fbd..4f8c16f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
@@ -18,17 +18,17 @@
package org.apache.drill;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.work.ExecErrorConstants;
import org.apache.drill.exec.work.foreman.SqlUnsupportedException;
import org.apache.drill.exec.work.foreman.UnsupportedDataTypeException;
import org.apache.drill.exec.work.foreman.UnsupportedFunctionException;
import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(UnlikelyTest.class)
-public class TestDisabledFunctionality extends BaseTestQuery{
+public class TestDisabledFunctionality extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class);
@Test(expected = UserException.class) // see DRILL-2054
@@ -263,12 +263,8 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test(expected = UnsupportedFunctionException.class) // see DRILL-2181
public void testFlattenWithinGroupBy() throws Exception {
try {
- String root = FileUtils.getResourceAsFile("/store/text/sample.json").toURI().toString();
- String query = String.format("select flatten(j.topping) tt " +
- "from dfs_test.`%s` j " +
- "group by flatten(j.topping)", root);
-
- test(query);
+ test("select flatten(j.topping) tt " +
+ "from cp.`store/text/sample.json` j group by flatten(j.topping)");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -278,12 +274,9 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test(expected = UnsupportedFunctionException.class) // see DRILL-2181
public void testFlattenWithinOrderBy() throws Exception {
try {
- String root = FileUtils.getResourceAsFile("/store/text/sample.json").toURI().toString();
- String query = String.format("select flatten(j.topping) tt " +
- "from dfs_test.`%s` j " +
- "order by flatten(j.topping)", root);
-
- test(query);
+ test("select flatten(j.topping) tt " +
+ "from cp.`store/text/sample.json` j " +
+ "order by flatten(j.topping)");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -293,11 +286,8 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test(expected = UnsupportedFunctionException.class) // see DRILL-2181
public void testFlattenWithinAggFunction() throws Exception {
try {
- String root = FileUtils.getResourceAsFile("/store/text/sample.json").toURI().toString();
- String query = String.format("select count(flatten(j.topping)) tt " +
- "from dfs_test.`%s` j", root);
-
- test(query);
+ test("select count(flatten(j.topping)) tt " +
+ "from cp.`store/text/sample.json` j");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -307,11 +297,8 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test(expected = UnsupportedFunctionException.class) // see DRILL-2181
public void testFlattenWithinDistinct() throws Exception {
try {
- String root = FileUtils.getResourceAsFile("/store/text/sample.json").toURI().toString();
- String query = String.format("select Distinct (flatten(j.topping)) tt " +
- "from dfs_test.`%s` j", root);
-
- test(query);
+ test("select Distinct (flatten(j.topping)) tt " +
+ "from cp.`store/text/sample.json` j");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -333,8 +320,7 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test (expected = UnsupportedFunctionException.class) //DRILL-3802
public void testDisableRollup() throws Exception{
try {
- final String query = "select n_regionkey, count(*) as cnt from cp.`tpch/nation.parquet` group by rollup(n_regionkey, n_name)";
- test(query);
+ test("select n_regionkey, count(*) as cnt from cp.`tpch/nation.parquet` group by rollup(n_regionkey, n_name)");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -344,8 +330,7 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test (expected = UnsupportedFunctionException.class) //DRILL-3802
public void testDisableCube() throws Exception{
try {
- final String query = "select n_regionkey, count(*) as cnt from cp.`tpch/nation.parquet` group by cube(n_regionkey, n_name)";
- test(query);
+ test("select n_regionkey, count(*) as cnt from cp.`tpch/nation.parquet` group by cube(n_regionkey, n_name)");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -355,8 +340,7 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test (expected = UnsupportedFunctionException.class) //DRILL-3802
public void testDisableGroupingSets() throws Exception{
try {
- final String query = "select n_regionkey, count(*) as cnt from cp.`tpch/nation.parquet` group by grouping sets(n_regionkey, n_name)";
- test(query);
+ test("select n_regionkey, count(*) as cnt from cp.`tpch/nation.parquet` group by grouping sets(n_regionkey, n_name)");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -366,8 +350,7 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test (expected = UnsupportedFunctionException.class) //DRILL-3802
public void testDisableGrouping() throws Exception{
try {
- final String query = "select n_regionkey, count(*), GROUPING(n_regionkey) from cp.`tpch/nation.parquet` group by n_regionkey;";
- test(query);
+ test("select n_regionkey, count(*), GROUPING(n_regionkey) from cp.`tpch/nation.parquet` group by n_regionkey;");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -377,8 +360,7 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test (expected = UnsupportedFunctionException.class) //DRILL-3802
public void testDisableGrouping_ID() throws Exception{
try {
- final String query = "select n_regionkey, count(*), GROUPING_ID(n_regionkey) from cp.`tpch/nation.parquet` group by n_regionkey;";
- test(query);
+ test("select n_regionkey, count(*), GROUPING_ID(n_regionkey) from cp.`tpch/nation.parquet` group by n_regionkey;");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -388,8 +370,7 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test (expected = UnsupportedFunctionException.class) //DRILL-3802
public void testDisableGroup_ID() throws Exception{
try {
- final String query = "select n_regionkey, count(*), GROUP_ID() from cp.`tpch/nation.parquet` group by n_regionkey;";
- test(query);
+ test("select n_regionkey, count(*), GROUP_ID() from cp.`tpch/nation.parquet` group by n_regionkey;");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
@@ -399,8 +380,7 @@ public class TestDisabledFunctionality extends BaseTestQuery{
@Test (expected = UnsupportedFunctionException.class) //DRILL-3802
public void testDisableGroupingInFilter() throws Exception{
try {
- final String query = "select n_regionkey, count(*) from cp.`tpch/nation.parquet` group by n_regionkey HAVING GROUPING(n_regionkey) = 1";
- test(query);
+ test("select n_regionkey, count(*) from cp.`tpch/nation.parquet` group by n_regionkey HAVING GROUPING(n_regionkey) = 1");
} catch(UserException ex) {
throwAsUnsupportedException(ex);
throw ex;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java b/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java
index f270d1e..052b761 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDropTable.java
@@ -37,17 +37,16 @@ public class TestDropTable extends PlanTestBase {
@Test
public void testDropJsonTable() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("alter session set `store.format` = 'json'");
final String tableName = "simple_json";
// create a json table
- test(String.format(CREATE_SIMPLE_TABLE, tableName));
+ test(CREATE_SIMPLE_TABLE, tableName);
// drop the table
- final String dropSql = String.format(DROP_TABLE, tableName);
testBuilder()
- .sqlQuery(dropSql)
+ .sqlQuery(DROP_TABLE, tableName)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Table [%s] dropped", tableName))
@@ -56,16 +55,15 @@ public class TestDropTable extends PlanTestBase {
@Test
public void testDropParquetTable() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
final String tableName = "simple_json";
// create a parquet table
- test(String.format(CREATE_SIMPLE_TABLE, tableName));
+ test(CREATE_SIMPLE_TABLE, tableName);
// drop the table
- final String dropSql = String.format(DROP_TABLE, tableName);
testBuilder()
- .sqlQuery(dropSql)
+ .sqlQuery(DROP_TABLE, tableName)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Table [%s] dropped", tableName))
@@ -74,18 +72,16 @@ public class TestDropTable extends PlanTestBase {
@Test
public void testDropTextTable() throws Exception {
- test("use dfs_test.tmp");
-
+ test("use dfs.tmp");
test("alter session set `store.format` = 'csv'");
final String csvTable = "simple_csv";
// create a csv table
- test(String.format(CREATE_SIMPLE_TABLE, csvTable));
+ test(CREATE_SIMPLE_TABLE, csvTable);
// drop the table
- String dropSql = String.format(DROP_TABLE, csvTable);
testBuilder()
- .sqlQuery(dropSql)
+ .sqlQuery(DROP_TABLE, csvTable)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Table [%s] dropped", csvTable))
@@ -95,12 +91,11 @@ public class TestDropTable extends PlanTestBase {
final String psvTable = "simple_psv";
// create a psv table
- test(String.format(CREATE_SIMPLE_TABLE, psvTable));
+ test(CREATE_SIMPLE_TABLE, psvTable);
// drop the table
- dropSql = String.format(DROP_TABLE, psvTable);
testBuilder()
- .sqlQuery(dropSql)
+ .sqlQuery(DROP_TABLE, psvTable)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Table [%s] dropped", psvTable))
@@ -110,12 +105,11 @@ public class TestDropTable extends PlanTestBase {
final String tsvTable = "simple_tsv";
// create a tsv table
- test(String.format(CREATE_SIMPLE_TABLE, tsvTable));
+ test(CREATE_SIMPLE_TABLE, tsvTable);
// drop the table
- dropSql = String.format(DROP_TABLE, tsvTable);
testBuilder()
- .sqlQuery(dropSql)
+ .sqlQuery(DROP_TABLE, tsvTable)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Table [%s] dropped", tsvTable))
@@ -124,7 +118,7 @@ public class TestDropTable extends PlanTestBase {
@Test
public void testNonHomogenousDrop() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
final String tableName = "homogenous_table";
// create a parquet table
@@ -133,14 +127,14 @@ public class TestDropTable extends PlanTestBase {
// create a json table within the same directory
test("alter session set `store.format` = 'json'");
final String nestedJsonTable = tableName + Path.SEPARATOR + "json_table";
- test(String.format(CREATE_SIMPLE_TABLE, BACK_TICK + nestedJsonTable + BACK_TICK));
+ test(CREATE_SIMPLE_TABLE, BACK_TICK + nestedJsonTable + BACK_TICK);
test("show files from " + tableName);
boolean dropFailed = false;
// this should fail, because the directory contains non-homogenous files
try {
- test(String.format(DROP_TABLE, tableName));
+ test(DROP_TABLE, tableName);
} catch (UserException e) {
Assert.assertTrue(e.getMessage().contains("VALIDATION ERROR"));
dropFailed = true;
@@ -150,7 +144,7 @@ public class TestDropTable extends PlanTestBase {
// drop the individual json table
testBuilder()
- .sqlQuery(String.format(DROP_TABLE, BACK_TICK + nestedJsonTable + BACK_TICK))
+ .sqlQuery(DROP_TABLE, BACK_TICK + nestedJsonTable + BACK_TICK)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Table [%s] dropped", nestedJsonTable))
@@ -158,7 +152,7 @@ public class TestDropTable extends PlanTestBase {
// Now drop should succeed
testBuilder()
- .sqlQuery(String.format(DROP_TABLE, tableName))
+ .sqlQuery(DROP_TABLE, tableName)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Table [%s] dropped", tableName))
@@ -169,7 +163,7 @@ public class TestDropTable extends PlanTestBase {
public void testDropOnImmutableSchema() throws Exception {
boolean dropFailed = false;
try {
- test("drop table dfs.`/tmp`");
+ test("drop table dfs.`tmp`");
} catch (UserException e) {
Assert.assertTrue(e.getMessage().contains("VALIDATION ERROR"));
dropFailed = true;
@@ -182,12 +176,12 @@ public class TestDropTable extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testDropTableIfExistsWhileTableExists() throws Exception {
final String existentTableName = "test_table_exists";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
// successful dropping of existent table
- test(String.format(CREATE_SIMPLE_TABLE, existentTableName));
+ test(CREATE_SIMPLE_TABLE, existentTableName);
testBuilder()
- .sqlQuery(String.format(DROP_TABLE_IF_EXISTS, existentTableName))
+ .sqlQuery(DROP_TABLE_IF_EXISTS, existentTableName)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Table [%s] dropped", existentTableName))
@@ -198,11 +192,11 @@ public class TestDropTable extends PlanTestBase {
@Category(UnlikelyTest.class)
public void testDropTableIfExistsWhileTableDoesNotExist() throws Exception {
final String nonExistentTableName = "test_table_not_exists";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
// dropping of non existent table without error
testBuilder()
- .sqlQuery(String.format(DROP_TABLE_IF_EXISTS, nonExistentTableName))
+ .sqlQuery(DROP_TABLE_IF_EXISTS, nonExistentTableName)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(false, String.format("Table [%s] not found", nonExistentTableName))
@@ -214,18 +208,18 @@ public class TestDropTable extends PlanTestBase {
public void testDropTableIfExistsWhileItIsAView() throws Exception {
final String viewName = "test_view";
try{
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
// dropping of non existent table without error if the view with such name is existed
- test(String.format(CREATE_SIMPLE_VIEW, viewName));
+ test(CREATE_SIMPLE_VIEW, viewName);
testBuilder()
- .sqlQuery(String.format(DROP_TABLE_IF_EXISTS, viewName))
+ .sqlQuery(DROP_TABLE_IF_EXISTS, viewName)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(false, String.format("Table [%s] not found", viewName))
.go();
} finally {
- test(String.format(DROP_VIEW_IF_EXISTS, viewName));
+ test(DROP_VIEW_IF_EXISTS, viewName);
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
index f147f21..7fea9e1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
@@ -19,12 +19,14 @@ package org.apache.drill;
import com.google.common.collect.Lists;
import mockit.Deencapsulation;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.drill.categories.SlowTest;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.common.config.CommonConstants;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserRemoteException;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.VersionMismatchException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
@@ -35,24 +37,30 @@ import org.apache.drill.exec.proto.UserBitShared.Registry;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.sys.store.DataChangeVersion;
import org.apache.drill.exec.util.JarUtil;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestBuilder;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
import org.junit.runner.RunWith;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
+import java.io.File;
import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
+import static org.apache.drill.test.HadoopUtils.hadoopToJavaPath;
import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -72,29 +80,52 @@ import static org.mockito.Mockito.verify;
@Category({SlowTest.class, SqlFunctionTest.class})
public class TestDynamicUDFSupport extends BaseTestQuery {
- private static final Path jars = new Path(TestTools.getWorkingPath(), "src/test/resources/jars");
+ private static final Path jars = TestTools.WORKING_PATH
+ .resolve(TestTools.TEST_RESOURCES)
+ .resolve("jars");
private static final String default_binary_name = "DrillUDF-1.0.jar";
+ private static final String UDF_SUB_DIR = "udf";
private static final String default_source_name = JarUtil.getSourceName(default_binary_name);
-
- @Rule
- public final TemporaryFolder base = new TemporaryFolder();
-
- private static FileSystem localFileSystem;
+ private static URI fsUri;
+ private static File udfDir;
@BeforeClass
- public static void init() throws IOException {
- localFileSystem = getLocalFileSystem();
- }
+ public static void setup() throws IOException {
+ udfDir = dirTestWatcher.makeSubDir(Paths.get(UDF_SUB_DIR));
- @Before
- public void setup() {
Properties overrideProps = new Properties();
- overrideProps.setProperty(ExecConstants.UDF_DIRECTORY_ROOT, base.getRoot().getPath());
- overrideProps.setProperty(ExecConstants.DRILL_TMP_DIR, base.getRoot().getPath());
+ overrideProps.setProperty(ExecConstants.UDF_DIRECTORY_ROOT, udfDir.getAbsolutePath());
overrideProps.setProperty(ExecConstants.UDF_DIRECTORY_FS, FileSystem.DEFAULT_FS);
updateTestCluster(1, DrillConfig.create(overrideProps));
+
+ fsUri = getLocalFileSystem().getUri();
}
+ @Rule
+ public final TestWatcher clearDirs = new TestWatcher() {
+ @Override
+ protected void succeeded(Description description) {
+ reset();
+ }
+
+ @Override
+ protected void failed(Throwable e, Description description) {
+ reset();
+ }
+
+ private void reset() {
+ try {
+ closeClient();
+ FileUtils.cleanDirectory(udfDir);
+ dirTestWatcher.clear();
+ setupDefaultTestCluster();
+ setup();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
@Test
public void testSyntax() throws Exception {
test("create function using jar 'jar_name.jar'");
@@ -132,11 +163,10 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
@Test
public void testAbsentBinaryInStaging() throws Exception {
- Path staging = getDrillbitContext().getRemoteFunctionRegistry().getStagingArea();
- FileSystem fs = getDrillbitContext().getRemoteFunctionRegistry().getFs();
+ final Path staging = hadoopToJavaPath(getDrillbitContext().getRemoteFunctionRegistry().getStagingArea());
String summary = String.format("File %s does not exist on file system %s",
- new Path(staging, default_binary_name).toUri().getPath(), fs.getUri());
+ staging.resolve(default_binary_name).toUri().getPath(), fsUri);
testBuilder()
.sqlQuery("create function using jar '%s'", default_binary_name)
@@ -148,12 +178,12 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
@Test
public void testAbsentSourceInStaging() throws Exception {
- Path staging = getDrillbitContext().getRemoteFunctionRegistry().getStagingArea();
- FileSystem fs = getDrillbitContext().getRemoteFunctionRegistry().getFs();
- copyJar(fs, jars, staging, default_binary_name);
+ final Path staging = hadoopToJavaPath(getDrillbitContext().getRemoteFunctionRegistry().getStagingArea());
+
+ copyJar(jars, staging, default_binary_name);
String summary = String.format("File %s does not exist on file system %s",
- new Path(staging, default_source_name).toUri().getPath(), fs.getUri());
+ staging.resolve(default_source_name).toUri().getPath(), fsUri);
testBuilder()
.sqlQuery("create function using jar '%s'", default_binary_name)
@@ -214,10 +244,12 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
assertFalse("Staging area should be empty", fs.listFiles(remoteFunctionRegistry.getStagingArea(), false).hasNext());
assertFalse("Temporary area should be empty", fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
+ final Path path = hadoopToJavaPath(remoteFunctionRegistry.getRegistryArea());
+
assertTrue("Binary should be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
+ path.resolve(default_binary_name).toFile().exists());
assertTrue("Source should be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
+ path.resolve(default_source_name).toFile().exists());
Registry registry = remoteFunctionRegistry.getRegistry(new DataChangeVersion());
assertEquals("Registry should contain one jar", registry.getJarList().size(), 1);
@@ -243,8 +275,10 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
@Test
public void testDuplicatedJarInLocalRegistry() throws Exception {
copyDefaultJarsToStagingArea();
+
test("create function using jar '%s'", default_binary_name);
test("select custom_lower('A') from (values(1))");
+
copyDefaultJarsToStagingArea();
String summary = "Jar with %s name has been already registered";
@@ -291,7 +325,11 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
@Test
public void testSuccessfulRegistrationAfterSeveralRetryAttempts() throws Exception {
- RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
+ final RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
+ final Path registryPath = hadoopToJavaPath(remoteFunctionRegistry.getRegistryArea());
+ final Path stagingPath = hadoopToJavaPath(remoteFunctionRegistry.getStagingArea());
+ final Path tmpPath = hadoopToJavaPath(remoteFunctionRegistry.getTmpArea());
+
copyDefaultJarsToStagingArea();
doThrow(new VersionMismatchException("Version mismatch detected", 1))
@@ -312,15 +350,13 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
verify(remoteFunctionRegistry, times(3))
.updateRegistry(any(Registry.class), any(DataChangeVersion.class));
- FileSystem fs = remoteFunctionRegistry.getFs();
-
- assertFalse("Staging area should be empty", fs.listFiles(remoteFunctionRegistry.getStagingArea(), false).hasNext());
- assertFalse("Temporary area should be empty", fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
+ assertTrue("Staging area should be empty", ArrayUtils.isEmpty(stagingPath.toFile().listFiles()));
+ assertTrue("Temporary area should be empty", ArrayUtils.isEmpty(tmpPath.toFile().listFiles()));
assertTrue("Binary should be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
+ registryPath.resolve(default_binary_name).toFile().exists());
assertTrue("Source should be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
+ registryPath.resolve(default_source_name).toFile().exists());
Registry registry = remoteFunctionRegistry.getRegistry(new DataChangeVersion());
assertEquals("Registry should contain one jar", registry.getJarList().size(), 1);
@@ -361,7 +397,11 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
@Test
public void testExceedRetryAttemptsDuringRegistration() throws Exception {
- RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
+ final RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
+ final Path registryPath = hadoopToJavaPath(remoteFunctionRegistry.getRegistryArea());
+ final Path stagingPath = hadoopToJavaPath(remoteFunctionRegistry.getStagingArea());
+ final Path tmpPath = hadoopToJavaPath(remoteFunctionRegistry.getTmpArea());
+
copyDefaultJarsToStagingArea();
doThrow(new VersionMismatchException("Version mismatch detected", 1))
@@ -379,17 +419,13 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
verify(remoteFunctionRegistry, times(remoteFunctionRegistry.getRetryAttempts() + 1))
.updateRegistry(any(Registry.class), any(DataChangeVersion.class));
- FileSystem fs = remoteFunctionRegistry.getFs();
-
assertTrue("Binary should be present in staging area",
- fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_binary_name)));
+ stagingPath.resolve(default_binary_name).toFile().exists());
assertTrue("Source should be present in staging area",
- fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_source_name)));
+ stagingPath.resolve(default_source_name).toFile().exists());
- assertFalse("Registry area should be empty",
- fs.listFiles(remoteFunctionRegistry.getRegistryArea(), false).hasNext());
- assertFalse("Temporary area should be empty",
- fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
+ assertTrue("Registry area should be empty", ArrayUtils.isEmpty(registryPath.toFile().listFiles()));
+ assertTrue("Temporary area should be empty", ArrayUtils.isEmpty(tmpPath.toFile().listFiles()));
assertEquals("Registry should be empty",
remoteFunctionRegistry.getRegistry(new DataChangeVersion()).getJarList().size(), 0);
@@ -397,7 +433,9 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
@Test
public void testExceedRetryAttemptsDuringUnregistration() throws Exception {
- RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
+ final RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
+ final Path registryPath = hadoopToJavaPath(remoteFunctionRegistry.getRegistryArea());
+
copyDefaultJarsToStagingArea();
test("create function using jar '%s'", default_binary_name);
@@ -417,12 +455,10 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
verify(remoteFunctionRegistry, times(remoteFunctionRegistry.getRetryAttempts() + 1))
.updateRegistry(any(Registry.class), any(DataChangeVersion.class));
- FileSystem fs = remoteFunctionRegistry.getFs();
-
assertTrue("Binary should be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
+ registryPath.resolve(default_binary_name).toFile().exists());
assertTrue("Source should be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
+ registryPath.resolve(default_source_name).toFile().exists());
Registry registry = remoteFunctionRegistry.getRegistry(new DataChangeVersion());
assertEquals("Registry should contain one jar", registry.getJarList().size(), 1);
@@ -446,13 +482,13 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
.baselineValues("a")
.go();
- Path localUdfDirPath = Deencapsulation.getField(
- getDrillbitContext().getFunctionImplementationRegistry(), "localUdfDir");
+ Path localUdfDirPath = hadoopToJavaPath((org.apache.hadoop.fs.Path) Deencapsulation.getField(
+ getDrillbitContext().getFunctionImplementationRegistry(), "localUdfDir"));
assertTrue("Binary should exist in local udf directory",
- localFileSystem.exists(new Path(localUdfDirPath, default_binary_name)));
+ localUdfDirPath.resolve(default_binary_name).toFile().exists());
assertTrue("Source should exist in local udf directory",
- localFileSystem.exists(new Path(localUdfDirPath, default_source_name)));
+ localUdfDirPath.resolve(default_source_name).toFile().exists());
}
@Test
@@ -513,13 +549,13 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
test("create function using jar '%s'", default_binary_name);
test("select custom_lower('A') from (values(1))");
- Path localUdfDirPath = Deencapsulation.getField(
- getDrillbitContext().getFunctionImplementationRegistry(), "localUdfDir");
+ Path localUdfDirPath = hadoopToJavaPath((org.apache.hadoop.fs.Path)Deencapsulation.getField(
+ getDrillbitContext().getFunctionImplementationRegistry(), "localUdfDir"));
assertTrue("Binary should exist in local udf directory",
- localFileSystem.exists(new Path(localUdfDirPath, default_binary_name)));
+ localUdfDirPath.resolve(default_binary_name).toFile().exists());
assertTrue("Source should exist in local udf directory",
- localFileSystem.exists(new Path(localUdfDirPath, default_source_name)));
+ localUdfDirPath.resolve(default_source_name).toFile().exists());
String summary = "The following UDFs in jar %s have been unregistered:\n" +
"[custom_lower(VARCHAR-REQUIRED)]";
@@ -537,20 +573,21 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
assertThat(e.getMessage(), containsString("No match found for function signature custom_lower(<CHARACTER>)"));
}
- RemoteFunctionRegistry remoteFunctionRegistry = getDrillbitContext().getRemoteFunctionRegistry();
+ final RemoteFunctionRegistry remoteFunctionRegistry = getDrillbitContext().getRemoteFunctionRegistry();
+ final Path registryPath = hadoopToJavaPath(remoteFunctionRegistry.getRegistryArea());
+
assertEquals("Remote registry should be empty",
remoteFunctionRegistry.getRegistry(new DataChangeVersion()).getJarList().size(), 0);
- FileSystem fs = remoteFunctionRegistry.getFs();
assertFalse("Binary should not be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
+ registryPath.resolve(default_binary_name).toFile().exists());
assertFalse("Source should not be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
+ registryPath.resolve(default_source_name).toFile().exists());
assertFalse("Binary should not be present in local udf directory",
- localFileSystem.exists(new Path(localUdfDirPath, default_binary_name)));
+ localUdfDirPath.resolve(default_binary_name).toFile().exists());
assertFalse("Source should not be present in local udf directory",
- localFileSystem.exists(new Path(localUdfDirPath, default_source_name)));
+ localUdfDirPath.resolve(default_source_name).toFile().exists());
}
@Test
@@ -567,7 +604,7 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
Thread.sleep(1000);
- Path src = new Path(jars, "v2");
+ Path src = jars.resolve("v2");
copyJarsToStagingArea(src, default_binary_name, default_source_name);
test("create function using jar '%s'", default_binary_name);
testBuilder()
@@ -593,15 +630,18 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
@Test
public void testRegistrationFailDuringRegistryUpdate() throws Exception {
final RemoteFunctionRegistry remoteFunctionRegistry = spyRemoteFunctionRegistry();
- final FileSystem fs = remoteFunctionRegistry.getFs();
+ final Path registryPath = hadoopToJavaPath(remoteFunctionRegistry.getRegistryArea());
+ final Path stagingPath = hadoopToJavaPath(remoteFunctionRegistry.getStagingArea());
+ final Path tmpPath = hadoopToJavaPath(remoteFunctionRegistry.getTmpArea());
+
final String errorMessage = "Failure during remote registry update.";
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
assertTrue("Binary should be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_binary_name)));
+ registryPath.resolve(default_binary_name).toFile().exists());
assertTrue("Source should be present in registry area",
- fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
+ registryPath.resolve(default_source_name).toFile().exists());
throw new RuntimeException(errorMessage);
}
}).when(remoteFunctionRegistry).updateRegistry(any(Registry.class), any(DataChangeVersion.class));
@@ -615,15 +655,11 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
.baselineValues(false, errorMessage)
.go();
- assertFalse("Registry area should be empty",
- fs.listFiles(remoteFunctionRegistry.getRegistryArea(), false).hasNext());
- assertFalse("Temporary area should be empty",
- fs.listFiles(remoteFunctionRegistry.getTmpArea(), false).hasNext());
+ assertTrue("Registry area should be empty", ArrayUtils.isEmpty(registryPath.toFile().listFiles()));
+ assertTrue("Temporary area should be empty", ArrayUtils.isEmpty(tmpPath.toFile().listFiles()));
- assertTrue("Binary should be present in staging area",
- fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_binary_name)));
- assertTrue("Source should be present in staging area",
- fs.exists(new Path(remoteFunctionRegistry.getStagingArea(), default_source_name)));
+ assertTrue("Binary should be present in staging area", stagingPath.resolve(default_binary_name).toFile().exists());
+ assertTrue("Source should be present in staging area", stagingPath.resolve(default_source_name).toFile().exists());
}
@Test
@@ -914,13 +950,17 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
private void copyJarsToStagingArea(Path src, String binaryName, String sourceName) throws IOException {
RemoteFunctionRegistry remoteFunctionRegistry = getDrillbitContext().getRemoteFunctionRegistry();
- copyJar(remoteFunctionRegistry.getFs(), src, remoteFunctionRegistry.getStagingArea(), binaryName);
- copyJar(remoteFunctionRegistry.getFs(), src, remoteFunctionRegistry.getStagingArea(), sourceName);
+
+ final Path path = hadoopToJavaPath(remoteFunctionRegistry.getStagingArea());
+
+ copyJar(src, path, binaryName);
+ copyJar(src, path, sourceName);
}
- private void copyJar(FileSystem fs, Path src, Path dest, String name) throws IOException {
- Path jarPath = new Path(src, name);
- fs.copyFromLocalFile(jarPath, dest);
+ private void copyJar(Path src, Path dest, String name) throws IOException {
+ final File destFile = dest.resolve(name).toFile();
+ FileUtils.deleteQuietly(destFile);
+ FileUtils.copyFile(src.resolve(name).toFile(), destFile);
}
private RemoteFunctionRegistry spyRemoteFunctionRegistry() {
[10/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/control/TestCustomTunnel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/control/TestCustomTunnel.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/control/TestCustomTunnel.java
index 9770a7e..bc9ba9d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/control/TestCustomTunnel.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/control/TestCustomTunnel.java
@@ -27,7 +27,7 @@ import io.netty.util.internal.ThreadLocalRandom;
import java.util.Arrays;
import java.util.Random;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.rpc.UserRpcException;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java
index 12cd823..81b027f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/data/TestBitBitKerberos.java
@@ -26,7 +26,7 @@ import mockit.Injectable;
import mockit.Mock;
import mockit.MockUp;
import mockit.NonStrictExpectations;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SecurityTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.DrillProperties;
@@ -96,7 +96,7 @@ public class TestBitBitKerberos extends BaseTestQuery {
final Config config = DrillConfig.create(cloneDefaultTestConfigProperties());
krbHelper = new KerberosHelper(TestBitBitKerberos.class.getSimpleName());
- krbHelper.setupKdc();
+ krbHelper.setupKdc(dirTestWatcher.getTmpDir());
newConfig = new DrillConfig(
config.withValue(ExecConstants.AUTHENTICATION_MECHANISMS,
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/security/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/security/KerberosHelper.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/security/KerberosHelper.java
index 3320cef..451e0aa 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/security/KerberosHelper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/security/KerberosHelper.java
@@ -26,8 +26,6 @@ import java.io.IOException;
import java.net.ServerSocket;
import java.nio.file.Files;
-import static org.apache.drill.exec.ExecTest.getTempDir;
-
public class KerberosHelper {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(KerberosHelper.class);
@@ -59,9 +57,9 @@ public class KerberosHelper {
this.testName = testName;
}
- public void setupKdc() throws Exception {
+ public void setupKdc(File workspace) throws Exception {
+ this.workspace = workspace;
kdc = new SimpleKdcServer();
- workspace = new File(getTempDir("kerberos_target"));
kdcDir = new File(workspace, testName);
if(!kdcDir.mkdirs()) {
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java
index 9b551ce..bbe3537 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/TemporaryTablesAutomaticDropTest.java
@@ -19,31 +19,27 @@ package org.apache.drill.exec.rpc.user;
import mockit.Mock;
import mockit.MockUp;
-import mockit.integration.junit4.JMockit;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.store.StoragePluginRegistry;
-import org.apache.drill.exec.util.TestUtilities;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.drill.exec.util.StoragePluginTestUtils;
+import org.apache.drill.test.DirTestWatcher;
import org.junit.Before;
import org.junit.Test;
-import org.junit.runner.RunWith;
+import java.io.File;
import java.util.Properties;
import java.util.UUID;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-@RunWith(JMockit.class)
public class TemporaryTablesAutomaticDropTest extends BaseTestQuery {
private static final String session_id = "sessionId";
- private FileSystem fs;
-
@Before
public void setup() throws Exception {
new MockUp<UUID>() {
@@ -52,53 +48,56 @@ public class TemporaryTablesAutomaticDropTest extends BaseTestQuery {
return UUID.nameUUIDFromBytes(session_id.getBytes());
}
};
+
Properties testConfigurations = cloneDefaultTestConfigProperties();
- testConfigurations.put(ExecConstants.DEFAULT_TEMPORARY_WORKSPACE, TEMP_SCHEMA);
+ testConfigurations.put(ExecConstants.DEFAULT_TEMPORARY_WORKSPACE, DFS_TMP_SCHEMA);
updateTestCluster(1, DrillConfig.create(testConfigurations));
-
- fs = getLocalFileSystem();
}
@Test
public void testAutomaticDropWhenClientIsClosed() throws Exception {
- Path sessionTemporaryLocation = createAndCheckSessionTemporaryLocation("client_closed",
- getDfsTestTmpSchemaLocation());
+ final File sessionTemporaryLocation =
+ createAndCheckSessionTemporaryLocation("client_closed", dirTestWatcher.getDfsTestTmpDir());
+
updateClient("new_client");
- assertFalse("Session temporary location should be absent", fs.exists(sessionTemporaryLocation));
+ assertFalse("Session temporary location should be absent", sessionTemporaryLocation.exists());
}
@Test
public void testAutomaticDropWhenDrillbitIsClosed() throws Exception {
- Path sessionTemporaryLocation = createAndCheckSessionTemporaryLocation("drillbit_closed",
- getDfsTestTmpSchemaLocation());
+ final File sessionTemporaryLocation =
+ createAndCheckSessionTemporaryLocation("drillbit_closed", dirTestWatcher.getDfsTestTmpDir());
bits[0].close();
- assertFalse("Session temporary location should be absent", fs.exists(sessionTemporaryLocation));
+ assertFalse("Session temporary location should be absent", sessionTemporaryLocation.exists());
}
@Test
public void testAutomaticDropOfSeveralSessionTemporaryLocations() throws Exception {
- Path firstSessionTemporaryLocation = createAndCheckSessionTemporaryLocation("first_location",
- getDfsTestTmpSchemaLocation());
- StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
- String tempDir = TestUtilities.createTempDir();
+ final File firstSessionTemporaryLocation =
+ createAndCheckSessionTemporaryLocation("first_location", dirTestWatcher.getDfsTestTmpDir());
+ final StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
+ final File tempDir = DirTestWatcher.createTempDir(dirTestWatcher.getDir());
+
try {
- TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, tempDir);
- Path secondSessionTemporaryLocation = createAndCheckSessionTemporaryLocation("second_location", tempDir);
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry, tempDir);
+ final File secondSessionTemporaryLocation = createAndCheckSessionTemporaryLocation("second_location", tempDir);
updateClient("new_client");
- assertFalse("First session temporary location should be absent", fs.exists(firstSessionTemporaryLocation));
- assertFalse("Second session temporary location should be absent", fs.exists(secondSessionTemporaryLocation));
+ assertFalse("First session temporary location should be absent", firstSessionTemporaryLocation.exists());
+ assertFalse("Second session temporary location should be absent", secondSessionTemporaryLocation.exists());
} finally {
- TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, getDfsTestTmpSchemaLocation());
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry, dirTestWatcher.getDfsTestTmpDir());
}
}
- private Path createAndCheckSessionTemporaryLocation(String suffix, String schemaLocation) throws Exception {
- String temporaryTableName = "temporary_table_automatic_drop_" + suffix;
- test("create TEMPORARY table %s.%s as select 'A' as c1 from (values(1))", TEMP_SCHEMA, temporaryTableName);
- Path sessionTemporaryLocation = new Path(schemaLocation,
- UUID.nameUUIDFromBytes(session_id.getBytes()).toString());
- assertTrue("Session temporary location should exist", fs.exists(sessionTemporaryLocation));
+ private File createAndCheckSessionTemporaryLocation(String suffix, File schemaLocation) throws Exception {
+ final String temporaryTableName = "temporary_table_automatic_drop_" + suffix;
+ final File sessionTemporaryLocation = schemaLocation
+ .toPath()
+ .resolve(UUID.nameUUIDFromBytes(session_id.getBytes()).toString())
+ .toFile();
+
+ test("create TEMPORARY table %s.%s as select 'A' as c1 from (values(1))", DFS_TMP_SCHEMA, temporaryTableName);
+ assertTrue("Session temporary location should exist", sessionTemporaryLocation.exists());
return sessionTemporaryLocation;
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java
index b4c0eb2..97360d3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.rpc.user.security;
import com.typesafe.config.ConfigValueFactory;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SecurityTest;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.common.config.DrillConfig;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
index b1a6e98..d6495e9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.rpc.user.security;
import com.google.common.collect.Lists;
import com.typesafe.config.ConfigValueFactory;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SecurityTest;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.common.config.DrillConfig;
@@ -57,7 +57,7 @@ public class TestUserBitKerberos extends BaseTestQuery {
public static void setupTest() throws Exception {
krbHelper = new KerberosHelper(TestUserBitKerberos.class.getSimpleName());
- krbHelper.setupKdc();
+ krbHelper.setupKdc(dirTestWatcher.getTmpDir());
// Create a new DrillConfig which has user authentication enabled and authenticator set to
// UserAuthenticatorTestImpl.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
index 2ddbcef..4f411ae 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.rpc.user.security;
import com.google.common.collect.Lists;
import com.typesafe.config.ConfigValueFactory;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SecurityTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.DrillProperties;
@@ -61,7 +61,7 @@ public class TestUserBitKerberosEncryption extends BaseTestQuery {
@BeforeClass
public static void setupTest() throws Exception {
krbHelper = new KerberosHelper(TestUserBitKerberosEncryption.class.getSimpleName());
- krbHelper.setupKdc();
+ krbHelper.setupKdc(dirTestWatcher.getTmpDir());
// Create a new DrillConfig which has user authentication enabled and authenticator set to
// UserAuthenticatorTestImpl.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java
index 2228e30..2fb08e3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSL.java
@@ -20,10 +20,10 @@ package org.apache.drill.exec.rpc.user.security;
import com.typesafe.config.ConfigValueFactory;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import junit.framework.TestCase;
-import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -39,9 +39,6 @@ import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;
public class TestUserBitSSL extends BaseTestQuery {
- private static final org.slf4j.Logger logger =
- org.slf4j.LoggerFactory.getLogger(TestUserBitSSL.class);
-
private static DrillConfig newConfig;
private static Properties initProps; // initial client properties
private static ClassLoader classLoader;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java
index 890a924..a2cb675 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSSLServer.java
@@ -18,41 +18,31 @@
package org.apache.drill.exec.rpc.user.security;
import com.typesafe.config.ConfigValueFactory;
-import junit.framework.TestCase;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.ExecConstants;
-import org.apache.hadoop.conf.Configuration;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
-import java.text.MessageFormat;
import java.util.Properties;
-import static org.apache.drill.exec.ssl.SSLConfig.HADOOP_SSL_CONF_TPL_KEY;
import static org.junit.Assert.assertEquals;
public class TestUserBitSSLServer extends BaseTestQuery {
- private static final org.slf4j.Logger logger =
- org.slf4j.LoggerFactory.getLogger(TestUserBitSSLServer.class);
-
private static DrillConfig sslConfig;
private static Properties initProps; // initial client properties
private static ClassLoader classLoader;
private static String ksPath;
private static String tsPath;
- private static String emptyTSPath;
@BeforeClass
public static void setupTest() throws Exception {
-
classLoader = TestUserBitSSLServer.class.getClassLoader();
ksPath = new File(classLoader.getResource("ssl/keystore.ks").getFile()).getAbsolutePath();
tsPath = new File(classLoader.getResource("ssl/truststore.ks").getFile()).getAbsolutePath();
- emptyTSPath = new File(classLoader.getResource("ssl/emptytruststore.ks").getFile()).getAbsolutePath();
sslConfig = new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties())
.withValue(ExecConstants.USER_SSL_ENABLED, ConfigValueFactory.fromAnyRef(true))
.withValue(ExecConstants.SSL_KEYSTORE_TYPE, ConfigValueFactory.fromAnyRef("JKS"))
@@ -67,13 +57,6 @@ public class TestUserBitSSLServer extends BaseTestQuery {
initProps.setProperty(DrillProperties.DISABLE_HOST_VERIFICATION, "true");
}
- @AfterClass
- public static void cleanTest() throws Exception {
- DrillConfig restoreConfig =
- new DrillConfig(DrillConfig.create(cloneDefaultTestConfigProperties()), false);
- updateTestCluster(1, restoreConfig);
- }
-
@Test
public void testInvalidKeystorePath() throws Exception {
DrillConfig testConfig = new DrillConfig(DrillConfig.create(sslConfig)
@@ -138,5 +121,4 @@ public class TestUserBitSSLServer extends BaseTestQuery {
}
assertEquals(failureCaught, false);
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java
index bbb957d..1ca38f5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitSaslCompatibility.java
@@ -17,15 +17,14 @@
*/
package org.apache.drill.exec.rpc.user.security;
-
import com.google.common.collect.Lists;
import com.typesafe.config.ConfigValueFactory;
-import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.rpc.NonTransientRpcException;
import org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index 1548d7d..92d91f7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -31,8 +31,8 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.math3.util.Pair;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.QueryTestUtil;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.QueryTestUtil;
import org.apache.drill.SingleRowListener;
import org.apache.drill.common.DrillAutoCloseables;
import org.apache.drill.common.concurrent.ExtendedLatch;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java
index 00e356e..0a7f731 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptions.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.server;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.OptionsTest;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.test.UserExceptionMatcher;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java
index 82a524a..61eb85d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.server;
import com.google.common.base.Joiner;
import com.typesafe.config.ConfigValueFactory;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.util.DrillStringUtils;
@@ -42,10 +42,6 @@ import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorT
import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_1_PASSWORD;
import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_2;
import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_2_PASSWORD;
-
-
-
-
import java.util.Properties;
/**
@@ -129,7 +125,7 @@ public class TestOptionsAuthEnabled extends BaseTestQuery {
@Test
public void testAdminUserOptions() throws Exception {
- try (ClusterFixture cluster = ClusterFixture.standardCluster();
+ try (ClusterFixture cluster = ClusterFixture.standardCluster(dirTestWatcher);
ClientFixture client = cluster.clientFixture()) {
OptionManager optionManager = cluster.drillbit().getContext().getOptionManager();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java
index ba19e0d..926024a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestTpcdsSf1Leaks.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.server;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import static org.apache.drill.exec.ExecConstants.SLICE_TARGET;
import static org.apache.drill.exec.ExecConstants.SLICE_TARGET_DEFAULT;
@@ -34,7 +34,7 @@ import org.junit.rules.Timeout;
* To run this unit class you need to download the following data file:
* http://apache-drill.s3.amazonaws.com/files/tpcds-sf1-parquet.tgz
* and untar it in a some folder (e.g. /tpcds-sf1-parquet) then add the following workspace to
- * exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
+ * the dfs storage plugin
*
* ,"tpcds" : {
* location: "/tpcds-sf1-parquet",
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/PersistedOptionValueTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/PersistedOptionValueTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/PersistedOptionValueTest.java
index 182442b..165dad6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/PersistedOptionValueTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/PersistedOptionValueTest.java
@@ -21,9 +21,8 @@ package org.apache.drill.exec.server.options;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.serialization.JacksonSerializer;
-import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.junit.Assert;
import org.junit.Test;
@@ -47,10 +46,10 @@ public class PersistedOptionValueTest {
private void testHelper(String booleanOptionFile, String doubleOptionFile,
String longOptionFile, String stringOptionFile) throws IOException {
JacksonSerializer serializer = new JacksonSerializer<>(new ObjectMapper(), PersistedOptionValue.class);
- String booleanOptionJson = FileUtils.getResourceAsString(booleanOptionFile);
- String doubleOptionJson = FileUtils.getResourceAsString(doubleOptionFile);
- String longOptionJson = FileUtils.getResourceAsString(longOptionFile);
- String stringOptionJson = FileUtils.getResourceAsString(stringOptionFile);
+ String booleanOptionJson = DrillFileUtils.getResourceAsString(booleanOptionFile);
+ String doubleOptionJson = DrillFileUtils.getResourceAsString(doubleOptionFile);
+ String longOptionJson = DrillFileUtils.getResourceAsString(longOptionFile);
+ String stringOptionJson = DrillFileUtils.getResourceAsString(stringOptionFile);
PersistedOptionValue booleanValue = (PersistedOptionValue) serializer.deserialize(booleanOptionJson.getBytes());
PersistedOptionValue doubleValue = (PersistedOptionValue) serializer.deserialize(doubleOptionJson.getBytes());
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/TestConfigLinkage.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/TestConfigLinkage.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/TestConfigLinkage.java
new file mode 100644
index 0000000..c263d1d
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/options/TestConfigLinkage.java
@@ -0,0 +1,332 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.server.options;
+
+import org.apache.drill.categories.OptionsTest;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.store.sys.SystemTable;
+import org.apache.drill.test.BaseDirTestWatcher;
+import org.apache.drill.test.ClientFixture;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+/*
+ * Tests to test if the linkage between the two config option systems
+ * i.e., the linkage between boot-config system and system/session options.
+ * Tests to assert if the config options are read in the order of session , system, boot-config.
+ * Max width per node is slightly different from other options since it is set to zero by default
+ * in the config and the option value is computed dynamically everytime if the value is zero
+ * i.e., if the value is not set in system/session.
+ * */
+
+@Category(OptionsTest.class)
+public class TestConfigLinkage {
+ public static final String MOCK_PROPERTY = "mock.prop";
+
+ @Rule
+ public BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
+ public static OptionDefinition createMockPropOptionDefinition() {
+ return new OptionDefinition(new TypeValidators.StringValidator(MOCK_PROPERTY), new OptionMetaData(OptionValue.AccessibleScopes.ALL, false, true));
+ }
+
+ @Test
+ public void testDefaultInternalValue() throws Exception {
+ OptionDefinition optionDefinition = createMockPropOptionDefinition();
+
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a")
+ .putDefinition(optionDefinition);
+
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String mockProp = client.queryBuilder().
+ sql("SELECT string_val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS.getTableName(), MOCK_PROPERTY).singletonString();
+ String mockProp2 = client.queryBuilder().
+ sql("SELECT val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS_VAL.getTableName(), MOCK_PROPERTY).singletonString();
+
+ assertEquals("a", mockProp);
+ assertEquals("a", mockProp2);
+ }
+ }
+
+ @Test
+ public void testDefaultValidatorInternalValue() throws Exception {
+ OptionDefinition optionDefinition = createMockPropOptionDefinition();
+
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).
+ putDefinition(optionDefinition).
+ configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a");
+
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String mockProp = client.queryBuilder().
+ sql("SELECT string_val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS.getTableName(), MOCK_PROPERTY).singletonString();
+ String mockProp2 = client.queryBuilder().
+ sql("SELECT val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS_VAL.getTableName(), MOCK_PROPERTY).singletonString();
+
+ assertEquals("a", mockProp);
+ assertEquals("a", mockProp2);
+ }
+ }
+
+ /* Test if session option takes precedence */
+ @Test
+ public void testSessionOption() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .sessionOption(ExecConstants.SLICE_TARGET, 10);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SESSION'", SystemTable.OPTION_VAL
+ .getTableName())
+ .singletonString();
+ assertEquals(slice_target, "10");
+ }
+ }
+
+ /* Test if system option takes precedence over the boot option */
+ @Test
+ public void testSystemOption() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .systemOption(ExecConstants.SLICE_TARGET, 10000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ assertEquals(slice_target, "10000");
+ }
+ }
+
+ @Test
+ public void testInternalSystemOption() throws Exception {
+ OptionDefinition optionDefinition = createMockPropOptionDefinition();
+
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).
+ putDefinition(optionDefinition).
+ configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
+ systemOption(MOCK_PROPERTY, "blah");
+
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String mockProp = client.queryBuilder().
+ sql("SELECT string_val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS.getTableName(), MOCK_PROPERTY)
+ .singletonString();
+ String mockProp2 = client.queryBuilder().
+ sql("SELECT val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS_VAL.getTableName(), MOCK_PROPERTY)
+ .singletonString();
+
+ assertEquals("blah", mockProp);
+ assertEquals("blah", mockProp2);
+ }
+ }
+
+ /* Test if config option takes precedence if config option is not set */
+ @Test
+ public void testConfigOption() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .setOptionDefault(ExecConstants.SLICE_TARGET, 1000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'BOOT'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ assertEquals(slice_target, "1000");
+ }
+ }
+
+ /* Test if altering system option takes precedence over config option */
+ @Test
+ public void testAlterSystem() throws Exception {
+ try (ClusterFixture cluster = ClusterFixture.standardCluster(dirTestWatcher);
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SYSTEM SET `planner.slice_target` = 10000").run();
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ assertEquals(slice_target, "10000");
+ }
+ }
+
+ /* Test if altering session option takes precedence over system option */
+ @Test
+ public void testSessionPrecedence() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ .systemOption(ExecConstants.SLICE_TARGET, 100000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SESSION SET `planner.slice_target` = 10000").run();
+ String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SESSION'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ assertEquals(slice_target, "10000");
+ }
+ }
+
+ /* Test if setting maxwidth option through config takes effect */
+ @Test
+ public void testMaxWidthPerNodeConfig() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher)
+ .setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'BOOT'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ assertEquals("2", maxWidth);
+ }
+ }
+
+ /* Test if setting maxwidth at system level takes precedence */
+ @Test
+ public void testMaxWidthPerNodeSystem() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher)
+ .systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ assertEquals("3", maxWidth);
+ }
+ }
+
+ /* Test if setting maxwidth at session level takes precedence */
+ @Test
+ public void testMaxWidthPerNodeSession() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher)
+ .sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'SESSION'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ assertEquals("2", maxWidth);
+ }
+ }
+
+ /* Test if max width is computed correctly using the cpu load average
+ when the option is not set at either system or session level
+ */
+ @Test
+ public void testMaxWidthPerNodeDefault() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher)
+ .setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ long maxWidth = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(0.70, 0);
+ int availProc = Runtime.getRuntime().availableProcessors();
+ long maxWidthPerNode = Math.max(1, Math.min(availProc, Math.round(availProc * 0.70)));
+ assertEquals(maxWidthPerNode, maxWidth);
+ }
+ }
+
+ /* Test if the scope is set during BOOT time and scope is actually BOOT */
+ @Test
+ public void testScope() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher)
+ .setOptionDefault(ExecConstants.SLICE_TARGET, 100000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ Assert.assertEquals("BOOT",scope);
+ }
+ }
+
+ /* Test if the option is set at SYSTEM scope and the scope is actually SYSTEM */
+ @Test
+ public void testScopeSystem() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher)
+ .systemOption(ExecConstants.SLICE_TARGET, 10000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ Assert.assertEquals("SYSTEM",scope);
+ }
+ }
+
+ /* Test if the option is set at SESSION scope and the scope is actually SESSION */
+ @Test
+ public void testScopeSession() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher)
+ .sessionOption(ExecConstants.SLICE_TARGET, 100000);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ Assert.assertEquals("SESSION",scope);
+ }
+ }
+
+ /* Test if the option is altered at SYSTEM scope and the scope is actually SYSTEM */
+ @Test
+ public void testScopeAlterSystem() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SYSTEM set `planner.slice_target`= 10000").run();
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ Assert.assertEquals("SYSTEM",scope);
+ }
+ }
+
+ /* Test if the option is altered at SESSION scope and the scope is actually SESSION */
+ @Test
+ public void testScopeAlterSession() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher);
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SESSION set `planner.slice_target`= 10000").run();
+ String scope = client.queryBuilder()
+ .sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTION_VAL.getTableName())
+ .singletonString();
+ Assert.assertEquals("SESSION",scope);
+ }
+ }
+
+ @Test
+ public void testAlterInternalSystemOption() throws Exception {
+ OptionDefinition optionDefinition = createMockPropOptionDefinition();
+
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).
+ configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
+ putDefinition(optionDefinition);
+
+ try (ClusterFixture cluster = builder.build();
+ ClientFixture client = cluster.clientFixture()) {
+ client.queryBuilder().sql("ALTER SYSTEM SET `%s` = 'bleh'", MOCK_PROPERTY).run();
+
+ client.queryBuilder().sql("SELECT * FROM sys.%s", SystemTable.INTERNAL_OPTIONS.getTableName()).printCsv();
+ client.queryBuilder().sql("SELECT * FROM sys.%s", SystemTable.INTERNAL_OPTIONS_VAL.getTableName()).printCsv();
+
+ String mockProp = client.queryBuilder().
+ sql("SELECT string_val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS, MOCK_PROPERTY).singletonString();
+ String mockProp2 = client.queryBuilder().
+ sql("SELECT val FROM sys.%s where name='%s'", SystemTable.INTERNAL_OPTIONS_VAL, MOCK_PROPERTY).singletonString();
+
+ assertEquals("bleh", mockProp);
+ assertEquals("bleh", mockProp2);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java
index 4f4390f..fc04f88 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java
@@ -20,23 +20,27 @@ package org.apache.drill.exec.server.rest;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.server.options.OptionDefinition;
-import org.apache.drill.exec.server.options.OptionValidator;
+import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.test.RestClientFixture;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
-import static org.apache.drill.test.TestConfigLinkage.MOCK_PROPERTY;
-import static org.apache.drill.test.TestConfigLinkage.createMockPropOptionDefinition;
+import static org.apache.drill.exec.server.options.TestConfigLinkage.MOCK_PROPERTY;
+import static org.apache.drill.exec.server.options.TestConfigLinkage.createMockPropOptionDefinition;
public class StatusResourcesTest {
+ @Rule
+ public final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
@Test
public void testRetrieveInternalOption() throws Exception {
OptionDefinition optionDefinition = createMockPropOptionDefinition();
- ClusterFixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).
configProperty(ExecConstants.HTTP_ENABLE, true).
configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
configProperty(ExecConstants.HTTP_PORT_HUNT, true).
@@ -60,7 +64,7 @@ public class StatusResourcesTest {
@Test
public void testRetrievePublicOption() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).
configProperty(ExecConstants.HTTP_ENABLE, true).
configProperty(ExecConstants.HTTP_PORT_HUNT, true).
configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, false).
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java
index 5bf55af..c85cbb6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestBaseViewSupport.java
@@ -18,8 +18,8 @@
package org.apache.drill.exec.sql;
import com.google.common.base.Strings;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestBuilder;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
@@ -34,14 +34,14 @@ public class TestBaseViewSupport extends BaseTestQuery {
/**
* Create view with given parameters.
*
- * Current default schema "dfs_test"
+ * Current default schema "dfs"
*
* CREATE VIEW tmp.viewName(f1, f2) AS SELECT * FROM cp.`region.json`
*
* For the above CREATE VIEW query, function parameters are:
* viewSchema = "tmp"
* viewName = "viewName"
- * finalSchema = "dfs_test.tmp"
+ * finalSchema = "dfs.tmp"
* viewFields = "(f1, f2)"
* viewDef = "SELECT * FROM cp.`region.json`"
*
@@ -76,13 +76,13 @@ public class TestBaseViewSupport extends BaseTestQuery {
/**
* Drop view with given parameters.
*
- * Current schema "dfs_test"
+ * Current schema "dfs"
* DROP VIEW tmp.viewName
*
* For the above DROP VIEW query, function parameters values are:
* viewSchema = "tmp"
* "viewName" = "viewName"
- * "finalSchema" = "dfs_test.tmp"
+ * "finalSchema" = "dfs.tmp"
*
* @param viewSchema
* @param viewName
@@ -107,13 +107,13 @@ public class TestBaseViewSupport extends BaseTestQuery {
/**
* Drop view if exists with given parameters.
*
- * Current schema "dfs_test"
+ * Current schema "dfs"
* DROP VIEW IF EXISTS tmp.viewName
*
* For the above DROP VIEW IF EXISTS query, function parameters values are:
* viewSchema = "tmp"
* "viewName" = "viewName"
- * "finalSchema" = "dfs_test.tmp"
+ * "finalSchema" = "dfs.tmp"
* "ifViewExists" = null
*
* @param viewSchema
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java
index 2e3b796..764cef4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTAS.java
@@ -18,8 +18,7 @@
package org.apache.drill.exec.sql;
import com.google.common.collect.Maps;
-import org.apache.commons.io.FileUtils;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.exec.ExecConstants;
@@ -31,7 +30,6 @@ import org.apache.hadoop.fs.Path;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import java.io.File;
import java.util.Map;
import static org.junit.Assert.assertEquals;
@@ -41,7 +39,7 @@ public class TestCTAS extends BaseTestQuery {
@Test // DRILL-2589
@Category(UnlikelyTest.class)
public void withDuplicateColumnsInDef1() throws Exception {
- ctasErrorTestHelper("CREATE TABLE %s.%s AS SELECT region_id, region_id FROM cp.`region.json`",
+ ctasErrorTestHelper("CREATE TABLE dfs.tmp.%s AS SELECT region_id, region_id FROM cp.`region.json`",
String.format("Duplicate column name [%s]", "region_id")
);
}
@@ -49,7 +47,7 @@ public class TestCTAS extends BaseTestQuery {
@Test // DRILL-2589
@Category(UnlikelyTest.class)
public void withDuplicateColumnsInDef2() throws Exception {
- ctasErrorTestHelper("CREATE TABLE %s.%s AS SELECT region_id, sales_city, sales_city FROM cp.`region.json`",
+ ctasErrorTestHelper("CREATE TABLE dfs.tmp.%s AS SELECT region_id, sales_city, sales_city FROM cp.`region.json`",
String.format("Duplicate column name [%s]", "sales_city")
);
}
@@ -58,7 +56,7 @@ public class TestCTAS extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void withDuplicateColumnsInDef3() throws Exception {
ctasErrorTestHelper(
- "CREATE TABLE %s.%s(regionid, regionid) " +
+ "CREATE TABLE dfs.tmp.%s(regionid, regionid) " +
"AS SELECT region_id, sales_city FROM cp.`region.json`",
String.format("Duplicate column name [%s]", "regionid")
);
@@ -68,7 +66,7 @@ public class TestCTAS extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void withDuplicateColumnsInDef4() throws Exception {
ctasErrorTestHelper(
- "CREATE TABLE %s.%s(regionid, salescity, salescity) " +
+ "CREATE TABLE dfs.tmp.%s(regionid, salescity, salescity) " +
"AS SELECT region_id, sales_city, sales_city FROM cp.`region.json`",
String.format("Duplicate column name [%s]", "salescity")
);
@@ -78,7 +76,7 @@ public class TestCTAS extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void withDuplicateColumnsInDef5() throws Exception {
ctasErrorTestHelper(
- "CREATE TABLE %s.%s(regionid, salescity, SalesCity) " +
+ "CREATE TABLE dfs.tmp.%s(regionid, salescity, SalesCity) " +
"AS SELECT region_id, sales_city, sales_city FROM cp.`region.json`",
String.format("Duplicate column name [%s]", "SalesCity")
);
@@ -87,7 +85,7 @@ public class TestCTAS extends BaseTestQuery {
@Test // DRILL-2589
public void whenInEqualColumnCountInTableDefVsInTableQuery() throws Exception {
ctasErrorTestHelper(
- "CREATE TABLE %s.%s(regionid, salescity) " +
+ "CREATE TABLE dfs.tmp.%s(regionid, salescity) " +
"AS SELECT region_id, sales_city, sales_region FROM cp.`region.json`",
"table's field list and the table's query field list have different counts."
);
@@ -96,7 +94,7 @@ public class TestCTAS extends BaseTestQuery {
@Test // DRILL-2589
public void whenTableQueryColumnHasStarAndTableFiledListIsSpecified() throws Exception {
ctasErrorTestHelper(
- "CREATE TABLE %s.%s(regionid, salescity) " +
+ "CREATE TABLE dfs.tmp.%s(regionid, salescity) " +
"AS SELECT region_id, * FROM cp.`region.json`",
"table's query field list has a '*', which is invalid when table's field list is specified."
);
@@ -106,18 +104,10 @@ public class TestCTAS extends BaseTestQuery {
@Category(UnlikelyTest.class)
public void createTableWhenATableWithSameNameAlreadyExists() throws Exception{
final String newTblName = "createTableWhenTableAlreadyExists";
+ final String ctasQuery = String.format("CREATE TABLE dfs.tmp.%s AS SELECT * from cp.`region.json`", newTblName);
- try {
- final String ctasQuery =
- String.format("CREATE TABLE %s.%s AS SELECT * from cp.`region.json`", TEMP_SCHEMA, newTblName);
-
- test(ctasQuery);
-
- errorMsgTestHelper(ctasQuery,
- String.format("A table or view with given name [%s] already exists in schema [%s]", newTblName, TEMP_SCHEMA));
- } finally {
- FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), newTblName));
- }
+ test(ctasQuery);
+ errorMsgTestHelper(ctasQuery, String.format("A table or view with given name [%s] already exists in schema [dfs.tmp]", newTblName));
}
@Test // DRILL-2422
@@ -126,143 +116,97 @@ public class TestCTAS extends BaseTestQuery {
final String newTblName = "createTableWhenAViewWithSameNameAlreadyExists";
try {
- test(String.format("CREATE VIEW %s.%s AS SELECT * from cp.`region.json`", TEMP_SCHEMA, newTblName));
+ test("CREATE VIEW dfs.tmp.%s AS SELECT * from cp.`region.json`", newTblName);
- final String ctasQuery =
- String.format("CREATE TABLE %s.%s AS SELECT * FROM cp.`employee.json`", TEMP_SCHEMA, newTblName);
+ final String ctasQuery = String.format("CREATE TABLE dfs.tmp.%s AS SELECT * FROM cp.`employee.json`", newTblName);
errorMsgTestHelper(ctasQuery,
String.format("A table or view with given name [%s] already exists in schema [%s]",
- newTblName, "dfs_test.tmp"));
+ newTblName, "dfs.tmp"));
} finally {
- test(String.format("DROP VIEW %s.%s", TEMP_SCHEMA, newTblName));
+ test("DROP VIEW dfs.tmp.%s", newTblName);
}
}
@Test
public void ctasPartitionWithEmptyList() throws Exception {
final String newTblName = "ctasPartitionWithEmptyList";
+ final String ctasQuery = String.format("CREATE TABLE dfs.tmp.%s PARTITION BY AS SELECT * from cp.`region.json`", newTblName);
- try {
- final String ctasQuery = String.format("CREATE TABLE %s.%s PARTITION BY AS SELECT * from cp.`region.json`", TEMP_SCHEMA, newTblName);
-
- errorMsgTestHelper(ctasQuery,"PARSE ERROR: Encountered \"AS\"");
- } finally {
- FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), newTblName));
- }
+ errorMsgTestHelper(ctasQuery,"PARSE ERROR: Encountered \"AS\"");
}
@Test // DRILL-3377
public void partitionByCtasColList() throws Exception {
final String newTblName = "partitionByCtasColList";
- try {
- final String ctasQuery = String.format("CREATE TABLE %s.%s (cnt, rkey) PARTITION BY (cnt) " +
- "AS SELECT count(*), n_regionkey from cp.`tpch/nation.parquet` group by n_regionkey",
- TEMP_SCHEMA, newTblName);
-
- test(ctasQuery);
+ test("CREATE TABLE dfs.tmp.%s (cnt, rkey) PARTITION BY (cnt) " +
+ "AS SELECT count(*), n_regionkey from cp.`tpch/nation.parquet` group by n_regionkey", newTblName);
- final String selectFromCreatedTable = String.format(" select cnt, rkey from %s.%s", TEMP_SCHEMA, newTblName);
- final String baselineQuery = "select count(*) as cnt, n_regionkey as rkey from cp.`tpch/nation.parquet` group by n_regionkey";
- testBuilder()
- .sqlQuery(selectFromCreatedTable)
- .unOrdered()
- .sqlBaselineQuery(baselineQuery)
- .build()
- .run();
- } finally {
- FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), newTblName));
- }
+ testBuilder()
+ .sqlQuery("select cnt, rkey from dfs.tmp.%s", newTblName)
+ .unOrdered()
+ .sqlBaselineQuery("select count(*) as cnt, n_regionkey as rkey from cp.`tpch/nation.parquet` group by n_regionkey")
+ .build()
+ .run();
}
@Test // DRILL-3374
public void partitionByCtasFromView() throws Exception {
- final String newTblName = "partitionByCtasColList";
+ final String newTblName = "partitionByCtasFromView";
final String newView = "partitionByCtasColListView";
- try {
- final String viewCreate = String.format("create or replace view %s.%s (col_int, col_varchar) " +
- "AS select cast(n_nationkey as int), cast(n_name as varchar(30)) from cp.`tpch/nation.parquet`",
- TEMP_SCHEMA, newView);
-
- final String ctasQuery = String.format("CREATE TABLE %s.%s PARTITION BY (col_int) AS SELECT * from %s.%s",
- TEMP_SCHEMA, newTblName, TEMP_SCHEMA, newView);
- test(viewCreate);
- test(ctasQuery);
+ test("create or replace view dfs.tmp.%s (col_int, col_varchar) " +
+ "AS select cast(n_nationkey as int), cast(n_name as varchar(30)) from cp.`tpch/nation.parquet`", newView);
+ test("CREATE TABLE dfs.tmp.%s PARTITION BY (col_int) AS SELECT * from dfs.tmp.%s",
+ newTblName, newView);
- final String baselineQuery = "select cast(n_nationkey as int) as col_int, cast(n_name as varchar(30)) as col_varchar " +
- "from cp.`tpch/nation.parquet`";
- final String selectFromCreatedTable = String.format("select col_int, col_varchar from %s.%s", TEMP_SCHEMA, newTblName);
- testBuilder()
- .sqlQuery(selectFromCreatedTable)
- .unOrdered()
- .sqlBaselineQuery(baselineQuery)
- .build()
- .run();
+ testBuilder()
+ .sqlQuery("select col_int, col_varchar from dfs.tmp.%s", newTblName)
+ .unOrdered()
+ .sqlBaselineQuery("select cast(n_nationkey as int) as col_int, cast(n_name as varchar(30)) as col_varchar " +
+ "from cp.`tpch/nation.parquet`")
+ .build()
+ .run();
- final String viewDrop = String.format("DROP VIEW %s.%s", TEMP_SCHEMA, newView);
- test(viewDrop);
- } finally {
- FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), newTblName));
- }
+ test("DROP VIEW dfs.tmp.%s", newView);
}
@Test // DRILL-3382
public void ctasWithQueryOrderby() throws Exception {
final String newTblName = "ctasWithQueryOrderby";
- try {
- final String ctasQuery = String.format("CREATE TABLE %s.%s " +
- "AS SELECT n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` order by n_nationkey",
- TEMP_SCHEMA, newTblName);
-
- test(ctasQuery);
-
- final String selectFromCreatedTable = String.format(" select n_nationkey, n_name, n_comment from %s.%s", TEMP_SCHEMA, newTblName);
- final String baselineQuery = "select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` order by n_nationkey";
+ test("CREATE TABLE dfs.tmp.%s AS SELECT n_nationkey, n_name, n_comment from " +
+ "cp.`tpch/nation.parquet` order by n_nationkey", newTblName);
- testBuilder()
- .sqlQuery(selectFromCreatedTable)
- .ordered()
- .sqlBaselineQuery(baselineQuery)
- .build()
- .run();
- } finally {
- FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), newTblName));
- }
+ testBuilder()
+ .sqlQuery("select n_nationkey, n_name, n_comment from dfs.tmp.%s", newTblName)
+ .ordered()
+ .sqlBaselineQuery("select n_nationkey, n_name, n_comment from cp.`tpch/nation.parquet` order by n_nationkey")
+ .build()
+ .run();
}
@Test // DRILL-4392
public void ctasWithPartition() throws Exception {
final String newTblName = "nation_ctas";
- try {
- final String ctasQuery = String.format("CREATE TABLE %s.%s " +
- "partition by (n_regionkey) AS SELECT n_nationkey, n_regionkey from cp.`tpch/nation.parquet` order by n_nationkey limit 1",
- TEMP_SCHEMA, newTblName);
-
- test(ctasQuery);
-
- final String selectFromCreatedTable = String.format(" select * from %s.%s", TEMP_SCHEMA, newTblName);
- final String baselineQuery = "select n_nationkey, n_regionkey from cp.`tpch/nation.parquet` order by n_nationkey limit 1";
+ test("CREATE TABLE dfs.tmp.%s partition by (n_regionkey) AS " +
+ "SELECT n_nationkey, n_regionkey from cp.`tpch/nation.parquet` order by n_nationkey limit 1", newTblName);
- testBuilder()
- .sqlQuery(selectFromCreatedTable)
- .ordered()
- .sqlBaselineQuery(baselineQuery)
- .build()
- .run();
- } finally {
- FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), newTblName));
- }
+ testBuilder()
+ .sqlQuery("select * from dfs.tmp.%s", newTblName)
+ .ordered()
+ .sqlBaselineQuery("select n_nationkey, n_regionkey from cp.`tpch/nation.parquet` order by n_nationkey limit 1")
+ .build()
+ .run();
}
@Test
public void testPartitionByForAllTypes() throws Exception {
final String location = "partitioned_tables_with_nulls";
final String ctasQuery = "create table %s partition by (%s) as %s";
- final String tablePath = "%s.`%s/%s_%s`";
+ final String tablePath = "dfs.tmp.`%s/%s_%s`";
// key - new table suffix, value - data query
final Map<String, String> variations = Maps.newHashMap();
@@ -270,39 +214,36 @@ public class TestCTAS extends BaseTestQuery {
variations.put("optional", "select * from cp.`parquet/alltypes_optional.parquet`");
variations.put("nulls_only", "select * from cp.`parquet/alltypes_optional.parquet` where %s is null");
- try {
- final QueryDataBatch result = testSqlWithResults("select * from cp.`parquet/alltypes_required.parquet` limit 0").get(0);
- for (UserBitShared.SerializedField field : result.getHeader().getDef().getFieldList()) {
- final String fieldName = field.getNamePart().getName();
-
- for (Map.Entry<String, String> variation : variations.entrySet()) {
- final String table = String.format(tablePath, TEMP_SCHEMA, location, fieldName, variation.getKey());
- final String dataQuery = String.format(variation.getValue(), fieldName);
- test(ctasQuery, table, fieldName, dataQuery, fieldName);
- testBuilder()
- .sqlQuery("select * from %s", table)
- .unOrdered()
- .sqlBaselineQuery(dataQuery)
- .build()
- .run();
- }
+ final QueryDataBatch result = testSqlWithResults("select * from cp.`parquet/alltypes_required.parquet` limit 0").get(0);
+ for (UserBitShared.SerializedField field : result.getHeader().getDef().getFieldList()) {
+ final String fieldName = field.getNamePart().getName();
+
+ for (Map.Entry<String, String> variation : variations.entrySet()) {
+ final String table = String.format(tablePath, location, fieldName, variation.getKey());
+ final String dataQuery = String.format(variation.getValue(), fieldName);
+ test(ctasQuery, table, fieldName, dataQuery, fieldName);
+ testBuilder()
+ .sqlQuery("select * from %s", table)
+ .unOrdered()
+ .sqlBaselineQuery(dataQuery)
+ .build()
+ .run();
}
- result.release();
- } finally {
- FileUtils.deleteQuietly(new File(getDfsTestTmpSchemaLocation(), location));
}
+
+ result.release();
}
@Test
public void createTableWithCustomUmask() throws Exception {
- test("use %s", TEMP_SCHEMA);
+ test("use dfs.tmp");
String tableName = "with_custom_permission";
StorageStrategy storageStrategy = new StorageStrategy("000", false);
FileSystem fs = getLocalFileSystem();
try {
test("alter session set `%s` = '%s'", ExecConstants.PERSISTENT_TABLE_UMASK, storageStrategy.getUmask());
test("create table %s as select 'A' from (values(1))", tableName);
- Path tableLocation = new Path(getDfsTestTmpSchemaLocation(), tableName);
+ Path tableLocation = new Path(dirTestWatcher.getDfsTestTmpDir().getAbsolutePath(), tableName);
assertEquals("Directory permission should match",
storageStrategy.getFolderPermission(), fs.getFileStatus(tableLocation).getPermission());
assertEquals("File permission should match",
@@ -314,7 +255,7 @@ public class TestCTAS extends BaseTestQuery {
}
private static void ctasErrorTestHelper(final String ctasSql, final String expErrorMsg) throws Exception {
- final String createTableSql = String.format(ctasSql, TEMP_SCHEMA, "testTableName");
+ final String createTableSql = String.format(ctasSql, "testTableName");
errorMsgTestHelper(createTableSql, expErrorMsg);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java
index 904c0fe..bc06af3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestCTTAS.java
@@ -20,17 +20,15 @@ package org.apache.drill.exec.sql;
import com.google.common.collect.Lists;
import mockit.Mock;
import mockit.MockUp;
-import mockit.integration.junit4.JMockit;
-import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.dfs.FileSystemConfig;
+import org.apache.drill.exec.store.dfs.WorkspaceConfig;
+import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.categories.SqlTest;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.StorageStrategy;
-import org.apache.drill.exec.store.dfs.FileSystemConfig;
-import org.apache.drill.exec.store.dfs.WorkspaceConfig;
-import org.apache.drill.exec.util.TestUtilities;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
@@ -39,25 +37,27 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
+import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_PLUGIN_NAME;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
-@RunWith(JMockit.class)
@Category(SqlTest.class)
public class TestCTTAS extends BaseTestQuery {
private static final UUID session_id = UUID.nameUUIDFromBytes("sessionId".getBytes());
private static final String temp2_wk = "tmp2";
- private static final String temp2_schema = String.format("%s.%s", TEST_SCHEMA, temp2_wk);
+ private static final String temp2_schema = String.format("%s.%s", DFS_PLUGIN_NAME, temp2_wk);
private static FileSystem fs;
private static FsPermission expectedFolderPermission;
@@ -67,14 +67,16 @@ public class TestCTTAS extends BaseTestQuery {
public static void init() throws Exception {
MockUp<UUID> uuidMockUp = mockRandomUUID(session_id);
Properties testConfigurations = cloneDefaultTestConfigProperties();
- testConfigurations.put(ExecConstants.DEFAULT_TEMPORARY_WORKSPACE, TEMP_SCHEMA);
+ testConfigurations.put(ExecConstants.DEFAULT_TEMPORARY_WORKSPACE, DFS_TMP_SCHEMA);
updateTestCluster(1, DrillConfig.create(testConfigurations));
uuidMockUp.tearDown();
+ File tmp2 = dirTestWatcher.makeSubDir(Paths.get("tmp2"));
+
StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
- FileSystemConfig pluginConfig = (FileSystemConfig) pluginRegistry.getPlugin(TEST_SCHEMA).getConfig();
- pluginConfig.workspaces.put(temp2_wk, new WorkspaceConfig(TestUtilities.createTempDir(), true, null));
- pluginRegistry.createOrUpdate(TEST_SCHEMA, pluginConfig, true);
+ FileSystemConfig pluginConfig = (FileSystemConfig) pluginRegistry.getPlugin(DFS_PLUGIN_NAME).getConfig();
+ pluginConfig.workspaces.put(temp2_wk, new WorkspaceConfig(tmp2.getAbsolutePath(), true, null));
+ pluginRegistry.createOrUpdate(DFS_PLUGIN_NAME, pluginConfig, true);
fs = getLocalFileSystem();
expectedFolderPermission = new FsPermission(StorageStrategy.TEMPORARY.getFolderPermission());
@@ -93,7 +95,7 @@ public class TestCTTAS extends BaseTestQuery {
@Test
public void testSyntax() throws Exception {
test("create TEMPORARY table temporary_keyword as select 1 from (values(1))");
- test("create TEMPORARY table %s.temporary_keyword_with_wk as select 1 from (values(1))", TEMP_SCHEMA);
+ test("create TEMPORARY table %s.temporary_keyword_with_wk as select 1 from (values(1))", DFS_TMP_SCHEMA);
}
@Test
@@ -118,7 +120,7 @@ public class TestCTTAS extends BaseTestQuery {
testBuilder()
.sqlQuery("select * from %s", temporaryTableName)
.unOrdered()
- .sqlBaselineQuery("select * from %s.%s", TEMP_SCHEMA, temporaryTableName)
+ .sqlBaselineQuery("select * from %s.%s", DFS_TMP_SCHEMA, temporaryTableName)
.go();
}
} finally {
@@ -148,7 +150,7 @@ public class TestCTTAS extends BaseTestQuery {
@Test
public void testResolveTemporaryTableWithPartialSchema() throws Exception {
String temporaryTableName = "temporary_table_with_partial_schema";
- test("use %s", TEST_SCHEMA);
+ test("use %s", DFS_PLUGIN_NAME);
test("create temporary table tmp.%s as select 'A' as c1 from (values(1))", temporaryTableName);
testBuilder()
@@ -176,8 +178,7 @@ public class TestCTTAS extends BaseTestQuery {
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: Temporary tables are not allowed to be created / dropped " +
- "outside of default temporary workspace [%s].",
- TEMP_SCHEMA)));
+ "outside of default temporary workspace [%s].", DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -191,7 +192,7 @@ public class TestCTTAS extends BaseTestQuery {
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: A table or view with given name [%s]" +
- " already exists in schema [%s]", temporaryTableName, TEMP_SCHEMA)));
+ " already exists in schema [%s]", temporaryTableName, DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -205,7 +206,7 @@ public class TestCTTAS extends BaseTestQuery {
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: A table or view with given name [%s]" +
- " already exists in schema [%s]", temporaryTableName.toUpperCase(), TEMP_SCHEMA)));
+ " already exists in schema [%s]", temporaryTableName.toUpperCase(), DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -214,12 +215,12 @@ public class TestCTTAS extends BaseTestQuery {
public void testCreateWhenTemporaryTableExistsWithSchema() throws Exception {
String temporaryTableName = "temporary_table_exists_with_schema";
try {
- test("create TEMPORARY table %s.%s as select 'A' as c1 from (values(1))", TEMP_SCHEMA, temporaryTableName);
- test("create TEMPORARY table %s.%s as select 'A' as c1 from (values(1))", TEMP_SCHEMA, temporaryTableName);
+ test("create TEMPORARY table %s.%s as select 'A' as c1 from (values(1))", DFS_TMP_SCHEMA, temporaryTableName);
+ test("create TEMPORARY table %s.%s as select 'A' as c1 from (values(1))", DFS_TMP_SCHEMA, temporaryTableName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: A table or view with given name [%s]" +
- " already exists in schema [%s]", temporaryTableName, TEMP_SCHEMA)));
+ " already exists in schema [%s]", temporaryTableName, DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -228,12 +229,12 @@ public class TestCTTAS extends BaseTestQuery {
public void testCreateWhenPersistentTableExists() throws Exception {
String persistentTableName = "persistent_table_exists";
try {
- test("create table %s.%s as select 'A' as c1 from (values(1))", TEMP_SCHEMA, persistentTableName);
+ test("create table %s.%s as select 'A' as c1 from (values(1))", DFS_TMP_SCHEMA, persistentTableName);
test("create TEMPORARY table %s as select 'A' as c1 from (values(1))", persistentTableName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: A table or view with given name [%s]" +
- " already exists in schema [%s]", persistentTableName, TEMP_SCHEMA)));
+ " already exists in schema [%s]", persistentTableName, DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -242,12 +243,12 @@ public class TestCTTAS extends BaseTestQuery {
public void testCreateWhenViewExists() throws Exception {
String viewName = "view_exists";
try {
- test("create view %s.%s as select 'A' as c1 from (values(1))", TEMP_SCHEMA, viewName);
+ test("create view %s.%s as select 'A' as c1 from (values(1))", DFS_TMP_SCHEMA, viewName);
test("create TEMPORARY table %s as select 'A' as c1 from (values(1))", viewName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: A table or view with given name [%s]" +
- " already exists in schema [%s]", viewName, TEMP_SCHEMA)));
+ " already exists in schema [%s]", viewName, DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -257,11 +258,11 @@ public class TestCTTAS extends BaseTestQuery {
String temporaryTableName = "temporary_table_exists_before_persistent";
try {
test("create TEMPORARY table %s as select 'A' as c1 from (values(1))", temporaryTableName);
- test("create table %s.%s as select 'A' as c1 from (values(1))", TEMP_SCHEMA, temporaryTableName);
+ test("create table %s.%s as select 'A' as c1 from (values(1))", DFS_TMP_SCHEMA, temporaryTableName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: A table or view with given name [%s]" +
- " already exists in schema [%s]", temporaryTableName, TEMP_SCHEMA)));
+ " already exists in schema [%s]", temporaryTableName, DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -271,11 +272,11 @@ public class TestCTTAS extends BaseTestQuery {
String temporaryTableName = "temporary_table_exists_before_view";
try {
test("create TEMPORARY table %s as select 'A' as c1 from (values(1))", temporaryTableName);
- test("create view %s.%s as select 'A' as c1 from (values(1))", TEMP_SCHEMA, temporaryTableName);
+ test("create view %s.%s as select 'A' as c1 from (values(1))", DFS_TMP_SCHEMA, temporaryTableName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: A non-view table with given name [%s] already exists in schema [%s]",
- temporaryTableName, TEMP_SCHEMA)));
+ temporaryTableName, DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -348,7 +349,7 @@ public class TestCTTAS extends BaseTestQuery {
test("create TEMPORARY table %s as select 'A' as c1 from (values(1))", temporaryTableName);
try {
- test("create view %s.view_with_temp_table as select * from %s", TEMP_SCHEMA, temporaryTableName);
+ test("create view %s.view_with_temp_table as select * from %s", DFS_TMP_SCHEMA, temporaryTableName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
"VALIDATION ERROR: Temporary tables usage is disallowed. Used temporary table name: [%s]", temporaryTableName)));
@@ -360,7 +361,7 @@ public class TestCTTAS extends BaseTestQuery {
public void testTemporaryTablesInViewExpansionLogic() throws Exception {
String tableName = "table_for_expansion_logic_test";
String viewName = "view_for_expansion_logic_test";
- test("use %s", TEMP_SCHEMA);
+ test("use %s", DFS_TMP_SCHEMA);
test("create table %s as select 'TABLE' as c1 from (values(1))", tableName);
test("create view %s as select * from %s", viewName, tableName);
@@ -398,10 +399,10 @@ public class TestCTTAS extends BaseTestQuery {
@Test
public void testManualDropWithSchema() throws Exception {
String temporaryTableName = "temporary_table_to_drop_with_schema";
- test("create TEMPORARY table %s.%s as select 'A' as c1 from (values(1))", TEMP_SCHEMA, temporaryTableName);
+ test("create TEMPORARY table %s.%s as select 'A' as c1 from (values(1))", DFS_TMP_SCHEMA, temporaryTableName);
testBuilder()
- .sqlQuery("drop table %s.%s", TEMP_SCHEMA, temporaryTableName)
+ .sqlQuery("drop table %s.%s", DFS_TMP_SCHEMA, temporaryTableName)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(true, String.format("Temporary table [%s] dropped", temporaryTableName))
@@ -414,11 +415,11 @@ public class TestCTTAS extends BaseTestQuery {
test("create TEMPORARY table %s as select 'A' as c1 from (values(1))", temporaryTableName);
testBuilder()
- .sqlQuery("drop view if exists %s.%s", TEMP_SCHEMA, temporaryTableName)
+ .sqlQuery("drop view if exists %s.%s", DFS_TMP_SCHEMA, temporaryTableName)
.unOrdered()
.baselineColumns("ok", "summary")
.baselineValues(false, String.format("View [%s] not found in schema [%s].",
- temporaryTableName, TEMP_SCHEMA))
+ temporaryTableName, DFS_TMP_SCHEMA))
.go();
}
@@ -428,10 +429,10 @@ public class TestCTTAS extends BaseTestQuery {
test("create TEMPORARY table %s as select 'A' as c1 from (values(1))", temporaryTableName);
try {
- test("drop view %s.%s", TEMP_SCHEMA, temporaryTableName);
+ test("drop view %s.%s", DFS_TMP_SCHEMA, temporaryTableName);
} catch (UserRemoteException e) {
assertThat(e.getMessage(), containsString(String.format(
- "VALIDATION ERROR: Unknown view [%s] in schema [%s]", temporaryTableName, TEMP_SCHEMA)));
+ "VALIDATION ERROR: Unknown view [%s] in schema [%s]", temporaryTableName, DFS_TMP_SCHEMA)));
throw e;
}
}
@@ -449,7 +450,7 @@ public class TestCTTAS extends BaseTestQuery {
}
private List<Path> findTemporaryTableLocation(String tableName) throws IOException {
- Path sessionTempLocation = new Path(getDfsTestTmpSchemaLocation(), session_id.toString());
+ Path sessionTempLocation = new Path(dirTestWatcher.getDfsTestTmpDir().getAbsolutePath(), session_id.toString());
assertTrue("Session temporary location must exist", fs.exists(sessionTempLocation));
assertEquals("Session temporary location permission should match",
expectedFolderPermission, fs.getFileStatus(sessionTempLocation).getPermission());
@@ -465,5 +466,4 @@ public class TestCTTAS extends BaseTestQuery {
}
return matchingPath;
}
-
}
[02/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java
index 7fdccca..9b85973 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java
@@ -49,8 +49,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3AscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3 ASC NULLS FIRST" )
@@ -63,8 +62,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3AscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3 ASC NULLS LAST" )
@@ -78,8 +76,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3AscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3 ASC" )
@@ -92,8 +89,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3DescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3 DESC NULLS FIRST" )
@@ -106,8 +102,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3DescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3 DESC NULLS LAST" )
@@ -120,8 +115,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3DescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3 DESC" )
@@ -134,8 +128,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3DefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3 NULLS FIRST" )
@@ -148,7 +141,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3DefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3 NULLS LAST" )
@@ -161,8 +154,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDonutsTopping3DefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, tbl.topping[3].type AS topping3 \n" +
"FROM cp.`donuts.json` AS tbl \n" +
"ORDER BY topping3" )
@@ -179,8 +171,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -192,8 +183,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharAscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -206,8 +196,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -219,8 +208,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharDescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -232,8 +220,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharDescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -245,8 +232,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharDescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -258,8 +244,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharDefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -271,7 +256,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharDefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -283,8 +268,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderVarCharDefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_VarChar AS VARCHAR ) AS as_VARCHAR \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -300,8 +284,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -313,8 +296,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntAscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -327,8 +309,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -340,8 +321,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntDescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -353,8 +333,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntDescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -366,8 +345,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntDescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -379,8 +357,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntDefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -392,7 +369,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntDefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -404,8 +381,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntDefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Int AS INT ) AS as_INT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -421,8 +397,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -434,13 +409,12 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatAscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
- .sql( "SELECT tbl.id, \n" +
+ withNoDefaultSchema()
+ .sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
"ORDER BY as_FLOAT ASC NULLS LAST" )
- .returns( "id=3; as_FLOAT=19.0\n" +
+ .returns( "id=3; as_FLOAT=19.0\n" +
"id=1; as_FLOAT=180.0\n" +
"id=2; as_FLOAT=null"
);
@@ -448,8 +422,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -461,8 +434,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatDescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -474,8 +446,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatDescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -487,8 +458,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatDescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -500,8 +470,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatDefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -513,7 +482,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatDefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -525,8 +494,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderFloatDefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Float AS FLOAT ) AS as_FLOAT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -542,8 +510,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -555,8 +522,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntAscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -569,8 +535,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -582,8 +547,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntDescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -595,8 +559,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntDescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -608,8 +571,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntDescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -621,8 +583,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntDefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -634,7 +595,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntDefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -646,8 +607,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderBigIntDefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_BigInt AS BIGINT ) AS as_BIGINT \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -663,8 +623,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -676,8 +635,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateAscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -690,8 +648,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -703,8 +660,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateDescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -716,8 +672,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateDescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -729,8 +684,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateDescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -742,8 +696,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateDefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -755,7 +708,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateDefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -767,8 +720,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDateDefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Date AS DATE ) AS as_DATE \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -784,8 +736,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -797,8 +748,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalAscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -811,8 +761,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -824,8 +773,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalDescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -837,8 +785,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalDescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -850,8 +797,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalDescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -863,8 +809,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalDefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -876,7 +821,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalDefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -888,8 +833,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderIntervalDefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Interval AS INTERVAL HOUR ) AS as_INTERVAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -905,8 +849,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -918,8 +861,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalAscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -932,8 +874,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -945,8 +886,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalDescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -958,8 +898,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalDescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -971,8 +910,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalDescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -984,8 +922,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalDefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -997,7 +934,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalDefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1009,8 +946,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimalDefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal AS DECIMAL ) AS as_DECIMAL \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1028,8 +964,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5AscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1041,8 +976,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5AscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1055,8 +989,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5AscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1068,8 +1001,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5DescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1081,8 +1013,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5DescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1094,8 +1025,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5DescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1107,8 +1037,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5DefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1120,7 +1049,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5DefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1132,8 +1061,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal5DefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal5 AS DECIMAL(5) ) AS as_DECIMAL5 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1151,8 +1079,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35AscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1164,8 +1091,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35AscNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1178,8 +1104,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35AscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1191,8 +1116,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35DescNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1204,8 +1128,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35DescNullsLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1217,8 +1140,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35DescNullsDefaultFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1230,8 +1152,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35DefaultedAscNullsFirst() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1243,7 +1164,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35DefaultedAscNullsLast() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1255,8 +1176,7 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
@Test
public void testOrderDecimal35DefaultedAscNullsDefaultLast() throws Exception {
- JdbcAssert
- .withNoDefaultSchema()
+ withNoDefaultSchema()
.sql( "SELECT tbl.id, \n" +
" CAST( tbl.for_Decimal35 AS DECIMAL(35) ) AS as_DECIMAL35 \n" +
"FROM cp.`null_ordering_and_grouping_data.json` AS tbl \n" +
@@ -1265,6 +1185,4 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
"id=1; as_DECIMAL35=43210987654321098765432109876543210\n" +
"id=2; as_DECIMAL35=null" );
}
-
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestActionBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestActionBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestActionBase.java
index 5b307d3..baabca4 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestActionBase.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestActionBase.java
@@ -18,16 +18,13 @@
package org.apache.drill.jdbc.test;
import java.io.IOException;
-import java.nio.file.Paths;
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
-import java.sql.Statement;
import java.util.concurrent.TimeUnit;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTestBase;
import org.junit.AfterClass;
@@ -35,34 +32,28 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.rules.TestRule;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
import com.google.common.base.Stopwatch;
public class JdbcTestActionBase extends JdbcTestBase {
// Set a timeout unless we're debugging.
- @Rule
- public TestRule TIMEOUT = TestTools.getTimeoutRule(40000);
+ static Connection connection;
- protected static final String WORKING_PATH;
static {
Driver.load();
- WORKING_PATH = Paths.get("").toAbsolutePath().toString();
-
}
- protected void testQuery(final String sql) throws Exception {
- testAction(new JdbcAction() {
+ @Rule
+ public TestRule TIMEOUT = TestTools.getTimeoutRule(40000);
- @Override
- public ResultSet getResult(Connection c) throws SQLException {
- Statement s = c.createStatement();
- ResultSet r = s.executeQuery(sql);
- return r;
- }
+ @BeforeClass
+ public static void openClient() throws Exception {
+ connection = connect();
+ }
- });
+ @AfterClass
+ public static void closeClient() throws IOException, SQLException {
+ connection.close();
}
protected void testAction(JdbcAction action) throws Exception {
@@ -103,36 +94,7 @@ public class JdbcTestActionBase extends JdbcTestBase {
}
- public static interface JdbcAction {
+ public interface JdbcAction {
ResultSet getResult(Connection c) throws SQLException;
}
-
- static void resetConnection() throws Exception {
- closeClient();
- openClient();
- }
-
- static Connection connection;
-
- @BeforeClass
- public static void openClient() throws Exception {
- connection = DriverManager.getConnection("jdbc:drill:zk=local", JdbcAssert.getDefaultProperties());
- }
-
- @AfterClass
- public static void closeClient() throws IOException, SQLException {
- connection.close();
- }
-
- public final TestRule resetWatcher = new TestWatcher() {
- @Override
- protected void failed(Throwable e, Description description) {
- try {
- resetConnection();
- } catch (Exception e1) {
- throw new RuntimeException("Failure while resetting client.", e1);
- }
- }
- };
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
index bde08b9..6c3289f 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
@@ -17,14 +17,13 @@
*/
package org.apache.drill.jdbc.test;
-import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
import java.util.concurrent.TimeUnit;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.jdbc.DrillResultSet;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTestBase;
@@ -38,16 +37,13 @@ public class JdbcTestQueryBase extends JdbcTestBase {
@Rule
public TestRule TIMEOUT = TestTools.getTimeoutRule(40000);
- protected static final String WORKING_PATH;
static{
Driver.load();
- WORKING_PATH = Paths.get("").toAbsolutePath().toString();
-
}
protected static void testQuery(String sql) throws Exception{
boolean success = false;
- try (Connection conn = connect("jdbc:drill:zk=local")) {
+ try (Connection conn = connect()) {
for (int x = 0; x < 1; x++) {
Stopwatch watch = Stopwatch.createStarted();
Statement s = conn.createStatement();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
index 554d428..1b78bd5 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.jdbc.test;
-import java.nio.file.Paths;
import java.sql.Date;
import org.apache.drill.exec.planner.physical.PlannerSettings;
@@ -43,36 +42,31 @@ public class TestAggregateFunctionsQuery extends JdbcTestQueryBase {
testQuery(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
}
- public static final String WORKING_PATH;
static{
Driver.load();
- WORKING_PATH = Paths.get("").toAbsolutePath().toString();
-
}
+
@Test
public void testDateAggFunction() throws Exception{
- String query = new String("SELECT max(cast(HIRE_DATE as date)) as MAX_DATE, min(cast(HIRE_DATE as date)) as MIN_DATE" +
- " FROM `employee.json`");
-
+ String query = "SELECT max(cast(HIRE_DATE as date)) as MAX_DATE, min(cast(HIRE_DATE as date)) as MIN_DATE" +
+ " FROM `employee.json`";
String t = new Date(ISOChronology.getInstance().getDateTimeMillis(1998, 1, 1, 0)).toString();
String t1 = new Date(ISOChronology.getInstance().getDateTimeMillis(1993, 5, 1, 0)).toString();
- //String t = new Date(1998, 1, 1).toString();
- //String t1 = new Date(1993, 5, 1).toString();
- String result = String.format("MAX_DATE="+ t + "; " + "MIN_DATE=" + t1 + "\n");
+ String result = "MAX_DATE="+ t + "; " + "MIN_DATE=" + t1 + "\n";
- JdbcAssert.withFull("cp")
+ withFull("cp")
.sql(query)
.returns(result);
}
@Test
public void testIntervalAggFunction() throws Exception{
- String query = new String("select max(date_diff(date'2014-5-2', cast(HIRE_DATE as date))) as MAX_DAYS, min(date_diff(date'2014-5-2', cast(HIRE_DATE as date))) MIN_DAYS" +
- " FROM `employee.json`");
+ String query = "select max(date_diff(date'2014-5-2', cast(HIRE_DATE as date))) as MAX_DAYS, min(date_diff(date'2014-5-2', cast(HIRE_DATE as date))) MIN_DAYS" +
+ " FROM `employee.json`";
- JdbcAssert.withFull("cp")
+ withFull("cp")
.sql(query)
.returns(
"MAX_DAYS=P7671D; " +
@@ -82,14 +76,14 @@ public class TestAggregateFunctionsQuery extends JdbcTestQueryBase {
@Test
public void testDecimalAggFunction() throws Exception{
- String query = new String("SELECT " +
+ String query = "SELECT " +
"max(cast(EMPLOYEE_ID as decimal(9, 2))) as MAX_DEC9, min(cast(EMPLOYEE_ID as decimal(9, 2))) as MIN_DEC9," +
"max(cast(EMPLOYEE_ID as decimal(18, 4))) as MAX_DEC18, min(cast(EMPLOYEE_ID as decimal(18, 4))) as MIN_DEC18," +
"max(cast(EMPLOYEE_ID as decimal(28, 9))) as MAX_DEC28, min(cast(EMPLOYEE_ID as decimal(28, 9))) as MIN_DEC28," +
"max(cast(EMPLOYEE_ID as decimal(38, 11))) as MAX_DEC38, min(cast(EMPLOYEE_ID as decimal(38, 11))) as MIN_DEC38" +
- " FROM `employee.json`");
+ " FROM `employee.json`";
- JdbcAssert.withFull("cp")
+ withFull("cp")
.sql(query)
.returns(
"MAX_DEC9=1156.00; " +
@@ -103,13 +97,11 @@ public class TestAggregateFunctionsQuery extends JdbcTestQueryBase {
);
}
-
@Test
public void testVarCharAggFunction() throws Exception{
- String query = new String("select max(full_name) as MAX_NAME, min(full_name) as MIN_NAME" +
- " FROM `employee.json`");
+ String query = "select max(full_name) as MAX_NAME, min(full_name) as MIN_NAME FROM `employee.json`";
- JdbcAssert.withFull("cp")
+ withFull("cp")
.sql(query)
.returns(
"MAX_NAME=Zach Lovell; " +
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestInformationSchemaColumns.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestInformationSchemaColumns.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestInformationSchemaColumns.java
index 16a3c09..7e0934c 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestInformationSchemaColumns.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestInformationSchemaColumns.java
@@ -19,6 +19,7 @@ package org.apache.drill.jdbc.test;
import static java.sql.ResultSetMetaData.columnNoNulls;
import static java.sql.ResultSetMetaData.columnNullable;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
@@ -56,7 +57,6 @@ import org.junit.experimental.categories.Category;
@Category(JdbcTest.class)
public class TestInformationSchemaColumns extends JdbcTestBase {
- private static final String VIEW_SCHEMA = "dfs_test.tmp";
private static final String VIEW_NAME =
TestInformationSchemaColumns.class.getSimpleName() + "_View";
@@ -173,7 +173,7 @@ public class TestInformationSchemaColumns extends JdbcTestBase {
// (Note: Can't use JdbcTest's connect(...) because JdbcTest closes
// Connection--and other JDBC objects--on test method failure, but this test
// class uses some objects across methods.)
- connection = new Driver().connect( "jdbc:drill:zk=local", JdbcAssert.getDefaultProperties() );
+ connection = new Driver().connect( "jdbc:drill:zk=local", getDefaultProperties());
final Statement stmt = connection.createStatement();
ResultSet util;
@@ -206,7 +206,7 @@ public class TestInformationSchemaColumns extends JdbcTestBase {
TODO(DRILL-3253)(end) */
// Create temporary test-columns view:
- util = stmt.executeQuery( "USE dfs_test.tmp" );
+ util = stmt.executeQuery( "USE dfs.tmp" );
assertTrue( util.next() );
assertTrue( "Error setting schema for test: " + util.getString( 2 ), util.getBoolean( 1 ) );
@@ -267,48 +267,48 @@ public class TestInformationSchemaColumns extends JdbcTestBase {
// Set up result rows for temporary test view and Hivetest columns:
- mdrOptBOOLEAN = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptBOOLEAN" );
+ mdrOptBOOLEAN = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptBOOLEAN" );
// TODO(DRILL-2470): Uncomment when TINYINT is implemented:
//mdrReqTINYINT = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqTINYINT" );
// TODO(DRILL-2470): Uncomment when SMALLINT is implemented:
//mdrOptSMALLINT = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptSMALLINT" );
- mdrReqINTEGER = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTEGER" );
- mdrOptBIGINT = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptBIGINT" );
+ mdrReqINTEGER = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTEGER" );
+ mdrOptBIGINT = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptBIGINT" );
// TODO(DRILL-2683): Uncomment when REAL is implemented:
//mdrOptREAL = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptREAL" );
- mdrOptFLOAT = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptFLOAT" );
- mdrReqDOUBLE = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqDOUBLE" );
-
- mdrReqDECIMAL_5_3 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqDECIMAL_5_3" );
-
- mdrReqVARCHAR_10 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqVARCHAR_10" );
- mdrOptVARCHAR = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptVARCHAR" );
- mdrReqCHAR_5 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqCHAR_5" );
- mdrOptVARBINARY_16 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptVARBINARY_16" );
- mdrOptBINARY_1048576 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptBINARY_1048576" );
-
- mdrReqDATE = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqDATE" );
- mdrReqTIME = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqTIME" );
- mdrOptTIME_7 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptTIME_7" );
- mdrOptTIMESTAMP = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrOptTIMESTAMP" );
-
- mdrReqINTERVAL_Y = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Y" );
- mdrReqINTERVAL_3Y_Mo = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3Y_Mo" );
- mdrReqINTERVAL_Mo = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Mo" );
- mdrReqINTERVAL_D = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_D" );
- mdrReqINTERVAL_4D_H = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_4D_H" );
- mdrReqINTERVAL_3D_Mi = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3D_Mi" );
- mdrReqINTERVAL_2D_S5 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_2D_S5" );
- mdrReqINTERVAL_H = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_H" );
- mdrReqINTERVAL_1H_Mi = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_1H_Mi" );
- mdrReqINTERVAL_3H_S1 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3H_S1" );
- mdrReqINTERVAL_Mi = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Mi" );
- mdrReqINTERVAL_5Mi_S = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_5Mi_S" );
- mdrReqINTERVAL_S = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_S" );
- mdrReqINTERVAL_3S = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3S" );
- mdrReqINTERVAL_3S1 = setUpRow( VIEW_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3S1" );
+ mdrOptFLOAT = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptFLOAT" );
+ mdrReqDOUBLE = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqDOUBLE" );
+
+ mdrReqDECIMAL_5_3 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqDECIMAL_5_3" );
+
+ mdrReqVARCHAR_10 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqVARCHAR_10" );
+ mdrOptVARCHAR = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptVARCHAR" );
+ mdrReqCHAR_5 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqCHAR_5" );
+ mdrOptVARBINARY_16 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptVARBINARY_16" );
+ mdrOptBINARY_1048576 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptBINARY_1048576" );
+
+ mdrReqDATE = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqDATE" );
+ mdrReqTIME = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqTIME" );
+ mdrOptTIME_7 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptTIME_7" );
+ mdrOptTIMESTAMP = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrOptTIMESTAMP" );
+
+ mdrReqINTERVAL_Y = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Y" );
+ mdrReqINTERVAL_3Y_Mo = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3Y_Mo" );
+ mdrReqINTERVAL_Mo = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Mo" );
+ mdrReqINTERVAL_D = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_D" );
+ mdrReqINTERVAL_4D_H = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_4D_H" );
+ mdrReqINTERVAL_3D_Mi = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3D_Mi" );
+ mdrReqINTERVAL_2D_S5 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_2D_S5" );
+ mdrReqINTERVAL_H = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_H" );
+ mdrReqINTERVAL_1H_Mi = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_1H_Mi" );
+ mdrReqINTERVAL_3H_S1 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3H_S1" );
+ mdrReqINTERVAL_Mi = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_Mi" );
+ mdrReqINTERVAL_5Mi_S = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_5Mi_S" );
+ mdrReqINTERVAL_S = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_S" );
+ mdrReqINTERVAL_3S = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3S" );
+ mdrReqINTERVAL_3S1 = setUpRow( DFS_TMP_SCHEMA, VIEW_NAME, "mdrReqINTERVAL_3S1" );
/* TODO(DRILL-3253)(start): Update this once we have test plugin supporting all needed types.
mdrReqARRAY = setUpRow( "hive_test.default", "infoschematest", "listtype" );
@@ -472,7 +472,7 @@ public class TestInformationSchemaColumns extends JdbcTestBase {
@Test
public void test_TABLE_SCHEMA_hasRightValue_mdrOptBOOLEAN() throws SQLException {
- assertThat( mdrOptBOOLEAN.getString( "TABLE_SCHEMA" ), equalTo( VIEW_SCHEMA ) );
+ assertThat( mdrOptBOOLEAN.getString( "TABLE_SCHEMA" ), equalTo( DFS_TMP_SCHEMA ) );
}
// Not bothering with other _local_view_ test columns for TABLE_SCHEM.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java
index 08617fc..b13b967 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcDistQuery.java
@@ -19,7 +19,6 @@ package org.apache.drill.jdbc.test;
import java.nio.file.Paths;
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
@@ -27,11 +26,12 @@ import java.sql.Statement;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTestBase;
import org.apache.drill.categories.JdbcTest;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
@@ -43,17 +43,17 @@ import com.google.common.collect.Lists;
@Category(JdbcTest.class)
public class TestJdbcDistQuery extends JdbcTestBase {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcDistQuery.class);
-
-
// Set a timeout unless we're debugging.
- @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(50000);
+ @Rule
+ public TestRule TIMEOUT = TestTools.getTimeoutRule(50000);
- private static final String WORKING_PATH;
static{
Driver.load();
- WORKING_PATH = Paths.get("").toAbsolutePath().toString();
+ }
+ @BeforeClass
+ public static void setup() {
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data"));
}
// TODO: Purge nextUntilEnd(...) and calls when remaining fragment race
@@ -69,134 +69,124 @@ public class TestJdbcDistQuery extends JdbcTestBase {
@Test
public void testSimpleQuerySingleFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY, R_NAME "
- + "from dfs_test.`%s/../../sample-data/regionsSF/`", WORKING_PATH));
+ testQuery("select R_REGIONKEY, R_NAME from dfs.`sample-data/regionsSF/`");
}
@Test
public void testSimpleQueryMultiFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY, R_NAME "
- + "from dfs_test.`%s/../../sample-data/regionsMF/`", WORKING_PATH));
+ testQuery("select R_REGIONKEY, R_NAME from dfs.`sample-data/regionsMF/`");
}
@Test
public void testWhereOverSFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY, R_NAME "
- + "from dfs_test.`%s/../../sample-data/regionsSF/` "
- + "WHERE R_REGIONKEY = 1", WORKING_PATH));
+ testQuery("select R_REGIONKEY, R_NAME from dfs.`sample-data/regionsSF/` WHERE R_REGIONKEY = 1");
}
@Test
public void testWhereOverMFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY, R_NAME "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` "
- + "WHERE R_REGIONKEY = 1", WORKING_PATH));
+ testQuery("select R_REGIONKEY, R_NAME from dfs.`sample-data/regionsMF/` WHERE R_REGIONKEY = 1");
}
@Test
public void testAggSingleFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY "
- + "from dfs_test.`%s/../../sample-data/regionsSF/` "
- + "group by R_REGIONKEY", WORKING_PATH));
+ testQuery("select R_REGIONKEY from dfs.`sample-data/regionsSF/` group by R_REGIONKEY");
}
@Test
public void testAggMultiFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` "
- + "group by R_REGIONKEY", WORKING_PATH));
+ testQuery("select R_REGIONKEY from dfs.`sample-data/regionsMF/` group by R_REGIONKEY");
}
@Test
public void testAggOrderByDiffGKeyMultiFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY, SUM(cast(R_REGIONKEY AS int)) As S "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` "
- + "group by R_REGIONKEY ORDER BY S", WORKING_PATH));
+ testQuery("select R_REGIONKEY, SUM(cast(R_REGIONKEY AS int)) As S "
+ + "from dfs.`sample-data/regionsMF/` "
+ + "group by R_REGIONKEY ORDER BY S");
}
@Test
public void testAggOrderBySameGKeyMultiFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY, SUM(cast(R_REGIONKEY AS int)) As S "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` "
+ testQuery("select R_REGIONKEY, SUM(cast(R_REGIONKEY AS int)) As S "
+ + "from dfs.`sample-data/regionsMF/` "
+ "group by R_REGIONKEY "
- + "ORDER BY R_REGIONKEY", WORKING_PATH));
+ + "ORDER BY R_REGIONKEY");
}
@Ignore
@Test
public void testJoinSingleFile() throws Exception{
- testQuery(String.format("select T1.R_REGIONKEY "
- + "from dfs_test.`%s/../../sample-data/regionsSF/` as T1 "
- + "join dfs_test.`%s/../../sample-data/nationsSF/` as T2 "
- + "on T1.R_REGIONKEY = T2.N_REGIONKEY", WORKING_PATH, WORKING_PATH));
+ testQuery("select T1.R_REGIONKEY "
+ + "from dfs.`sample-data/regionsSF/` as T1 "
+ + "join dfs.`sample-data/nationsSF/` as T2 "
+ + "on T1.R_REGIONKEY = T2.N_REGIONKEY");
}
@Ignore
@Test
public void testJoinMultiFile() throws Exception{
- testQuery(String.format("select T1.R_REGIONKEY "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` as T1 "
- + "join dfs_test.`%s/../../sample-data/nationsMF/` as T2 "
- + "on T1.R_REGIONKEY = T2.N_REGIONKEY", WORKING_PATH, WORKING_PATH));
+ testQuery("select T1.R_REGIONKEY "
+ + "from dfs.`sample-data/regionsMF/` as T1 "
+ + "join dfs.`sample-data/nationsMF/` as T2 "
+ + "on T1.R_REGIONKEY = T2.N_REGIONKEY");
}
@Ignore
@Test
public void testJoinMFileWhere() throws Exception{
- testQuery(String.format("select T1.R_REGIONKEY, T1.R_NAME "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` as T1 "
- + "join dfs_test.`%s/../../sample-data/nationsMF/` as T2 "
+ testQuery("select T1.R_REGIONKEY, T1.R_NAME "
+ + "from dfs.`sample-data/regionsMF/` as T1 "
+ + "join dfs.`sample-data/nationsMF/` as T2 "
+ "on T1.R_REGIONKEY = T2.N_REGIONKEY "
- + "WHERE T1.R_REGIONKEY = 3 ", WORKING_PATH, WORKING_PATH));
+ + "WHERE T1.R_REGIONKEY = 3 ");
}
@Test
//NPE at ExternalSortBatch.java : 151
public void testSortSingleFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY "
- + "from dfs_test.`%s/../../sample-data/regionsSF/` "
- + "order by R_REGIONKEY", WORKING_PATH));
+ testQuery("select R_REGIONKEY "
+ + "from dfs.`sample-data/regionsSF/` "
+ + "order by R_REGIONKEY");
}
@Test
//NPE at ExternalSortBatch.java : 151
public void testSortMultiFile() throws Exception{
- testQuery(String.format("select R_REGIONKEY "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` "
- + "order by R_REGIONKEY", WORKING_PATH));
+ testQuery("select R_REGIONKEY "
+ + "from dfs.`sample-data/regionsMF/` "
+ + "order by R_REGIONKEY");
}
@Test
public void testSortMFileWhere() throws Exception{
- testQuery(String.format("select R_REGIONKEY "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` "
+ testQuery("select R_REGIONKEY "
+ + "from dfs.`sample-data/regionsMF/` "
+ "WHERE R_REGIONKEY = 1 "
- + "order by R_REGIONKEY ", WORKING_PATH ));
+ + "order by R_REGIONKEY");
}
@Ignore
@Test
public void testJoinAggSortWhere() throws Exception{
- testQuery(String.format("select T1.R_REGIONKEY, COUNT(1) as CNT "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` as T1 "
- + "join dfs_test.`%s/../../sample-data/nationsMF/` as T2 "
+ testQuery("select T1.R_REGIONKEY, COUNT(1) as CNT "
+ + "from dfs.`sample-data/regionsMF/` as T1 "
+ + "join dfs.`sample-data/nationsMF/` as T2 "
+ "on T1.R_REGIONKEY = T2.N_REGIONKEY "
+ "WHERE T1.R_REGIONKEY = 3 "
+ "GROUP BY T1.R_REGIONKEY "
- + "ORDER BY T1.R_REGIONKEY",WORKING_PATH, WORKING_PATH ));
+ + "ORDER BY T1.R_REGIONKEY");
}
@Test
public void testSelectLimit() throws Exception{
- testQuery(String.format("select R_REGIONKEY, R_NAME "
- + "from dfs_test.`%s/../../sample-data/regionsMF/` "
- + "limit 2", WORKING_PATH));
+ testQuery("select R_REGIONKEY, R_NAME "
+ + "from dfs.`sample-data/regionsMF/` "
+ + "limit 2");
}
private void testQuery(String sql) throws Exception{
boolean success = false;
- try (Connection c = DriverManager.getConnection("jdbc:drill:zk=local", null);) {
+ try (Connection c = connect()) {
// ???? TODO: What is this currently redundant one-time loop for? (If
// it's kept around to make it easy to switch to looping multiple times
// (e.g., for debugging) then define a constant field or local variable
@@ -240,7 +230,7 @@ public class TestJdbcDistQuery extends JdbcTestBase {
@Test
public void testSchemaForEmptyResultSet() throws Exception {
String query = "select fullname, occupation, postal_code from cp.`customer.json` where 0 = 1";
- try (Connection c = DriverManager.getConnection("jdbc:drill:zk=local", null);) {
+ try (Connection c = connect()) {
Statement s = c.createStatement();
ResultSet r = s.executeQuery(query);
ResultSetMetaData md = r.getMetaData();
@@ -258,5 +248,4 @@ public class TestJdbcDistQuery extends JdbcTestBase {
nextUntilEnd(r);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
index f8c880d..b5f7d7c 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
@@ -21,7 +21,7 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.TestTools;
import org.apache.drill.categories.JdbcTest;
import org.junit.Rule;
import org.junit.Test;
@@ -30,9 +30,6 @@ import org.junit.rules.TestRule;
@Category(JdbcTest.class)
public class TestJdbcMetadata extends JdbcTestActionBase {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcMetadata.class);
-
-
@Rule
public TestRule TIMEOUT = TestTools.getTimeoutRule( 120_000 /* ms */ );
@@ -63,7 +60,7 @@ public class TestJdbcMetadata extends JdbcTestActionBase {
public ResultSet getResult(Connection c) throws SQLException {
return c.getMetaData().getSchemas("DRILL", "%fs%");
}
- }, 6);
+ }, 3);
}
@Test
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index ca72d9f..0066753 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -19,12 +19,15 @@ package org.apache.drill.jdbc.test;
import static org.junit.Assert.assertEquals;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.sql.Types;
import org.apache.drill.categories.JdbcTest;
+import org.apache.drill.jdbc.JdbcTestBase;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -33,7 +36,6 @@ import org.junit.experimental.categories.Category;
@Category(JdbcTest.class)
public class TestJdbcQuery extends JdbcTestQueryBase {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class);
// TODO: Purge nextUntilEnd(...) and calls when remaining fragment race
// conditions are fixed (not just DRILL-2245 fixes).
@@ -46,15 +48,20 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
// }
//}
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data"));
+ }
+
@Test // DRILL-3635
public void testFix3635() throws Exception {
// When we run a CTAS query, executeQuery() should return after the table has been flushed to disk even though we
// didn't yet receive a terminal message. To test this, we run CTAS then immediately run a query on the newly
// created table.
- final String tableName = "dfs_test.tmp.`testDDLs`";
+ final String tableName = "dfs.tmp.`testDDLs`";
- try (Connection conn = connect("jdbc:drill:zk=local")) {
+ try (Connection conn = connect()) {
Statement s = conn.createStatement();
s.executeQuery(String.format("CREATE TABLE %s AS SELECT * FROM cp.`employee.json`", tableName));
}
@@ -70,19 +77,19 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testCast() throws Exception{
- testQuery(String.format("select R_REGIONKEY, cast(R_NAME as varchar(15)) as region, cast(R_COMMENT as varchar(255)) as comment from dfs_test.`%s/../../sample-data/region.parquet`", WORKING_PATH));
+ testQuery("select R_REGIONKEY, cast(R_NAME as varchar(15)) as region, cast(R_COMMENT as varchar(255)) as comment from dfs.`sample-data/region.parquet`");
}
@Test
@Ignore
public void testWorkspace() throws Exception{
- testQuery(String.format("select * from dfs_test.home.`%s/../../sample-data/region.parquet`", WORKING_PATH));
+ testQuery("select * from dfs.root.`sample-data/region.parquet`");
}
@Test
@Ignore
public void testWildcard() throws Exception{
- testQuery(String.format("select * from dfs_test.`%s/../../sample-data/region.parquet`", WORKING_PATH));
+ testQuery("select * from dfs.`sample-data/region.parquet`");
}
@Test
@@ -98,24 +105,24 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
@Ignore
public void testLogicalExplain() throws Exception{
- testQuery(String.format("EXPLAIN PLAN WITHOUT IMPLEMENTATION FOR select * from dfs_test.`%s/../../sample-data/region.parquet`", WORKING_PATH));
+ testQuery("EXPLAIN PLAN WITHOUT IMPLEMENTATION FOR select * from dfs.`sample-data/region.parquet`");
}
@Test
@Ignore
public void testPhysicalExplain() throws Exception{
- testQuery(String.format("EXPLAIN PLAN FOR select * from dfs_test.`%s/../../sample-data/region.parquet`", WORKING_PATH));
+ testQuery("EXPLAIN PLAN FOR select * from dfs.`sample-data/region.parquet`");
}
@Test
@Ignore
public void checkUnknownColumn() throws Exception{
- testQuery(String.format("SELECT unknownColumn FROM dfs_test.`%s/../../sample-data/region.parquet`", WORKING_PATH));
+ testQuery("SELECT unknownColumn FROM dfs.`sample-data/region.parquet`");
}
@Test
public void testLikeNotLike() throws Exception{
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT TABLE_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS " +
"WHERE TABLE_NAME NOT LIKE 'C%' AND COLUMN_NAME LIKE 'TABLE_%E'")
.returns(
@@ -127,7 +134,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testSimilarNotSimilar() throws Exception{
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.`TABLES` "+
"WHERE TABLE_NAME SIMILAR TO '%(H|I)E%' AND TABLE_NAME NOT SIMILAR TO 'C%' ORDER BY TABLE_NAME")
.returns(
@@ -139,14 +146,14 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testIntegerLiteral() throws Exception{
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("select substring('asd' from 1 for 2) from INFORMATION_SCHEMA.`TABLES` limit 1")
.returns("EXPR$0=as\n");
}
@Test
public void testNullOpForNullableType() throws Exception{
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT * FROM cp.`test_null_op.json` WHERE intType IS NULL AND varCharType IS NOT NULL")
.returns("intType=null; varCharType=val2");
}
@@ -154,7 +161,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testNullOpForNonNullableType() throws Exception{
// output of (intType IS NULL) is a non-nullable type
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT * FROM cp.`test_null_op.json` "+
"WHERE (intType IS NULL) IS NULL AND (varCharType IS NOT NULL) IS NOT NULL")
.returns("");
@@ -162,22 +169,22 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testTrueOpForNullableType() throws Exception{
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT data FROM cp.`test_true_false_op.json` WHERE booleanType IS TRUE")
.returns("data=set to true");
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT data FROM cp.`test_true_false_op.json` WHERE booleanType IS FALSE")
.returns("data=set to false");
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT data FROM cp.`test_true_false_op.json` WHERE booleanType IS NOT TRUE")
.returns(
"data=set to false\n" +
"data=not set"
);
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT data FROM cp.`test_true_false_op.json` WHERE booleanType IS NOT FALSE")
.returns(
"data=set to true\n" +
@@ -188,22 +195,22 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testTrueOpForNonNullableType() throws Exception{
// Output of IS TRUE (and others) is a Non-nullable type
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT data FROM cp.`test_true_false_op.json` WHERE (booleanType IS TRUE) IS TRUE")
.returns("data=set to true");
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT data FROM cp.`test_true_false_op.json` WHERE (booleanType IS FALSE) IS FALSE")
.returns(
"data=set to true\n" +
"data=not set"
);
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT data FROM cp.`test_true_false_op.json` WHERE (booleanType IS NOT TRUE) IS NOT TRUE")
.returns("data=set to true");
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT data FROM cp.`test_true_false_op.json` WHERE (booleanType IS NOT FALSE) IS NOT FALSE")
.returns(
"data=set to true\n" +
@@ -213,7 +220,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testDateTimeAccessors() throws Exception{
- JdbcAssert.withNoDefaultSchema().withConnection(new Function<Connection, Void>() {
+ withNoDefaultSchema().withConnection(new Function<Connection, Void>() {
public Void apply(Connection connection) {
try {
final Statement statement = connection.createStatement();
@@ -258,7 +265,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testVerifyMetadata() throws Exception{
- JdbcAssert.withNoDefaultSchema().withConnection(new Function<Connection, Void>() {
+ withNoDefaultSchema().withConnection(new Function<Connection, Void>() {
public Void apply(Connection connection) {
try {
final Statement statement = connection.createStatement();
@@ -270,7 +277,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
assertEquals( Types.TIMESTAMP, resultSet.getMetaData().getColumnType(1) );
assertEquals( Types.DATE, resultSet.getMetaData().getColumnType(2) );
- System.out.println(JdbcAssert.toString(resultSet));
+ System.out.println(JdbcTestBase.toString(resultSet));
resultSet.close();
statement.close();
return null;
@@ -283,7 +290,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testCaseWithNoElse() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT employee_id, CASE WHEN employee_id < 100 THEN first_name END from cp.`employee.json` " +
"WHERE employee_id = 99 OR employee_id = 100")
.returns(
@@ -294,7 +301,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testCaseWithElse() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT employee_id, CASE WHEN employee_id < 100 THEN first_name ELSE 'Test' END from cp.`employee.json` " +
"WHERE employee_id = 99 OR employee_id = 100")
.returns(
@@ -305,7 +312,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testCaseWith2ThensAndNoElse() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT employee_id, CASE WHEN employee_id < 100 THEN first_name WHEN employee_id = 100 THEN last_name END " +
"from cp.`employee.json` " +
"WHERE employee_id = 99 OR employee_id = 100 OR employee_id = 101")
@@ -318,7 +325,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testCaseWith2ThensAndElse() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT employee_id, CASE WHEN employee_id < 100 THEN first_name WHEN employee_id = 100 THEN last_name ELSE 'Test' END " +
"from cp.`employee.json` " +
"WHERE employee_id = 99 OR employee_id = 100 OR employee_id = 101")
@@ -331,7 +338,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testAggWithDrillFunc() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT extract(year from max(to_timestamp(hire_date, 'yyyy-MM-dd HH:mm:SS.SSS' ))) as MAX_YEAR " +
"from cp.`employee.json` ")
.returns(
@@ -341,7 +348,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testLeftRightReplace() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT `left`('abcdef', 2) as LEFT_STR, `right`('abcdef', 2) as RIGHT_STR, `replace`('abcdef', 'ab', 'zz') as REPLACE_STR " +
"from cp.`employee.json` limit 1")
.returns(
@@ -353,7 +360,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testLengthUTF8VarCharInput() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("select length('Sheri', 'UTF8') as L_UTF8 " +
"from cp.`employee.json` where employee_id = 1")
.returns(
@@ -363,7 +370,7 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test
public void testTimeIntervalAddOverflow() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("select extract(hour from (interval '10 20' day to hour + time '10:00:00')) as TIME_INT_ADD " +
"from cp.`employee.json` where employee_id = 1")
.returns(
@@ -373,8 +380,8 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test // DRILL-1051
public void testOldDateTimeJulianCalendar() throws Exception {
- // Should work with any timezone
- JdbcAssert.withNoDefaultSchema()
+ // Should work with any timezone
+ withNoDefaultSchema()
.sql("select cast(to_timestamp('1581-12-01 23:32:01', 'yyyy-MM-dd HH:mm:ss') as date) as `DATE`, " +
"to_timestamp('1581-12-01 23:32:01', 'yyyy-MM-dd HH:mm:ss') as `TIMESTAMP`, " +
"cast(to_timestamp('1581-12-01 23:32:01', 'yyyy-MM-dd HH:mm:ss') as time) as `TIME` " +
@@ -384,8 +391,8 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test // DRILL-1051
public void testOldDateTimeLocalMeanTime() throws Exception {
- // Should work with any timezone
- JdbcAssert.withNoDefaultSchema()
+ // Should work with any timezone
+ withNoDefaultSchema()
.sql("select cast(to_timestamp('1883-11-16 01:32:01', 'yyyy-MM-dd HH:mm:ss') as date) as `DATE`, " +
"to_timestamp('1883-11-16 01:32:01', 'yyyy-MM-dd HH:mm:ss') as `TIMESTAMP`, " +
"cast(to_timestamp('1883-11-16 01:32:01', 'yyyy-MM-dd HH:mm:ss') as time) as `TIME` " +
@@ -395,9 +402,8 @@ public class TestJdbcQuery extends JdbcTestQueryBase {
@Test // DRILL-5792
public void testConvertFromInEmptyInputSql() throws Exception {
- JdbcAssert.withNoDefaultSchema()
+ withNoDefaultSchema()
.sql("SELECT CONVERT_FROM(columns[1], 'JSON') as col1 from cp.`empty.csv`")
.returns("");
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestLegacyJdbcMetadata.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestLegacyJdbcMetadata.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestLegacyJdbcMetadata.java
index 97f7931..caa9da1 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestLegacyJdbcMetadata.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestLegacyJdbcMetadata.java
@@ -28,7 +28,7 @@ import org.junit.BeforeClass;
public class TestLegacyJdbcMetadata extends TestJdbcMetadata {
@BeforeClass
public static void openClient() throws Exception {
- Properties defaultProperties = JdbcAssert.getDefaultProperties();
+ Properties defaultProperties = getDefaultProperties();
defaultProperties.setProperty("server.metadata.disabled", "true");
connection = DriverManager.getConnection("jdbc:drill:zk=local", defaultProperties);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/memory/base/src/main/java/org/apache/drill/exec/memory/ChildAllocator.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/ChildAllocator.java b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/ChildAllocator.java
index 1e4e030..c9aa758 100644
--- a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/ChildAllocator.java
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/ChildAllocator.java
@@ -30,16 +30,13 @@ class ChildAllocator extends BaseAllocator {
/**
* Constructor.
*
- * @param parentAllocator parent allocator -- the one creating this child
- * @param allocatorOwner a handle to the object making the request
- * @param allocationPolicy the allocation policy to use; the policy for all
- * allocators must match for each invocation of a drillbit
- * @param initReservation initial amount of space to reserve (obtained from the parent)
+ * @param parentAllocator parent allocator -- the one creating this child.
+ * @param name The name of the allocator.
+ * @param initReservation initial amount of space to reserve (obtained from the parent).
* @param maxAllocation maximum amount of space that can be obtained from this allocator;
* note this includes direct allocations (via {@see BufferAllocator#buffer(int, int)}
* et al) and requests from descendant allocators. Depending on the allocation policy in
- * force, even less memory may be available
- * @param flags one or more of BaseAllocator.F_* flags
+ * force, even less memory may be available.
*/
ChildAllocator(
BaseAllocator parentAllocator,
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index 2b6fde0..63f4528 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -31,8 +31,6 @@ import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
public abstract class BaseValueVector implements ValueVector {
-// private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
-
/**
* Physical maximum allocation. This is the value prior to Drill 1.11.
* This size causes memory fragmentation. Please use
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 5125626..a090cad 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.vector;
import java.io.Closeable;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import io.netty.buffer.DrillBuf;
import org.apache.drill.exec.exception.OutOfMemoryException;
[05/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
index f36c193..f0653f7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@@ -31,11 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import org.apache.commons.io.FileUtils;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.DrillTestWrapper.TestServices;
-import org.apache.drill.QueryTestUtil;
-import org.apache.drill.TestBuilder;
+import org.apache.drill.test.DrillTestWrapper.TestServices;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.FormatPluginConfig;
@@ -57,22 +54,24 @@ import org.apache.drill.exec.store.dfs.WorkspaceConfig;
import org.apache.drill.exec.store.mock.MockStorageEngine;
import org.apache.drill.exec.store.mock.MockStorageEngineConfig;
import org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider;
-import org.apache.drill.exec.util.TestUtilities;
+import org.apache.drill.exec.util.StoragePluginTestUtils;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.io.Resources;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DEFAULT_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.ROOT_SCHEMA;
+import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA;
+
/**
* Test fixture to start a Drillbit with provide options, create a client, and
* execute queries. Can be used in JUnit tests, or in ad-hoc programs. Provides
* a builder to set the necessary embedded Drillbit and client options, then
* creates the requested Drillbit and client.
*/
-
public class ClusterFixture extends BaseFixture implements AutoCloseable {
- // private static final org.slf4j.Logger logger =
- // org.slf4j.LoggerFactory.getLogger(ClientFixture.class);
public static final String ENABLE_FULL_CACHE = "drill.exec.test.use-full-cache";
public static final int MAX_WIDTH_PER_NODE = 2;
@@ -87,10 +86,10 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
// The CTTAS function requires that the default temporary workspace be
// writable. By default, the default temporary workspace points to
// dfs.tmp. But, the test setup marks dfs.tmp as read-only. To work
- // around this, tests are supposed to use dfs_test. So, we need to
- // set the default temporary workspace to dfs_test.tmp.
+ // around this, tests are supposed to use dfs. So, we need to
+ // set the default temporary workspace to dfs.tmp.
- put(ExecConstants.DEFAULT_TEMPORARY_WORKSPACE, BaseTestQuery.TEMP_SCHEMA);
+ put(ExecConstants.DEFAULT_TEMPORARY_WORKSPACE, DFS_TMP_SCHEMA);
put(ExecConstants.HTTP_ENABLE, false);
put(QueryTestUtil.TEST_QUERY_PRINTING_SILENT, true);
put("drill.catastrophic_to_standard_out", true);
@@ -121,7 +120,6 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
// mode.
put(ZookeeperPersistentStoreProvider.DRILL_EXEC_SYS_STORE_PROVIDER_ZK_BLOBROOT, "/tmp/drill/tests");
- put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, "file:/tmp/drill/tests");
}
};
@@ -132,30 +130,22 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
private boolean ownsZK;
private ZookeeperHelper zkHelper;
private RemoteServiceSet serviceSet;
- private File dfsTestTempDir;
protected List<ClientFixture> clients = new ArrayList<>();
protected RestClientFixture restClientFixture;
private boolean usesZk;
- private boolean preserveLocalFiles;
- private boolean isLocal;
private Properties clientProps;
-
- /**
- * Temporary directories created for this test cluster.
- * Each is removed when closing the cluster.
- */
-
- private List<File> tempDirs = new ArrayList<>();
+ private final ClusterFixtureBuilder builder;
ClusterFixture(ClusterFixtureBuilder builder) {
+ this.builder = Preconditions.checkNotNull(builder);
- setClientProps(builder);
- configureZk(builder);
+ setClientProps();
+ configureZk();
try {
- createConfig(builder);
+ createConfig();
allocator = RootAllocatorFactory.newRoot(config);
- startDrillbits(builder);
- applyOptions(builder);
+ startDrillbits();
+ applyOptions();
} catch (Exception e) {
// Translate exceptions to unchecked to avoid cluttering
// tests. Failures will simply fail the test itself.
@@ -166,9 +156,8 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
/**
* Set the client properties to be used by client fixture.
- * @param builder {@link ClusterFixtureBuilder#clientProps}
*/
- private void setClientProps(ClusterFixtureBuilder builder) {
+ private void setClientProps() {
clientProps = builder.clientProps;
}
@@ -176,8 +165,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
return clientProps;
}
- private void configureZk(ClusterFixtureBuilder builder) {
-
+ private void configureZk() {
// Start ZK if requested.
String zkConnect = null;
@@ -213,8 +201,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
}
}
- private void createConfig(ClusterFixtureBuilder builder) throws Exception {
-
+ private void createConfig() throws Exception {
// Create a config
// Because of the way DrillConfig works, we can set the ZK
// connection string only if a property set is provided.
@@ -226,44 +213,14 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
serviceSet = null;
usesZk = true;
- isLocal = false;
} else {
// Embedded Drillbit.
serviceSet = RemoteServiceSet.getLocalServiceSet();
- isLocal = true;
}
}
- private void startDrillbits(ClusterFixtureBuilder builder) throws Exception {
-// // Ensure that Drill uses the log directory determined here rather than
-// // it's hard-coded defaults. WIP: seems to be needed some times but
-// // not others.
-//
-// String logDir = null;
-// if (builder.tempDir != null) {
-// logDir = builder.tempDir.getAbsolutePath();
-// }
-// if (logDir == null) {
-// logDir = config.getString(ExecConstants.DRILL_TMP_DIR);
-// if (logDir != null) {
-// logDir += "/drill/log";
-// }
-// }
-// if (logDir == null) {
-// logDir = "/tmp/drill";
-// }
-// new File(logDir).mkdirs();
-// System.setProperty("drill.log-dir", logDir);
-
- dfsTestTempDir = makeTempDir("dfs-test");
-
- // Clean up any files that may have been left from the
- // last run.
-
- preserveLocalFiles = builder.preserveLocalFiles;
- removeLocalFiles();
-
+ private void startDrillbits() throws Exception {
// Start the Drillbits.
Preconditions.checkArgument(builder.bitCount > 0);
@@ -302,12 +259,16 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
}
private void configureStoragePlugins(Drillbit bit) throws Exception {
- // Create the dfs_test name space
+ // Create the dfs name space
+ builder.dirTestWatcher.newDfsTestTmpDir();
@SuppressWarnings("resource")
final StoragePluginRegistry pluginRegistry = bit.getContext().getStorage();
- TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTempDir.getAbsolutePath());
- TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry);
+ StoragePluginTestUtils.configureFormatPlugins(pluginRegistry);
+
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry, builder.dirTestWatcher.getDfsTestTmpDir(), TMP_SCHEMA);
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry, builder.dirTestWatcher.getRootDir(), ROOT_SCHEMA);
+ StoragePluginTestUtils.updateSchemaLocation(StoragePluginTestUtils.DFS_PLUGIN_NAME, pluginRegistry, builder.dirTestWatcher.getRootDir(), DEFAULT_SCHEMA);
// Create the mock data plugin
@@ -319,8 +280,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
((StoragePluginRegistryImpl) pluginRegistry).definePlugin(MockStorageEngineConfig.NAME, config, plugin);
}
- private void applyOptions(ClusterFixtureBuilder builder) throws Exception {
-
+ private void applyOptions() throws Exception {
// Apply system options
if (builder.systemOptions != null) {
@@ -342,7 +302,6 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
public Drillbit drillbit(String name) { return bits.get(name); }
public Collection<Drillbit> drillbits() { return bits.values(); }
public RemoteServiceSet serviceSet() { return serviceSet; }
- public File getDfsTestTmpDir() { return dfsTestTempDir; }
public ClientFixture.ClientBuilder clientBuilder() {
return new ClientFixture.ClientBuilder(this);
@@ -442,74 +401,6 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
}
}
zkHelper = null;
-
- // Delete any local files, if we wrote to the local
- // persistent store. But, leave the files if the user wants
- // to review them, for debugging, say. Note that, even if the
- // files are preserved here, they will be removed when the
- // next cluster fixture starts, else the CTTAS initialization
- // will fail.
-
- if (! preserveLocalFiles) {
- try {
- removeLocalFiles();
- } catch (Exception e) {
- ex = ex == null ? e : ex;
- }
- }
-
- // Remove temporary directories created for this cluster session.
-
- try {
- removeTempDirs();
- } catch (Exception e) {
- ex = ex == null ? e : ex;
- }
- if (ex != null) {
- throw ex;
- }
- }
-
- /**
- * Removes files stored locally in the "local store provider."
- * Required because CTTAS setup fails if these files are left from one
- * run to the next.
- *
- * @throws IOException if a directory cannot be deleted
- */
-
- private void removeLocalFiles() throws IOException {
-
- // Don't delete if this is not a local Drillbit.
-
- if (! isLocal) {
- return;
- }
-
- // Remove the local files if they exist.
-
- String localStoreLocation = config.getString(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH);
- removeDir(new File(localStoreLocation));
- }
-
- private void removeTempDirs() throws IOException {
- IOException ex = null;
- for (File dir : tempDirs) {
- try {
- removeDir(dir);
- } catch (IOException e) {
- ex = ex == null ? e : ex;
- }
- }
- if (ex != null) {
- throw ex;
- }
- }
-
- public void removeDir(File dir) throws IOException {
- if (dir.exists()) {
- FileUtils.deleteDirectory(dir);
- }
}
/**
@@ -537,7 +428,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
* Define a workspace within an existing storage plugin. Useful for
* pointing to local file system files outside the Drill source tree.
*
- * @param pluginName name of the plugin like "dfs" or "dfs_test".
+ * @param pluginName name of the plugin like "dfs".
* @param schemaName name of the new schema
* @param path directory location (usually local)
* @param defaultFormat default format for files in the schema
@@ -584,11 +475,14 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
public static final String EXPLAIN_PLAN_TEXT = "text";
public static final String EXPLAIN_PLAN_JSON = "json";
- public static ClusterFixtureBuilder builder() {
- ClusterFixtureBuilder builder = new ClusterFixtureBuilder()
+ public static ClusterFixtureBuilder builder(BaseDirTestWatcher dirTestWatcher) {
+ ClusterFixtureBuilder builder = new ClusterFixtureBuilder(dirTestWatcher)
.sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, MAX_WIDTH_PER_NODE);
Properties props = new Properties();
props.putAll(ClusterFixture.TEST_CONFIGURATIONS);
+ props.setProperty(ExecConstants.DRILL_TMP_DIR, dirTestWatcher.getTmpDir().getAbsolutePath());
+ props.setProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, dirTestWatcher.getStoreDir().getAbsolutePath());
+
builder.configBuilder.configProps(props);
return builder;
}
@@ -603,9 +497,8 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
*
* @return a fixture builder with no default properties set
*/
-
- public static ClusterFixtureBuilder bareBuilder() {
- return new ClusterFixtureBuilder();
+ public static ClusterFixtureBuilder bareBuilder(BaseDirTestWatcher dirTestWatcher) {
+ return new ClusterFixtureBuilder(dirTestWatcher);
}
/**
@@ -645,8 +538,8 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
* @return a cluster fixture with standard options
* @throws Exception if something goes wrong
*/
- public static ClusterFixture standardCluster() {
- return builder().build();
+ public static ClusterFixture standardCluster(BaseDirTestWatcher dirTestWatcher) {
+ return builder(dirTestWatcher).build();
}
/**
@@ -660,7 +553,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
public static String stringify(Object value) {
if (value instanceof String) {
- return "'" + (String) value + "'";
+ return "'" + value + "'";
} else {
return value.toString();
}
@@ -714,34 +607,20 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
}
/**
- * Create a temporary directory which will be removed when the
- * cluster closes.
- *
- * @param dirName the name of the leaf directory
- * @return the path to the temporary directory which is usually
- * under the temporary directory structure for this machine
- */
-
- public File makeTempDir(final String dirName) {
- File dir = getTempDir(dirName);
- tempDirs.add(dir);
- return dir;
- }
-
- /**
* Create a temporary data directory which will be removed when the
* cluster closes, and register it as a "dfs" name space.
*
- * @param key the name to use for the directory and the name space.
+ * @param key The name to use for the directory and the name space.
* Access the directory as "dfs.<key>".
- * @param defaultFormat default storage format for the workspace
+ * @param defaultFormat The default storage format for the workspace.
+ * @param formatPluginConfig The format plugin config.
* @return location of the directory which can be used to create
* temporary input files
*/
- public File makeDataDir(String key, String defaultFormat) {
- File dir = makeTempDir(key);
- defineWorkspace("dfs", key, dir.getAbsolutePath(), defaultFormat);
+ public File makeDataDir(String key, String defaultFormat, FormatPluginConfig formatPluginConfig) {
+ File dir = builder.dirTestWatcher.makeSubDir(Paths.get(key));
+ defineWorkspace("dfs", key, dir.getAbsolutePath(), defaultFormat, formatPluginConfig);
return dir;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
index 8295554..82bcf75 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
@@ -17,18 +17,18 @@
******************************************************************************/
package org.apache.drill.test;
-import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
+import com.google.common.base.Preconditions;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ZookeeperHelper;
import org.apache.drill.exec.server.options.OptionDefinition;
/**
* Build a Drillbit and client with the options provided. The simplest
- * builder starts an embedded Drillbit, with the "dfs_test" name space,
+ * builder starts an embedded Drillbit, with the "dfs" name space,
* a max width (parallelization) of 2.
*/
@@ -48,9 +48,6 @@ public class ClusterFixtureBuilder {
// in the defaults.
public static final int DEFAULT_ZK_REFRESH = 500; // ms
- public static final int DEFAULT_SERVER_RPC_THREADS = 10;
- public static final int DEFAULT_SCAN_THREADS = 8;
- public static final String OPTION_DEFAULTS_ROOT = "drill.exec.options.";
protected ConfigBuilder configBuilder = new ConfigBuilder();
protected List<RuntimeOption> sessionOptions;
@@ -60,16 +57,18 @@ public class ClusterFixtureBuilder {
protected int localZkCount;
protected ZookeeperHelper zkHelper;
protected boolean usingZk;
- protected File tempDir;
- protected boolean preserveLocalFiles;
protected Properties clientProps;
+ protected final BaseDirTestWatcher dirTestWatcher;
+
+ public ClusterFixtureBuilder(BaseDirTestWatcher dirTestWatcher) {
+ this.dirTestWatcher = Preconditions.checkNotNull(dirTestWatcher);
+ }
/**
* The configuration builder which this fixture builder uses.
* @return the configuration builder for use in setting "advanced"
* configuration options.
*/
-
public ConfigBuilder configBuilder() { return configBuilder; }
/**
@@ -87,7 +86,6 @@ public class ClusterFixtureBuilder {
* @return this builder
* @see {@link #configProperty(String, Object)}
*/
-
public ClusterFixtureBuilder configResource(String configResource) {
// TypeSafe gets unhappy about a leading slash, but other functions
@@ -98,21 +96,18 @@ public class ClusterFixtureBuilder {
return this;
}
- /**
- *
- */
public ClusterFixtureBuilder setOptionDefault(String key, Object value) {
- String option_name = OPTION_DEFAULTS_ROOT + key;
+ String option_name = ExecConstants.OPTION_DEFAULTS_ROOT + key;
configBuilder().put(option_name, value.toString());
return this;
}
+
/**
* Add an additional boot-time property for the embedded Drillbit.
* @param key config property name
* @param value property value
* @return this builder
*/
-
public ClusterFixtureBuilder configProperty(String key, Object value) {
configBuilder.put(key, value.toString());
return this;
@@ -145,9 +140,7 @@ public class ClusterFixtureBuilder {
* @param key the name of the session option
* @param value the value of the session option
* @return this builder
- * @see {@link ClusterFixture#alterSession(String, Object)}
*/
-
public ClusterFixtureBuilder sessionOption(String key, Object value) {
if (sessionOptions == null) {
sessionOptions = new ArrayList<>();
@@ -163,9 +156,7 @@ public class ClusterFixtureBuilder {
* @param key the name of the system option
* @param value the value of the system option
* @return this builder
- * @see {@link ClusterFixture#alterSystem(String, Object)}
*/
-
public ClusterFixtureBuilder systemOption(String key, Object value) {
if (systemOptions == null) {
systemOptions = new ArrayList<>();
@@ -252,44 +243,13 @@ public class ClusterFixtureBuilder {
return this;
}
- public ClusterFixtureBuilder tempDir(File path) {
- this.tempDir = path;
- return this;
- }
-
- /**
- * Starting with the addition of the CTTAS feature, a Drillbit will
- * not restart unless we delete all local storage files before
- * starting the Drillbit again. In particular, the stored copies
- * of the storage plugin configs cause the temporary workspace
- * check to fail. Normally the cluster fixture cleans up files
- * both before starting and after shutting down the cluster. Set this
- * option to preserve files after shutdown, perhaps to debug the
- * contents.
- * <p>
- * This clean-up is needed only if we enable local storage writes
- * (which we must do, unfortunately, to capture and analyze
- * storage profiles.)
- *
- * @return this builder
- */
-
- public ClusterFixtureBuilder keepLocalFiles() {
- preserveLocalFiles = true;
- return this;
- }
-
/**
* Enable saving of query profiles. The only way to save them is
* to enable local store provider writes, which also saves the
- * storage plugin configs. Doing so causes the CTTAS feature to
- * fail on the next run, so the test fixture deletes all local
- * files on start and close, unless
- * {@link #keepLocalFiles()} is set.
+ * storage plugin configs.
*
* @return this builder
*/
-
public ClusterFixtureBuilder saveProfiles() {
configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true);
systemOption(ExecConstants.ENABLE_QUERY_PROFILE_OPTION, true);
@@ -319,7 +279,6 @@ public class ClusterFixtureBuilder {
*
* @return
*/
-
public ClusterFixture build() {
return new ClusterFixture(this);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java
index 4a3823c..c85c591 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java
@@ -19,9 +19,7 @@ package org.apache.drill.test;
import java.io.IOException;
-import org.apache.drill.TestBuilder;
import org.apache.drill.common.AutoCloseables;
-import org.apache.drill.test.DrillTest;
import org.junit.AfterClass;
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java
new file mode 100644
index 0000000..99bbacc
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java
@@ -0,0 +1,832 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.HyperVectorValueIterator;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryType;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.HyperVectorWrapper;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.selection.SelectionVector2;
+import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.apache.drill.exec.util.Text;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.test.rowSet.RowSetComparison;
+import org.junit.Assert;
+
+/**
+ * An object to encapsulate the options for a Drill unit test, as well as the execution methods to perform the tests and
+ * validation of results.
+ *
+ * To construct an instance easily, look at the TestBuilder class. From an implementation of
+ * the BaseTestQuery class, and instance of the builder is accessible through the testBuilder() method.
+ */
+public class DrillTestWrapper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class);
+
+ public interface TestServices {
+ BufferAllocator allocator();
+
+ void test(String query) throws Exception;
+
+ List<QueryDataBatch> testRunAndReturn(QueryType type, Object query) throws Exception;
+ }
+
+ // TODO - when in JSON, read baseline in all text mode to avoid precision loss for decimal values
+
+ // This flag will enable all of the values that are validated to be logged. For large validations this is time consuming
+ // so this is not exposed in a way that it can be enabled for an individual test. It can be changed here while debugging
+ // a test to see all of the output, but as this framework is doing full validation, there is no reason to keep it on as
+ // it will only make the test slower.
+ private static boolean VERBOSE_DEBUG = false;
+
+ // Unit test doesn't expect any specific batch count
+ public static final int EXPECTED_BATCH_COUNT_NOT_SET = -1;
+
+ // The motivation behind the TestBuilder was to provide a clean API for test writers. The model is mostly designed to
+ // prepare all of the components necessary for running the tests, before the TestWrapper is initialized. There is however
+ // one case where the setup for the baseline is driven by the test query results, and this is implicit type enforcement
+ // for the baseline data. In this case there needs to be a call back into the TestBuilder once we know the type information
+ // from the test query.
+ private TestBuilder testBuilder;
+ /**
+ * Test query to run. Type of object depends on the {@link #queryType}
+ */
+ private Object query;
+ // The type of query provided
+ private UserBitShared.QueryType queryType;
+ // The type of query provided for the baseline
+ private UserBitShared.QueryType baselineQueryType;
+ // should ordering be enforced in the baseline check
+ private boolean ordered;
+ private TestServices services;
+ // queries to run before the baseline or test queries, can be used to set options
+ private String baselineOptionSettingQueries;
+ private String testOptionSettingQueries;
+ // two different methods are available for comparing ordered results, the default reads all of the records
+ // into giant lists of objects, like one giant on-heap batch of 'vectors'
+ // this flag enables the other approach which iterates through a hyper batch for the test query results and baseline
+ // while this does work faster and use less memory, it can be harder to debug as all of the elements are not in a
+ // single list
+ private boolean highPerformanceComparison;
+ // if the baseline is a single option test writers can provide the baseline values and columns
+ // without creating a file, these are provided to the builder in the baselineValues() and baselineColumns() methods
+ // and translated into a map in the builder
+ private String[] baselineColumns;
+ private List<Map<String, Object>> baselineRecords;
+
+ private int expectedNumBatches;
+
+ public DrillTestWrapper(TestBuilder testBuilder, TestServices services, Object query, QueryType queryType,
+ String baselineOptionSettingQueries, String testOptionSettingQueries,
+ QueryType baselineQueryType, boolean ordered, boolean highPerformanceComparison,
+ String[] baselineColumns, List<Map<String, Object>> baselineRecords, int expectedNumBatches) {
+ this.testBuilder = testBuilder;
+ this.services = services;
+ this.query = query;
+ this.queryType = queryType;
+ this.baselineQueryType = baselineQueryType;
+ this.ordered = ordered;
+ this.baselineOptionSettingQueries = baselineOptionSettingQueries;
+ this.testOptionSettingQueries = testOptionSettingQueries;
+ this.highPerformanceComparison = highPerformanceComparison;
+ this.baselineColumns = baselineColumns;
+ this.baselineRecords = baselineRecords;
+ this.expectedNumBatches = expectedNumBatches;
+ }
+
+ public void run() throws Exception {
+ if (testBuilder.getExpectedSchema() != null) {
+ compareSchemaOnly();
+ } else {
+ if (ordered) {
+ compareOrderedResults();
+ } else {
+ compareUnorderedResults();
+ }
+ }
+ }
+
+ private BufferAllocator getAllocator() {
+ return services.allocator();
+ }
+
+ private void compareHyperVectors(Map<String, HyperVectorValueIterator> expectedRecords,
+ Map<String, HyperVectorValueIterator> actualRecords) throws Exception {
+ for (String s : expectedRecords.keySet()) {
+ assertNotNull("Expected column '" + s + "' not found.", actualRecords.get(s));
+ assertEquals(expectedRecords.get(s).getTotalRecords(), actualRecords.get(s).getTotalRecords());
+ HyperVectorValueIterator expectedValues = expectedRecords.get(s);
+ HyperVectorValueIterator actualValues = actualRecords.get(s);
+ int i = 0;
+ while (expectedValues.hasNext()) {
+ compareValuesErrorOnMismatch(expectedValues.next(), actualValues.next(), i, s);
+ i++;
+ }
+ }
+ cleanupHyperValueIterators(expectedRecords.values());
+ cleanupHyperValueIterators(actualRecords.values());
+ }
+
+ private void cleanupHyperValueIterators(Collection<HyperVectorValueIterator> hyperBatches) {
+ for (HyperVectorValueIterator hvi : hyperBatches) {
+ for (ValueVector vv : hvi.getHyperVector().getValueVectors()) {
+ vv.clear();
+ }
+ }
+ }
+
+ public static void compareMergedVectors(Map<String, List<Object>> expectedRecords, Map<String, List<Object>> actualRecords) throws Exception {
+ for (String s : actualRecords.keySet()) {
+ assertNotNull("Unexpected extra column " + s + " returned by query.", expectedRecords.get(s));
+ assertEquals("Incorrect number of rows returned by query.", expectedRecords.get(s).size(), actualRecords.get(s).size());
+ List<?> expectedValues = expectedRecords.get(s);
+ List<?> actualValues = actualRecords.get(s);
+ assertEquals("Different number of records returned", expectedValues.size(), actualValues.size());
+
+ for (int i = 0; i < expectedValues.size(); i++) {
+ try {
+ compareValuesErrorOnMismatch(expectedValues.get(i), actualValues.get(i), i, s);
+ } catch (Exception ex) {
+ throw new Exception(ex.getMessage() + "\n\n" + printNearbyRecords(expectedRecords, actualRecords, i), ex);
+ }
+ }
+ }
+ if (actualRecords.size() < expectedRecords.size()) {
+ throw new Exception(findMissingColumns(expectedRecords.keySet(), actualRecords.keySet()));
+ }
+ }
+
+ private static String printNearbyRecords(Map<String, List<Object>> expectedRecords, Map<String, List<Object>> actualRecords, int offset) {
+ StringBuilder expected = new StringBuilder();
+ StringBuilder actual = new StringBuilder();
+ expected.append("Expected Records near verification failure:\n");
+ actual.append("Actual Records near verification failure:\n");
+ int firstRecordToPrint = Math.max(0, offset - 5);
+ List<?> expectedValuesInFirstColumn = expectedRecords.get(expectedRecords.keySet().iterator().next());
+ List<?> actualValuesInFirstColumn = expectedRecords.get(expectedRecords.keySet().iterator().next());
+ int numberOfRecordsToPrint = Math.min(Math.min(10, expectedValuesInFirstColumn.size()), actualValuesInFirstColumn.size());
+ for (int i = firstRecordToPrint; i < numberOfRecordsToPrint; i++) {
+ expected.append("Record Number: ").append(i).append(" { ");
+ actual.append("Record Number: ").append(i).append(" { ");
+ for (String s : actualRecords.keySet()) {
+ List<?> actualValues = actualRecords.get(s);
+ actual.append(s).append(" : ").append(actualValues.get(i)).append(",");
+ }
+ for (String s : expectedRecords.keySet()) {
+ List<?> expectedValues = expectedRecords.get(s);
+ expected.append(s).append(" : ").append(expectedValues.get(i)).append(",");
+ }
+ expected.append(" }\n");
+ actual.append(" }\n");
+ }
+
+ return expected.append("\n\n").append(actual).toString();
+
+ }
+
+ private Map<String, HyperVectorValueIterator> addToHyperVectorMap(final List<QueryDataBatch> records,
+ final RecordBatchLoader loader)
+ throws SchemaChangeException, UnsupportedEncodingException {
+ // TODO - this does not handle schema changes
+ Map<String, HyperVectorValueIterator> combinedVectors = new TreeMap<>();
+
+ long totalRecords = 0;
+ QueryDataBatch batch;
+ int size = records.size();
+ for (int i = 0; i < size; i++) {
+ batch = records.get(i);
+ loader.load(batch.getHeader().getDef(), batch.getData());
+ logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
+ totalRecords += loader.getRecordCount();
+ for (VectorWrapper<?> w : loader) {
+ String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr();
+ if (!combinedVectors.containsKey(field)) {
+ MaterializedField mf = w.getField();
+ ValueVector[] vvList = (ValueVector[]) Array.newInstance(mf.getValueClass(), 1);
+ vvList[0] = w.getValueVector();
+ combinedVectors.put(field, new HyperVectorValueIterator(mf, new HyperVectorWrapper<>(mf, vvList)));
+ } else {
+ combinedVectors.get(field).getHyperVector().addVector(w.getValueVector());
+ }
+
+ }
+ }
+ for (HyperVectorValueIterator hvi : combinedVectors.values()) {
+ hvi.determineTotalSize();
+ }
+ return combinedVectors;
+ }
+
+ private static class BatchIterator implements Iterable<VectorAccessible>, AutoCloseable {
+ private final List<QueryDataBatch> dataBatches;
+ private final RecordBatchLoader batchLoader;
+
+ public BatchIterator(List<QueryDataBatch> dataBatches, RecordBatchLoader batchLoader) {
+ this.dataBatches = dataBatches;
+ this.batchLoader = batchLoader;
+ }
+
+ @Override
+ public Iterator<VectorAccessible> iterator() {
+ return new Iterator<VectorAccessible>() {
+
+ int index = -1;
+
+ @Override
+ public boolean hasNext() {
+ return index < dataBatches.size() - 1;
+ }
+
+ @Override
+ public VectorAccessible next() {
+ index++;
+ if (index == dataBatches.size()) {
+ throw new RuntimeException("Tried to call next when iterator had no more items.");
+ }
+ batchLoader.clear();
+ QueryDataBatch batch = dataBatches.get(index);
+ try {
+ batchLoader.load(batch.getHeader().getDef(), batch.getData());
+ } catch (SchemaChangeException e) {
+ throw new RuntimeException(e);
+ }
+ return batchLoader;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Removing is not supported");
+ }
+ };
+ }
+
+ @Override
+ public void close() throws Exception {
+ batchLoader.clear();
+ }
+
+ }
+
+ /**
+ * Iterate over batches, and combine the batches into a map, where key is schema path, and value is
+ * the list of column values across all the batches.
+ * @param batches
+ * @return
+ * @throws SchemaChangeException
+ * @throws UnsupportedEncodingException
+ */
+ public static Map<String, List<Object>> addToCombinedVectorResults(Iterable<VectorAccessible> batches)
+ throws SchemaChangeException, UnsupportedEncodingException {
+ Map<String, List<Object>> combinedVectors = new TreeMap<>();
+ addToCombinedVectorResults(batches, null, combinedVectors);
+ return combinedVectors;
+ }
+
+ /**
+ * Add to result vectors and compare batch schema against expected schema while iterating batches.
+ * @param batches
+ * @param expectedSchema: the expected schema the batches should contain. Through SchemaChangeException
+ * if encounter different batch schema.
+ * @param combinedVectors: the vectors to hold the values when iterate the batches.
+ *
+ * @return number of batches
+ * @throws SchemaChangeException
+ * @throws UnsupportedEncodingException
+ */
+ public static int addToCombinedVectorResults(Iterable<VectorAccessible> batches, BatchSchema expectedSchema, Map<String, List<Object>> combinedVectors)
+ throws SchemaChangeException, UnsupportedEncodingException {
+ // TODO - this does not handle schema changes
+ int numBatch = 0;
+ long totalRecords = 0;
+ BatchSchema schema = null;
+ for (VectorAccessible loader : batches) {
+ numBatch++;
+ if (expectedSchema != null) {
+ if (! expectedSchema.equals(loader.getSchema())) {
+ throw new SchemaChangeException(String.format("Batch schema does not match expected schema\n" +
+ "Actual schema: %s. Expected schema : %s",
+ loader.getSchema(), expectedSchema));
+ }
+ }
+
+ // TODO: Clean: DRILL-2933: That load(...) no longer throws
+ // SchemaChangeException, so check/clean throws clause above.
+ if (schema == null) {
+ schema = loader.getSchema();
+ for (MaterializedField mf : schema) {
+ combinedVectors.put(SchemaPath.getSimplePath(mf.getName()).toExpr(), new ArrayList<>());
+ }
+ } else {
+ // TODO - actually handle schema changes, this is just to get access to the SelectionVectorMode
+ // of the current batch, the check for a null schema is used to only mutate the schema once
+ // need to add new vectors and null fill for previous batches? distinction between null and non-existence important?
+ schema = loader.getSchema();
+ }
+ logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
+ totalRecords += loader.getRecordCount();
+ for (VectorWrapper<?> w : loader) {
+ String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr();
+ ValueVector[] vectors;
+ if (w.isHyper()) {
+ vectors = w.getValueVectors();
+ } else {
+ vectors = new ValueVector[] {w.getValueVector()};
+ }
+ SelectionVector2 sv2 = null;
+ SelectionVector4 sv4 = null;
+ switch(schema.getSelectionVectorMode()) {
+ case TWO_BYTE:
+ sv2 = loader.getSelectionVector2();
+ break;
+ case FOUR_BYTE:
+ sv4 = loader.getSelectionVector4();
+ break;
+ }
+ if (sv4 != null) {
+ for (int j = 0; j < sv4.getCount(); j++) {
+ int complexIndex = sv4.get(j);
+ int batchIndex = complexIndex >> 16;
+ int recordIndexInBatch = complexIndex & 65535;
+ Object obj = vectors[batchIndex].getAccessor().getObject(recordIndexInBatch);
+ if (obj != null) {
+ if (obj instanceof Text) {
+ obj = obj.toString();
+ }
+ }
+ combinedVectors.get(field).add(obj);
+ }
+ }
+ else {
+ for (ValueVector vv : vectors) {
+ for (int j = 0; j < loader.getRecordCount(); j++) {
+ int index;
+ if (sv2 != null) {
+ index = sv2.getIndex(j);
+ } else {
+ index = j;
+ }
+ Object obj = vv.getAccessor().getObject(index);
+ if (obj != null) {
+ if (obj instanceof Text) {
+ obj = obj.toString();
+ }
+ }
+ combinedVectors.get(field).add(obj);
+ }
+ }
+ }
+ }
+ }
+ return numBatch;
+ }
+
+ protected void compareSchemaOnly() throws Exception {
+ RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+ List<QueryDataBatch> actual = null;
+ QueryDataBatch batch = null;
+ try {
+ test(testOptionSettingQueries);
+ actual = testRunAndReturn(queryType, query);
+ batch = actual.get(0);
+ loader.load(batch.getHeader().getDef(), batch.getData());
+
+ final BatchSchema schema = loader.getSchema();
+ final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = testBuilder.getExpectedSchema();
+ if (schema.getFieldCount() != expectedSchema.size()) {
+ throw new Exception("Expected and actual numbers of columns do not match.");
+ }
+
+ for (int i = 0; i < schema.getFieldCount(); ++i) {
+ final String actualSchemaPath = schema.getColumn(i).getName();
+ final TypeProtos.MajorType actualMajorType = schema.getColumn(i).getType();
+
+ final String expectedSchemaPath = expectedSchema.get(i).getLeft().getRootSegmentPath();
+ final TypeProtos.MajorType expectedMajorType = expectedSchema.get(i).getValue();
+
+ if (!actualSchemaPath.equals(expectedSchemaPath)
+ || !actualMajorType.equals(expectedMajorType)) {
+ throw new Exception(String.format("Schema path or type mismatch for column #%d:\n" +
+ "Expected schema path: %s\nActual schema path: %s\nExpected type: %s\nActual type: %s",
+ i, expectedSchemaPath, actualSchemaPath, Types.toString(expectedMajorType),
+ Types.toString(actualMajorType)));
+ }
+ }
+
+ } finally {
+ if (actual != null) {
+ for (QueryDataBatch b : actual) {
+ b.release();
+ }
+ }
+ loader.clear();
+ }
+ }
+
+ /**
+ * Use this method only if necessary to validate one query against another. If you are just validating against a
+ * baseline file use one of the simpler interfaces that will write the validation query for you.
+ *
+ * @throws Exception
+ */
+ protected void compareUnorderedResults() throws Exception {
+ RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+
+ List<QueryDataBatch> actual = Collections.emptyList();
+ List<QueryDataBatch> expected = Collections.emptyList();
+ List<Map<String, Object>> expectedRecords = new ArrayList<>();
+ List<Map<String, Object>> actualRecords = new ArrayList<>();
+
+ try {
+ test(testOptionSettingQueries);
+ actual = testRunAndReturn(queryType, query);
+
+ checkNumBatches(actual);
+
+ addTypeInfoIfMissing(actual.get(0), testBuilder);
+ addToMaterializedResults(actualRecords, actual, loader);
+
+ // If baseline data was not provided to the test builder directly, we must run a query for the baseline, this includes
+ // the cases where the baseline is stored in a file.
+ if (baselineRecords == null) {
+ test(baselineOptionSettingQueries);
+ expected = testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery());
+ addToMaterializedResults(expectedRecords, expected, loader);
+ } else {
+ expectedRecords = baselineRecords;
+ }
+
+ compareResults(expectedRecords, actualRecords);
+ } finally {
+ cleanupBatches(actual, expected);
+ }
+ }
+
+ /**
+ * Use this method only if necessary to validate one query against another. If you are just validating against a
+ * baseline file use one of the simpler interfaces that will write the validation query for you.
+ *
+ * @throws Exception
+ */
+ protected void compareOrderedResults() throws Exception {
+ if (highPerformanceComparison) {
+ if (baselineQueryType == null) {
+ throw new Exception("Cannot do a high performance comparison without using a baseline file");
+ }
+ compareResultsHyperVector();
+ } else {
+ compareMergedOnHeapVectors();
+ }
+ }
+
+ public void compareMergedOnHeapVectors() throws Exception {
+ RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+
+ List<QueryDataBatch> actual = Collections.emptyList();
+ List<QueryDataBatch> expected = Collections.emptyList();
+ Map<String, List<Object>> actualSuperVectors;
+ Map<String, List<Object>> expectedSuperVectors = null;
+
+ try {
+ test(testOptionSettingQueries);
+ actual = testRunAndReturn(queryType, query);
+
+ checkNumBatches(actual);
+
+ // To avoid extra work for test writers, types can optionally be inferred from the test query
+ addTypeInfoIfMissing(actual.get(0), testBuilder);
+
+ BatchIterator batchIter = new BatchIterator(actual, loader);
+ actualSuperVectors = addToCombinedVectorResults(batchIter);
+ batchIter.close();
+
+ // If baseline data was not provided to the test builder directly, we must run a query for the baseline, this includes
+ // the cases where the baseline is stored in a file.
+ if (baselineRecords == null) {
+ if (baselineQueryType == null && baselineColumns != null) {
+ checkAscendingOrdering(actualSuperVectors);
+ return;
+ } else {
+ test(baselineOptionSettingQueries);
+ expected = testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery());
+ BatchIterator exBatchIter = new BatchIterator(expected, loader);
+ expectedSuperVectors = addToCombinedVectorResults(exBatchIter);
+ exBatchIter.close();
+ }
+ } else {
+ // data is built in the TestBuilder in a row major format as it is provided by the user
+ // translate it here to vectorized, the representation expected by the ordered comparison
+ expectedSuperVectors = translateRecordListToHeapVectors(baselineRecords);
+ }
+
+ compareMergedVectors(expectedSuperVectors, actualSuperVectors);
+ } catch (Exception e) {
+ throw new Exception(e.getMessage() + "\nFor query: " + query , e);
+ } finally {
+ cleanupBatches(expected, actual);
+ }
+ }
+
+ private void checkAscendingOrdering(Map<String, List<Object>> results) {
+ int numRecords = results.get(baselineColumns[0]).size();
+
+ for (int index = 1; index < numRecords; index++) {
+ int prevIndex = index - 1;
+
+ for (String column: baselineColumns) {
+ List<Object> objects = results.get(column);
+ Object prevObject = objects.get(prevIndex);
+ Object currentObject = objects.get(index);
+
+ Assert.assertTrue(RowSetComparison.ObjectComparator.INSTANCE.compare(prevObject, currentObject) <= 0);
+ }
+ }
+ }
+
+ public static Map<String, List<Object>> translateRecordListToHeapVectors(List<Map<String, Object>> records) {
+ Map<String, List<Object>> ret = new TreeMap<>();
+ for (String s : records.get(0).keySet()) {
+ ret.put(s, new ArrayList<>());
+ }
+ for (Map<String, Object> m : records) {
+ for (String s : m.keySet()) {
+ ret.get(s).add(m.get(s));
+ }
+ }
+ return ret;
+ }
+
+ public void compareResultsHyperVector() throws Exception {
+ RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+
+ test(testOptionSettingQueries);
+ List<QueryDataBatch> results = testRunAndReturn(queryType, query);
+
+ checkNumBatches(results);
+
+ // To avoid extra work for test writers, types can optionally be inferred from the test query
+ addTypeInfoIfMissing(results.get(0), testBuilder);
+
+ Map<String, HyperVectorValueIterator> actualSuperVectors = addToHyperVectorMap(results, loader);
+
+ test(baselineOptionSettingQueries);
+ List<QueryDataBatch> expected = testRunAndReturn(baselineQueryType, testBuilder.getValidationQuery());
+
+ Map<String, HyperVectorValueIterator> expectedSuperVectors = addToHyperVectorMap(expected, loader);
+
+ compareHyperVectors(expectedSuperVectors, actualSuperVectors);
+ cleanupBatches(results, expected);
+ }
+
+ private void checkNumBatches(final List<QueryDataBatch> results) {
+ if (expectedNumBatches != EXPECTED_BATCH_COUNT_NOT_SET) {
+ final int actualNumBatches = results.size();
+ assertEquals(String.format("Expected %d batches but query returned %d non empty batch(es)%n", expectedNumBatches,
+ actualNumBatches), expectedNumBatches, actualNumBatches);
+ }
+ }
+
+ private void addTypeInfoIfMissing(QueryDataBatch batch, TestBuilder testBuilder) {
+ if (! testBuilder.typeInfoSet()) {
+ Map<SchemaPath, TypeProtos.MajorType> typeMap = getTypeMapFromBatch(batch);
+ testBuilder.baselineTypes(typeMap);
+ }
+
+ }
+
+ private Map<SchemaPath, TypeProtos.MajorType> getTypeMapFromBatch(QueryDataBatch batch) {
+ Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>();
+ for (int i = 0; i < batch.getHeader().getDef().getFieldCount(); i++) {
+ typeMap.put(SchemaPath.getSimplePath(MaterializedField.create(batch.getHeader().getDef().getField(i)).getName()),
+ batch.getHeader().getDef().getField(i).getMajorType());
+ }
+ return typeMap;
+ }
+
+ @SafeVarargs
+ private final void cleanupBatches(List<QueryDataBatch>... results) {
+ for (List<QueryDataBatch> resultList : results ) {
+ for (QueryDataBatch result : resultList) {
+ result.release();
+ }
+ }
+ }
+
+ public static void addToMaterializedResults(List<Map<String, Object>> materializedRecords,
+ List<QueryDataBatch> records,
+ RecordBatchLoader loader)
+ throws SchemaChangeException, UnsupportedEncodingException {
+ long totalRecords = 0;
+ QueryDataBatch batch;
+ int size = records.size();
+ for (int i = 0; i < size; i++) {
+ batch = records.get(0);
+ loader.load(batch.getHeader().getDef(), batch.getData());
+ // TODO: Clean: DRILL-2933: That load(...) no longer throws
+ // SchemaChangeException, so check/clean throws clause above.
+ logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords);
+ totalRecords += loader.getRecordCount();
+ for (int j = 0; j < loader.getRecordCount(); j++) {
+ Map<String, Object> record = new TreeMap<>();
+ for (VectorWrapper<?> w : loader) {
+ Object obj = w.getValueVector().getAccessor().getObject(j);
+ if (obj != null) {
+ if (obj instanceof Text) {
+ obj = obj.toString();
+ }
+ record.put(SchemaPath.getSimplePath(w.getField().getName()).toExpr(), obj);
+ }
+ record.put(SchemaPath.getSimplePath(w.getField().getName()).toExpr(), obj);
+ }
+ materializedRecords.add(record);
+ }
+ records.remove(0);
+ batch.release();
+ loader.clear();
+ }
+ }
+
+ public static boolean compareValuesErrorOnMismatch(Object expected, Object actual, int counter, String column) throws Exception {
+
+ if (compareValues(expected, actual, counter, column)) {
+ return true;
+ }
+ if (expected == null) {
+ throw new Exception("at position " + counter + " column '" + column + "' mismatched values, expected: null " +
+ "but received " + actual + "(" + actual.getClass().getSimpleName() + ")");
+ }
+ if (actual == null) {
+ throw new Exception("unexpected null at position " + counter + " column '" + column + "' should have been: " + expected);
+ }
+ if (actual instanceof byte[]) {
+ throw new Exception("at position " + counter + " column '" + column + "' mismatched values, expected: "
+ + new String((byte[])expected, "UTF-8") + " but received " + new String((byte[])actual, "UTF-8"));
+ }
+ if (!expected.equals(actual)) {
+ throw new Exception("at position " + counter + " column '" + column + "' mismatched values, expected: "
+ + expected + "(" + expected.getClass().getSimpleName() + ") but received " + actual + "(" + actual.getClass().getSimpleName() + ")");
+ }
+ return true;
+ }
+
+ public static boolean compareValues(Object expected, Object actual, int counter, String column) throws Exception {
+ if (expected == null) {
+ if (actual == null) {
+ if (VERBOSE_DEBUG) {
+ logger.debug("(1) at position " + counter + " column '" + column + "' matched value: " + expected );
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+ if (actual == null) {
+ return false;
+ }
+ if (actual instanceof byte[]) {
+ if (!Arrays.equals((byte[]) expected, (byte[]) actual)) {
+ return false;
+ } else {
+ if (VERBOSE_DEBUG) {
+ logger.debug("at position " + counter + " column '" + column + "' matched value " + new String((byte[])expected, "UTF-8"));
+ }
+ return true;
+ }
+ }
+ if (!expected.equals(actual)) {
+ return false;
+ } else {
+ if (VERBOSE_DEBUG) {
+ logger.debug("at position " + counter + " column '" + column + "' matched value: " + expected );
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Compare two result sets, ignoring ordering.
+ *
+ * @param expectedRecords - list of records from baseline
+ * @param actualRecords - list of records from test query, WARNING - this list is destroyed in this method
+ * @throws Exception
+ */
+ private void compareResults(List<Map<String, Object>> expectedRecords, List<Map<String, Object>> actualRecords) throws Exception {
+ assertEquals("Different number of records returned", expectedRecords.size(), actualRecords.size());
+
+ int i = 0;
+ int counter = 0;
+ boolean found;
+ for (Map<String, Object> expectedRecord : expectedRecords) {
+ i = 0;
+ found = false;
+ findMatch:
+ for (Map<String, Object> actualRecord : actualRecords) {
+ for (String s : actualRecord.keySet()) {
+ if (!expectedRecord.containsKey(s)) {
+ throw new Exception("Unexpected column '" + s + "' returned by query.");
+ }
+ if (!compareValues(expectedRecord.get(s), actualRecord.get(s), counter, s)) {
+ i++;
+ continue findMatch;
+ }
+ }
+ if (actualRecord.size() < expectedRecord.size()) {
+ throw new Exception(findMissingColumns(expectedRecord.keySet(), actualRecord.keySet()));
+ }
+ found = true;
+ break;
+ }
+ if (!found) {
+ StringBuilder sb = new StringBuilder();
+ for (int expectedRecordDisplayCount = 0;
+ expectedRecordDisplayCount < 10 && expectedRecordDisplayCount < expectedRecords.size();
+ expectedRecordDisplayCount++) {
+ sb.append(printRecord(expectedRecords.get(expectedRecordDisplayCount)));
+ }
+ String expectedRecordExamples = sb.toString();
+ sb.setLength(0);
+ for (int actualRecordDisplayCount = 0;
+ actualRecordDisplayCount < 10 && actualRecordDisplayCount < actualRecords.size();
+ actualRecordDisplayCount++) {
+ sb.append(printRecord(actualRecords.get(actualRecordDisplayCount)));
+ }
+ String actualRecordExamples = sb.toString();
+ throw new Exception(String.format("After matching %d records, did not find expected record in result set:\n %s\n\n" +
+ "Some examples of expected records:\n%s\n\n Some examples of records returned by the test query:\n%s",
+ counter, printRecord(expectedRecord), expectedRecordExamples, actualRecordExamples));
+ } else {
+ actualRecords.remove(i);
+ counter++;
+ }
+ }
+ assertEquals(0, actualRecords.size());
+ }
+
+ private static String findMissingColumns(Set<String> expected, Set<String> actual) {
+ String missingCols = "";
+ for (String colName : expected) {
+ if (!actual.contains(colName)) {
+ missingCols += colName + ", ";
+ }
+ }
+ return "Expected column(s) " + missingCols + " not found in result set: " + actual + ".";
+ }
+
+ private String printRecord(Map<String, ?> record) {
+ String ret = "";
+ for (String s : record.keySet()) {
+ ret += s + " : " + record.get(s) + ", ";
+ }
+ return ret + "\n";
+ }
+
+ private void test(String query) throws Exception {
+ services.test(query);
+ }
+
+ private List<QueryDataBatch> testRunAndReturn(QueryType type, Object query) throws Exception {
+ return services.testRunAndReturn(type, query);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
index 67ae4a3..8366b7a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
@@ -24,12 +24,21 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.memory.RootAllocator;
import org.apache.drill.exec.physical.impl.xsort.managed.ExternalSortBatch;
import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.test.LogFixture.LogFixtureBuilder;
import org.apache.drill.test.QueryBuilder.QuerySummary;
+import org.apache.drill.test.rowSet.RowSet;
+import org.apache.drill.test.rowSet.RowSetBuilder;
+import org.apache.drill.test.rowSet.SchemaBuilder;
+import org.apache.drill.test.rowSet.file.JsonFileBuilder;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import ch.qos.logback.classic.Level;
@@ -50,7 +59,6 @@ import ch.qos.logback.classic.Level;
* you can create your cluster fixture in a JUnit <tt>{@literal @}Before</tt>
* method, and shut it down in <tt>{@literal @}After</tt> method.
* <p>
- * See {@link org.apache.drill.test.package_info the package overview} for details.
*/
// Note: Test itself is ignored because this is an example, not a
@@ -60,6 +68,15 @@ import ch.qos.logback.classic.Level;
public class ExampleTest {
/**
+ * This test watcher creates all the temp directories that are required for an integration test with a Drillbit. The
+ * {@link ClusterFixture} and {@link BaseTestQuery} classes automatically configure their Drillbits to use the temp
+ * directories created by this test watcher. Please see {@link BaseDirTestWatcher} and package-info.java. Please see
+ * {@link #secondTest()} for an example.
+ */
+ @Rule
+ public final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
+ /**
* Example of the simplest possible test case: set up a default
* cluster (with one Drillbit), a corresponding client, run a
* query and print the results.
@@ -69,38 +86,61 @@ public class ExampleTest {
@Test
public void firstTest() throws Exception {
- try (ClusterFixture cluster = ClusterFixture.standardCluster();
+ try (ClusterFixture cluster = ClusterFixture.standardCluster(dirTestWatcher);
ClientFixture client = cluster.clientFixture()) {
client.queryBuilder().sql("SELECT * FROM `cp`.`employee.json` LIMIT 10").printCsv();
}
}
/**
- * Example that uses the fixture builder to build a cluster fixture. Lets
- * you set configuration (boot-time) options, session options, system options
- * and more.
* <p>
- * Also shows how to display the plan JSON and just run a query silently,
- * getting just the row count, batch count and run time.
- *
+ * Example that uses the fixture builder to build a cluster fixture. Lets
+ * you set configuration (boot-time) options, session options, system options
+ * and more.
+ * </p>
+ * <p>
+ * You can write test files to the {@link BaseDirTestWatcher#getRootDir()} and query them in the test.
+ * </p>
+ * <p>
+ * Also shows how to display the plan JSON and just run a query silently,
+ * getting just the row count, batch count and run time.
+ * </p>
* @throws Exception if anything goes wrong
*/
@Test
public void secondTest() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
- .configProperty(ExecConstants.SLICE_TARGET, 10)
- ;
+ try (RootAllocator allocator = new RootAllocator(100_000_000)) {
+ final File tableFile = dirTestWatcher
+ .getRootDir()
+ .toPath()
+ .resolve("employee.json")
+ .toFile();
- try (ClusterFixture cluster = builder.build();
- ClientFixture client = cluster.clientFixture()) {
- String sql = "SELECT * FROM `cp`.`employee.json` LIMIT 10";
- System.out.println( client.queryBuilder().sql(sql).explainJson() );
- QuerySummary results = client.queryBuilder().sql(sql).run();
- System.out.println(String.format("Read %d rows", results.recordCount()));
- // Usually we want to test something. Here, just test that we got
- // the 10 records.
- assertEquals(10, results.recordCount());
+ final BatchSchema schema = new SchemaBuilder()
+ .add("id", Types.required(TypeProtos.MinorType.VARCHAR))
+ .add("name", Types.required(TypeProtos.MinorType.VARCHAR))
+ .build();
+
+ final RowSet rowSet = new RowSetBuilder(allocator, schema)
+ .add("1", "kiwi")
+ .add("2", "watermelon")
+ .build();
+
+ new JsonFileBuilder(rowSet).build(tableFile);
+ rowSet.clear();
+
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).configProperty(ExecConstants.SLICE_TARGET, 10);
+
+ try (ClusterFixture cluster = builder.build(); ClientFixture client = cluster.clientFixture()) {
+ String sql = "SELECT * FROM `dfs`.`test/employee.json`";
+ System.out.println(client.queryBuilder().sql(sql).explainJson());
+ QuerySummary results = client.queryBuilder().sql(sql).run();
+ System.out.println(String.format("Read %d rows", results.recordCount()));
+ // Usually we want to test something. Here, just test that we got
+ // the 2 records.
+ assertEquals(2, results.recordCount());
+ }
}
}
@@ -125,7 +165,7 @@ public class ExampleTest {
@Test
public void thirdTest() throws Exception {
- try (ClusterFixture cluster = ClusterFixture.standardCluster();
+ try (ClusterFixture cluster = ClusterFixture.standardCluster(dirTestWatcher);
ClientFixture client = cluster.clientFixture()) {
String sql = "SELECT id_i, name_s10 FROM `mock`.`employees_5`";
client.queryBuilder().sql(sql).printCsv();
@@ -157,16 +197,15 @@ public class ExampleTest {
// All debug messages in the xsort package
.logger("org.apache.drill.exec.physical.impl.xsort", Level.DEBUG)
// And trace messages for one class.
- .logger(ExternalSortBatch.class, Level.TRACE)
- ;
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ .logger(ExternalSortBatch.class, Level.TRACE);
+
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
// Easy way to run single threaded for easy debugging
.maxParallelization(1)
// Set some session options
.sessionOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2L * 1024 * 1024 * 1024)
.sessionOption(PlannerSettings.EXCHANGE.getOptionName(), true)
- .sessionOption(PlannerSettings.HASHAGG.getOptionName(), false)
- ;
+ .sessionOption(PlannerSettings.HASHAGG.getOptionName(), false);
try (LogFixture logs = logBuilder.build();
ClusterFixture cluster = builder.build();
@@ -200,13 +239,11 @@ public class ExampleTest {
*
* @throws Exception if anything goes wrong
*/
-
@Test
public void fifthTest() throws Exception {
- ClusterFixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
.maxParallelization(1)
- .configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true)
- ;
+ .configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
@@ -231,7 +268,6 @@ public class ExampleTest {
*
* @param args not used
*/
-
public static void main(String args) {
try {
new ExampleTest().firstTest();
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/HadoopUtils.java b/exec/java-exec/src/test/java/org/apache/drill/test/HadoopUtils.java
new file mode 100644
index 0000000..ec6c91e
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/HadoopUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Paths;
+
+public class HadoopUtils {
+ public static final String LOCAL_FS_SCHEME = "file://";
+
+ public static org.apache.hadoop.fs.Path javaToHadoopPath(java.nio.file.Path javaPath) {
+ return new org.apache.hadoop.fs.Path(javaPath.toUri());
+ }
+
+ public static java.nio.file.Path hadoopToJavaPath(org.apache.hadoop.fs.Path hadoopPath) {
+ final String pathString = hadoopPath.toUri().getPath();
+ final URI uri;
+
+ try {
+ uri = new URI(LOCAL_FS_SCHEME + pathString);
+ } catch (URISyntaxException e) {
+ // This should never happen
+ throw new RuntimeException(e);
+ }
+
+ return Paths.get(uri);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/QueryBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/QueryBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/QueryBuilder.java
index e7bf61f..58f888d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/QueryBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/QueryBuilder.java
@@ -30,7 +30,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.drill.PlanTestBase;
-import org.apache.drill.QueryTestUtil;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/test/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/test/QueryTestUtil.java
new file mode 100644
index 0000000..db3e2ba
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/QueryTestUtil.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.drill.test.BaseTestQuery.SilentListener;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.client.PrintingResultsListener;
+import org.apache.drill.exec.client.QuerySubmitter.Format;
+import org.apache.drill.exec.compile.ClassTransformer;
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.proto.UserBitShared.QueryType;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.AwaitableUserResultsListener;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.SystemOptionManager;
+import org.apache.drill.exec.util.VectorUtil;
+
+/**
+ * Utilities useful for tests that issue SQL queries.
+ */
+public class QueryTestUtil {
+
+ public static final String TEST_QUERY_PRINTING_SILENT = "drill.test.query.printing.silent";
+
+ /**
+ * Constructor. All methods are static.
+ */
+ private QueryTestUtil() {
+ }
+
+ /**
+ * Create a DrillClient that can be used to query a drill cluster.
+ *
+ * @param drillConfig
+ * @param remoteServiceSet remote service set
+ * @param maxWidth maximum width per node
+ * @param props Connection properties contains properties such as "user", "password", "schema" etc
+ * @return the newly created client
+ * @throws RpcException if there is a problem setting up the client
+ */
+ public static DrillClient createClient(final DrillConfig drillConfig, final RemoteServiceSet remoteServiceSet,
+ final int maxWidth, final Properties props) throws RpcException, OutOfMemoryException {
+ final DrillClient drillClient = new DrillClient(drillConfig, remoteServiceSet.getCoordinator());
+ drillClient.connect(props);
+
+ final List<QueryDataBatch> results = drillClient.runQuery(
+ QueryType.SQL, String.format("alter session set `%s` = %d",
+ ExecConstants.MAX_WIDTH_PER_NODE_KEY, maxWidth));
+ for (QueryDataBatch queryDataBatch : results) {
+ queryDataBatch.release();
+ }
+
+ return drillClient;
+ }
+
+ /**
+ * Normalize the query relative to the test environment.
+ *
+ * <p>Looks for "${WORKING_PATH}" in the query string, and replaces it the current
+ * working patch obtained from {@link TestTools#WORKING_PATH}.
+ *
+ * @param query the query string
+ * @return the normalized query string
+ */
+ public static String normalizeQuery(final String query) {
+ if (query.contains("${WORKING_PATH}")) {
+ return query.replaceAll(Pattern.quote("${WORKING_PATH}"), Matcher.quoteReplacement(TestTools.WORKING_PATH.toString()));
+ } else if (query.contains("[WORKING_PATH]")) {
+ return query.replaceAll(Pattern.quote("[WORKING_PATH]"), Matcher.quoteReplacement(TestTools.WORKING_PATH.toString()));
+ }
+ return query;
+ }
+
+ /**
+ * Execute a SQL query, and print the results.
+ *
+ * @param drillClient drill client to use
+ * @param type type of the query
+ * @param queryString query string
+ * @return number of rows returned
+ * @throws Exception
+ */
+ public static int testRunAndPrint(
+ final DrillClient drillClient, final QueryType type, final String queryString) throws Exception {
+ final String query = normalizeQuery(queryString);
+ DrillConfig config = drillClient.getConfig();
+ AwaitableUserResultsListener resultListener =
+ new AwaitableUserResultsListener(
+ config.getBoolean(TEST_QUERY_PRINTING_SILENT) ?
+ new SilentListener() :
+ new PrintingResultsListener(config, Format.TSV, VectorUtil.DEFAULT_COLUMN_WIDTH)
+ );
+ drillClient.runQuery(type, query, resultListener);
+ return resultListener.await();
+ }
+
+ /**
+ * Execute one or more queries separated by semicolons, and print the results.
+ *
+ * @param drillClient drill client to use
+ * @param queryString the query string
+ * @throws Exception
+ */
+ public static void test(final DrillClient drillClient, final String queryString) throws Exception{
+ final String query = normalizeQuery(queryString);
+ String[] queries = query.split(";");
+ for (String q : queries) {
+ final String trimmedQuery = q.trim();
+ if (trimmedQuery.isEmpty()) {
+ continue;
+ }
+ testRunAndPrint(drillClient, QueryType.SQL, trimmedQuery);
+ }
+ }
+
+ /**
+ * Execute one or more queries separated by semicolons, and print the results, with the option to
+ * add formatted arguments to the query string.
+ *
+ * @param drillClient drill client to use
+ * @param query the query string; may contain formatting specifications to be used by
+ * {@link String#format(String, Object...)}.
+ * @param args optional args to use in the formatting call for the query string
+ * @throws Exception
+ */
+ public static void test(final DrillClient drillClient, final String query, Object... args) throws Exception {
+ test(drillClient, String.format(query, args));
+ }
+
+ /**
+ * Execute a single query with a user supplied result listener.
+ *
+ * @param drillClient drill client to use
+ * @param type type of query
+ * @param queryString the query string
+ * @param resultListener the result listener
+ */
+ public static void testWithListener(final DrillClient drillClient, final QueryType type,
+ final String queryString, final UserResultsListener resultListener) {
+ final String query = QueryTestUtil.normalizeQuery(queryString);
+ drillClient.runQuery(type, query, resultListener);
+ }
+
+ /**
+ * Set up the options to test the scalar replacement retry option (see
+ * ClassTransformer.java). Scalar replacement rewrites bytecode to replace
+ * value holders (essentially boxed values) with their member variables as
+ * locals. There is still one pattern that doesn't work, and occasionally new
+ * ones are introduced. This can be used in tests that exercise failing patterns.
+ *
+ * <p>This also flushes the compiled code cache.
+ *
+ * @param drillbit the drillbit
+ * @param srOption the scalar replacement option value to use
+ * @return the original scalar replacement option setting (so it can be restored)
+ */
+ @SuppressWarnings("resource")
+ public static OptionValue setupScalarReplacementOption(
+ final Drillbit drillbit, final ClassTransformer.ScalarReplacementOption srOption) {
+ // set the system option
+ final DrillbitContext drillbitContext = drillbit.getContext();
+ final SystemOptionManager optionManager = drillbitContext.getOptionManager();
+ final OptionValue originalOptionValue = optionManager.getOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION);
+ optionManager.setLocalOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase());
+
+ // flush the code cache
+ drillbitContext.getCompiler().flushCache();
+
+ return originalOptionValue;
+ }
+
+ /**
+ * Restore the original scalar replacement option returned from
+ * setupScalarReplacementOption().
+ *
+ * <p>This also flushes the compiled code cache.
+ *
+ * @param drillbit the drillbit
+ * @param srOption the scalar replacement option value to use
+ */
+ public static void restoreScalarReplacementOption(final Drillbit drillbit, final String srOption) {
+ @SuppressWarnings("resource")
+ final DrillbitContext drillbitContext = drillbit.getContext();
+ @SuppressWarnings("resource")
+ final OptionManager optionManager = drillbitContext.getOptionManager();
+ optionManager.setLocalOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption);
+
+ // flush the code cache
+ drillbitContext.getCompiler().flushCache();
+ }
+
+}
[18/22] drill git commit: DRILL-5783, DRILL-5841,
DRILL-5894: Rationalize test temp directories
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 0fafe53..bcfe8f3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -17,32 +17,38 @@
*/
package org.apache.drill;
-import static org.apache.drill.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.listOf;
import static org.junit.Assert.assertEquals;
import java.math.BigDecimal;
+import java.nio.file.Paths;
import org.apache.drill.categories.OperatorTest;
import org.apache.drill.categories.PlannerTest;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.test.BaseTestQuery;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({SqlFunctionTest.class, OperatorTest.class, PlannerTest.class})
public class TestExampleQueries extends BaseTestQuery {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class);
+ @BeforeClass
+ public static void setupTestFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("tpchmulti"));
+ dirTestWatcher.copyResourceToRoot(Paths.get("store","text"));
+ }
@Test // see DRILL-2328
@Category(UnlikelyTest.class)
public void testConcatOnNull() throws Exception {
try {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view concatNull as (select * from cp.`customer.json` where customer_id < 5);");
// Test Left Null
@@ -131,15 +137,15 @@ public class TestExampleQueries extends BaseTestQuery {
@Test // see DRILL-985
public void testViewFileName() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testexamplequeries as select * from cp.`tpch/nation.parquet`;");
- test("select * from dfs_test.tmp.`nation_view_testexamplequeries.view.drill`");
+ test("select * from dfs.tmp.`nation_view_testexamplequeries.view.drill`");
test("drop view nation_view_testexamplequeries");
}
@Test
public void testTextInClasspathStorage() throws Exception {
- test("select * from cp.`/store/text/classpath_storage_csv_test.csv`");
+ test("select * from cp.`store/text/classpath_storage_csv_test.csv`");
}
@Test
@@ -207,11 +213,9 @@ public class TestExampleQueries extends BaseTestQuery {
@Test
public void testPushExpInJoinConditionLeftJoin() throws Exception {
- test("select a.n_nationkey, b.r_regionkey from cp.`tpch/nation.parquet` a left join cp.`tpch/region.parquet` b " + "" +
- " on a.n_regionkey +100 = b.r_regionkey +200 " + // expressions in both sides of equal join filter
- // " and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') " + // left filter
- " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') "); // right filter
- // " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
+ test("select a.n_nationkey, b.r_regionkey from cp.`tpch/nation.parquet` a left join cp.`tpch/region.parquet` b " +
+ " on a.n_regionkey +100 = b.r_regionkey +200 " + // expressions in both sides of equal join filter // left filter
+ " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') "); // right filter // non-equal join filter
}
@Test
@@ -219,8 +223,6 @@ public class TestExampleQueries extends BaseTestQuery {
test("select a.n_nationkey, b.r_regionkey from cp.`tpch/nation.parquet` a right join cp.`tpch/region.parquet` b " + "" +
" on a.n_regionkey +100 = b.r_regionkey +200 " + // expressions in both sides of equal join filter
" and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') "); // left filter
- // " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') " + // right filter
- // " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
}
@Test
@@ -273,25 +275,19 @@ public class TestExampleQueries extends BaseTestQuery {
@Test
public void testText() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
- String query = String.format("select * from dfs_test.`%s`", root);
- test(query);
+ test("select * from cp.`store/text/data/regions.csv`");
}
@Test
public void testFilterOnArrayTypes() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
- String query = String.format("select columns[0] from dfs_test.`%s` " +
- " where cast(columns[0] as int) > 1 and cast(columns[1] as varchar(20))='ASIA'", root);
- test(query);
+ test("select columns[0] from cp.`store/text/data/regions.csv` " +
+ " where cast(columns[0] as int) > 1 and cast(columns[1] as varchar(20))='ASIA'");
}
@Test
@Ignore("DRILL-3774")
public void testTextPartitions() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/").toURI().toString();
- String query = String.format("select * from dfs_test.`%s`", root);
- test(query);
+ test("select * from cp.`store/text/data/`");
}
@Test
@@ -381,43 +377,33 @@ public class TestExampleQueries extends BaseTestQuery {
@Test
public void testTextJoin() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/nations.csv").toURI().toString();
- String root1 = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
- String query = String.format("select t1.columns[1] from dfs_test.`%s` t1, dfs_test.`%s` t2 where t1.columns[0] = t2.columns[0]", root, root1);
- test(query);
+ test("select t1.columns[1] from cp.`store/text/data/nations.csv` t1, cp.`store/text/data/regions.csv` t2 where t1.columns[0] = t2.columns[0]");
}
@Test // DRILL-811
public void testDRILL_811View() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testexamplequeries as select * from cp.`tpch/nation.parquet`;");
-
test("select n.n_nationkey, n.n_name, n.n_regionkey from nation_view_testexamplequeries n where n.n_nationkey > 8 order by n.n_regionkey");
-
test("select n.n_regionkey, count(*) as cnt from nation_view_testexamplequeries n where n.n_nationkey > 8 group by n.n_regionkey order by n.n_regionkey");
-
test("drop view nation_view_testexamplequeries ");
}
@Test // DRILL-811
public void testDRILL_811ViewJoin() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view nation_view_testexamplequeries as select * from cp.`tpch/nation.parquet`;");
test("create view region_view_testexamplequeries as select * from cp.`tpch/region.parquet`;");
-
test("select n.n_nationkey, n.n_regionkey, r.r_name from region_view_testexamplequeries r , nation_view_testexamplequeries n where r.r_regionkey = n.n_regionkey ");
-
test("select n.n_regionkey, count(*) as cnt from region_view_testexamplequeries r , nation_view_testexamplequeries n where r.r_regionkey = n.n_regionkey and n.n_nationkey > 8 group by n.n_regionkey order by n.n_regionkey");
-
test("select n.n_regionkey, count(*) as cnt from region_view_testexamplequeries r join nation_view_testexamplequeries n on r.r_regionkey = n.n_regionkey and n.n_nationkey > 8 group by n.n_regionkey order by n.n_regionkey");
-
test("drop view region_view_testexamplequeries ");
test("drop view nation_view_testexamplequeries ");
}
@Test // DRILL-811
public void testDRILL_811Json() throws Exception {
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test("create view region_view_testexamplequeries as select * from cp.`region.json`;");
test("select sales_city, sales_region from region_view_testexamplequeries where region_id > 50 order by sales_country; ");
test("drop view region_view_testexamplequeries ");
@@ -435,7 +421,7 @@ public class TestExampleQueries extends BaseTestQuery {
test("select t1.full_name from cp.`employee.json` t1, cp.`department.json` t2 where cast(t1.department_id as double) = t2.department_id and cast(t1.position_id as bigint) = t2.department_id");
// See DRILL-3995. Re-enable this once fixed.
-// test("select t1.full_name from cp.`employee.json` t1, cp.`department.json` t2 where t1.department_id = t2.department_id and t1.position_id = t2.department_id");
+ // test("select t1.full_name from cp.`employee.json` t1, cp.`department.json` t2 where t1.department_id = t2.department_id and t1.position_id = t2.department_id");
}
@Test
@@ -580,8 +566,8 @@ public class TestExampleQueries extends BaseTestQuery {
expectedRecordCount, actualRecordCount), expectedRecordCount, actualRecordCount);
// source is CSV
- String root = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
- String query = String.format("select rid, x.name from (select columns[0] as RID, columns[1] as NAME from dfs_test.`%s`) X where X.rid = 2", root);
+ String root = DrillFileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
+ String query = String.format("select rid, x.name from (select columns[0] as RID, columns[1] as NAME from dfs.`%s`) X where X.rid = 2", root);
actualRecordCount = testSql(query);
expectedRecordCount = 1;
assertEquals(String.format("Received unexpected number of rows in output: expected=%d, received=%s",
@@ -668,10 +654,7 @@ public class TestExampleQueries extends BaseTestQuery {
@Test
public void testExchangeRemoveForJoinPlan() throws Exception {
- final String WORKING_PATH = TestTools.getWorkingPath();
- final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
-
- String sql = String.format("select t2.n_nationkey from dfs_test.`%s/tpchmulti/region` t1 join dfs_test.`%s/tpchmulti/nation` t2 on t2.n_regionkey = t1.r_regionkey", TEST_RES_PATH, TEST_RES_PATH);
+ String sql = "select t2.n_nationkey from dfs.`tpchmulti/region` t1 join dfs.`tpchmulti/nation` t2 on t2.n_regionkey = t1.r_regionkey";
testBuilder()
.unOrdered()
@@ -679,18 +662,16 @@ public class TestExampleQueries extends BaseTestQuery {
.sqlQuery(sql)
.optionSettingQueriesForBaseline("alter session set `planner.slice_target` = 100000; alter session set `planner.join.row_count_estimate_factor` = 1.0") // Use default option setting.
.sqlBaselineQuery(sql)
- .build().run();
-
+ .build()
+ .run();
}
@Test //DRILL-2163
public void testNestedTypesPastJoinReportsValidResult() throws Exception {
- final String query = "select t1.uid, t1.events, t1.events[0].evnt_id as event_id, t2.transactions, " +
- "t2.transactions[0] as trans, t1.odd, t2.even from cp.`project/complex/a.json` t1, " +
- "cp.`project/complex/b.json` t2 where t1.uid = t2.uid";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select t1.uid, t1.events, t1.events[0].evnt_id as event_id, t2.transactions, " +
+ "t2.transactions[0] as trans, t1.odd, t2.even from cp.`project/complex/a.json` t1, " +
+ "cp.`project/complex/b.json` t2 where t1.uid = t2.uid")
.ordered()
.jsonBaselineFile("project/complex/drill-2163-result.json")
.build()
@@ -700,13 +681,11 @@ public class TestExampleQueries extends BaseTestQuery {
@Test
@Category(UnlikelyTest.class)
public void testSimilar() throws Exception {
- String query = "select n_nationkey " +
- "from cp.`tpch/nation.parquet` " +
- "where n_name similar to 'CHINA' " +
- "order by n_regionkey";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select n_nationkey " +
+ "from cp.`tpch/nation.parquet` " +
+ "where n_name similar to 'CHINA' " +
+ "order by n_regionkey")
.unOrdered()
.optionSettingQueriesForTestQuery("alter session set `planner.slice_target` = 1")
.baselineColumns("n_nationkey")
@@ -723,7 +702,7 @@ public class TestExampleQueries extends BaseTestQuery {
"select cast(r_regionkey as BIGINT) BIGINT_col, cast(r_regionkey as DECIMAL) bigint_col \n" +
"FROM cp.`tpch/region.parquet`;\n";
- test("USE dfs_test.tmp");
+ test("USE dfs.tmp");
test(creatTable);
testBuilder()
@@ -763,14 +742,10 @@ public class TestExampleQueries extends BaseTestQuery {
@Test // DRILL-2094
public void testOrderbyArrayElement() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/json/orderByArrayElement.json").toURI().toString();
-
- String query = String.format("select t.id, t.list[0] as SortingElem " +
- "from dfs_test.`%s` t " +
- "order by t.list[0]", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select t.id, t.list[0] as SortingElem " +
+ "from cp.`store/json/orderByArrayElement.json` t " +
+ "order by t.list[0]")
.ordered()
.baselineColumns("id", "SortingElem")
.baselineValues((long) 1, (long) 1)
@@ -783,12 +758,10 @@ public class TestExampleQueries extends BaseTestQuery {
@Test // DRILL-2479
public void testCorrelatedExistsWithInSubq() throws Exception {
- String query = "select count(*) as cnt from cp.`tpch/lineitem.parquet` l where exists "
- + " (select ps.ps_suppkey from cp.`tpch/partsupp.parquet` ps where ps.ps_suppkey = l.l_suppkey and ps.ps_partkey "
- + " in (select p.p_partkey from cp.`tpch/part.parquet` p where p.p_type like '%NICKEL'))";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select count(*) as cnt from cp.`tpch/lineitem.parquet` l where exists "
+ + " (select ps.ps_suppkey from cp.`tpch/partsupp.parquet` ps where ps.ps_suppkey = l.l_suppkey and ps.ps_partkey "
+ + " in (select p.p_partkey from cp.`tpch/part.parquet` p where p.p_type like '%NICKEL'))")
.unOrdered()
.baselineColumns("cnt")
.baselineValues(60175l)
@@ -797,15 +770,11 @@ public class TestExampleQueries extends BaseTestQuery {
@Test // DRILL-2094
public void testOrderbyArrayElementInSubquery() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/json/orderByArrayElement.json").toURI().toString();
-
- String query = String.format("select s.id from \n" +
- "(select id \n" +
- "from dfs_test.`%s` \n" +
- "order by list[0]) s", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select s.id from " +
+ "(select id " +
+ "from cp.`store/json/orderByArrayElement.json` " +
+ "order by list[0]) s")
.ordered()
.baselineColumns("id")
.baselineValues((long) 1)
@@ -818,17 +787,16 @@ public class TestExampleQueries extends BaseTestQuery {
@Test // DRILL-1978
public void testCTASOrderByCoumnNotInSelectClause() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
String queryCTAS1 = "CREATE TABLE TestExampleQueries_testCTASOrderByCoumnNotInSelectClause1 as " +
"select r_name from cp.`tpch/region.parquet` order by r_regionkey;";
- String queryCTAS2 = String.format("CREATE TABLE TestExampleQueries_testCTASOrderByCoumnNotInSelectClause2 as " +
- "SELECT columns[1] as col FROM dfs_test.`%s` ORDER BY cast(columns[0] as double)", root);
+ String queryCTAS2 = "CREATE TABLE TestExampleQueries_testCTASOrderByCoumnNotInSelectClause2 as " +
+ "SELECT columns[1] as col FROM cp.`store/text/data/regions.csv` ORDER BY cast(columns[0] as double)";
String query1 = "select * from TestExampleQueries_testCTASOrderByCoumnNotInSelectClause1";
String query2 = "select * from TestExampleQueries_testCTASOrderByCoumnNotInSelectClause2";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
test(queryCTAS1);
test(queryCTAS2);
@@ -858,12 +826,11 @@ public class TestExampleQueries extends BaseTestQuery {
@Test // DRILL-2221
public void createJsonWithEmptyList() throws Exception {
- final String file = FileUtils.getResourceAsFile("/store/json/record_with_empty_list.json").toURI().toString();
final String tableName = "jsonWithEmptyList";
- test("USE dfs_test.tmp");
+ test("USE dfs.tmp");
test("ALTER SESSION SET `store.format`='json'");
- test(String.format("CREATE TABLE %s AS SELECT * FROM `%s`", tableName, file));
- test(String.format("SELECT COUNT(*) FROM %s", tableName));
+ test("CREATE TABLE %s AS SELECT * FROM cp.`store/json/record_with_empty_list.json`", tableName);
+ test("SELECT COUNT(*) FROM %s", tableName);
test("ALTER SESSION SET `store.format`='parquet'");
}
@@ -986,15 +953,11 @@ public class TestExampleQueries extends BaseTestQuery {
@Test //DRILL-2953
public void testGbAndObDifferentExp() throws Exception {
- String root = FileUtils.getResourceAsFile("/store/text/data/nations.csv").toURI().toString();
- String query = String.format(
- "select cast(columns[0] as int) as nation_key " +
- " from dfs_test.`%s` " +
- " group by columns[0] " +
- " order by cast(columns[0] as int)", root);
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select cast(columns[0] as int) as nation_key " +
+ " from cp.`store/text/data/nations.csv` " +
+ " group by columns[0] " +
+ " order by cast(columns[0] as int)")
.ordered()
.csvBaselineFile("testframework/testExampleQueries/testGroupByStarSchemaless.tsv")
.baselineTypes(TypeProtos.MinorType.INT)
@@ -1002,14 +965,11 @@ public class TestExampleQueries extends BaseTestQuery {
.build()
.run();
- String query2 = String.format(
- "select cast(columns[0] as int) as nation_key " +
- " from dfs_test.`%s` " +
- " group by cast(columns[0] as int) " +
- " order by cast(columns[0] as int)", root);
-
testBuilder()
- .sqlQuery(query2)
+ .sqlQuery("select cast(columns[0] as int) as nation_key " +
+ " from cp.`store/text/data/nations.csv` " +
+ " group by cast(columns[0] as int) " +
+ " order by cast(columns[0] as int)")
.ordered()
.csvBaselineFile("testframework/testExampleQueries/testGroupByStarSchemaless.tsv")
.baselineTypes(TypeProtos.MinorType.INT)
@@ -1040,7 +1000,6 @@ public class TestExampleQueries extends BaseTestQuery {
"ALTER SESSION SET `planner.disable_exchanges` = true")
.build()
.run();
-
}
@Test
@@ -1057,18 +1016,18 @@ public class TestExampleQueries extends BaseTestQuery {
@Test
@Ignore
public void testPartitionCTAS() throws Exception {
- test("use dfs_test.tmp; " +
+ test("use dfs.tmp; " +
"create table mytable1 partition by (r_regionkey, r_comment) as select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`");
- test("use dfs_test.tmp; " +
+ test("use dfs.tmp; " +
"create table mytable2 partition by (r_regionkey, r_comment) as select * from cp.`tpch/region.parquet` where r_name = 'abc' ");
- test("use dfs_test.tmp; " +
+ test("use dfs.tmp; " +
"create table mytable3 partition by (r_regionkey, n_nationkey) as " +
" select r.r_regionkey, r.r_name, n.n_nationkey, n.n_name from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r " +
" where n.n_regionkey = r.r_regionkey");
- test("use dfs_test.tmp; " +
+ test("use dfs.tmp; " +
"create table mytable4 partition by (r_regionkey, r_comment) as " +
" select r.* from cp.`tpch/nation.parquet` n, cp.`tpch/region.parquet` r " +
" where n.n_regionkey = r.r_regionkey");
@@ -1115,26 +1074,18 @@ public class TestExampleQueries extends BaseTestQuery {
@Test // see DRILL-3557
@Category(UnlikelyTest.class)
public void testEmptyCSVinDirectory() throws Exception {
- final String root = FileUtils.getResourceAsFile("/store/text/directoryWithEmpyCSV").toURI().toString();
- final String toFile = FileUtils.getResourceAsFile("/store/text/directoryWithEmpyCSV/empty.csv").toURI().toString();
-
- String query1 = String.format("explain plan for select * from dfs_test.`%s`", root);
- String query2 = String.format("explain plan for select * from dfs_test.`%s`", toFile);
-
- test(query1);
- test(query2);
+ test("explain plan for select * from dfs.`store/text/directoryWithEmpyCSV`");
+ test("explain plan for select * from cp.`store/text/directoryWithEmpyCSV/empty.csv`");
}
@Test
@Category(UnlikelyTest.class)
public void testNegativeExtractOperator() throws Exception {
- String query = "select -EXTRACT(DAY FROM birth_date) as col \n" +
- "from cp.`employee.json` \n" +
- "order by col \n" +
- "limit 5";
-
testBuilder()
- .sqlQuery(query)
+ .sqlQuery("select -EXTRACT(DAY FROM birth_date) as col \n" +
+ "from cp.`employee.json` \n" +
+ "order by col \n" +
+ "limit 5")
.ordered()
.baselineColumns("col")
.baselineValues(-27l)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
index a302b71..1b551ec 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
@@ -17,8 +17,8 @@
******************************************************************************/
package org.apache.drill;
-import static org.apache.drill.TestBuilder.listOf;
-import static org.apache.drill.TestBuilder.mapOf;
+import static org.apache.drill.test.TestBuilder.listOf;
+import static org.apache.drill.test.TestBuilder.mapOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -34,6 +34,8 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.TestBuilder;
import org.hamcrest.CoreMatchers;
import org.junit.Test;
@@ -43,7 +45,7 @@ import com.google.common.collect.Lists;
// currently using it with the assumption that the csv and json readers are well tested, and handling diverse
// types in the test framework would require doing some redundant work to enable casting outside of Drill or
// some better tooling to generate parquet files that have all of the parquet types
-public class TestFrameworkTest extends BaseTestQuery{
+public class TestFrameworkTest extends BaseTestQuery {
private static String CSV_COLS = " cast(columns[0] as bigint) employee_id, columns[1] as first_name, columns[2] as last_name ";
@@ -171,7 +173,7 @@ public class TestFrameworkTest extends BaseTestQuery{
@Test
public void testMapOrdering() throws Exception {
testBuilder()
- .sqlQuery("select * from cp.`/testframework/map_reordering.json`")
+ .sqlQuery("select * from cp.`testframework/map_reordering.json`")
.unOrdered()
.jsonBaselineFile("testframework/map_reordering2.json")
.build().run();
@@ -203,7 +205,7 @@ public class TestFrameworkTest extends BaseTestQuery{
@Test
public void testBaselineValsVerificationWithComplexAndNulls() throws Exception {
testBuilder()
- .sqlQuery("select * from cp.`/jsoninput/input2.json` limit 1")
+ .sqlQuery("select * from cp.`jsoninput/input2.json` limit 1")
.ordered()
.baselineColumns("integer", "float", "x", "z", "l", "rl")
.baselineValues(2010l,
@@ -241,7 +243,7 @@ public class TestFrameworkTest extends BaseTestQuery{
public void testCSVVerification_extra_records_fails() throws Exception {
try {
testBuilder()
- .sqlQuery("select " + CSV_COLS + " from cp.`testframework/small_test_data_extra.tsv`")
+ .sqlQuery("select %s from cp.`testframework/small_test_data_extra.tsv`", CSV_COLS)
.ordered()
.csvBaselineFile("testframework/small_test_data.tsv")
.baselineTypes(TypeProtos.MinorType.BIGINT, TypeProtos.MinorType.VARCHAR, TypeProtos.MinorType.VARCHAR)
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
index 1e8fc44..0750d61 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
@@ -19,6 +19,7 @@ package org.apache.drill;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.test.BaseTestQuery;
import org.joda.time.DateTime;
import org.junit.AfterClass;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java
index e73fd41..88558ae 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsWithTypeExpoQueries.java
@@ -22,14 +22,21 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.test.BaseTestQuery;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.nio.file.Paths;
import java.util.List;
@Category(SqlFunctionTest.class)
public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("typeExposure"));
+ }
+
@Test
public void testConcatWithMoreThanTwoArgs() throws Exception {
final String query = "select concat(r_name, r_name, r_name, 'f') as col \n" +
@@ -52,24 +59,20 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testRow_NumberInView() throws Exception {
try {
- test("use dfs_test.tmp;");
- final String view1 =
- "create view TestFunctionsWithTypeExpoQueries_testViewShield1 as \n" +
- "select rnum, position_id, " +
- " ntile(4) over(order by position_id) " +
- " from (select position_id, row_number() " +
- " over(order by position_id) as rnum " +
- " from cp.`employee.json`)";
-
- final String view2 =
- "create view TestFunctionsWithTypeExpoQueries_testViewShield2 as \n" +
- "select row_number() over(order by position_id) as rnum, " +
- " position_id, " +
- " ntile(4) over(order by position_id) " +
- " from cp.`employee.json`";
-
- test(view1);
- test(view2);
+ test("use dfs.tmp;");
+ test(
+ "create view TestFunctionsWithTypeExpoQueries_testViewShield1 as \n" +
+ "select rnum, position_id, " +
+ " ntile(4) over(order by position_id) " +
+ " from (select position_id, row_number() " +
+ " over(order by position_id) as rnum " +
+ " from cp.`employee.json`)");
+ test(
+ "create view TestFunctionsWithTypeExpoQueries_testViewShield2 as \n" +
+ "select row_number() over(order by position_id) as rnum, " +
+ " position_id, " +
+ " ntile(4) over(order by position_id) " +
+ " from cp.`employee.json`");
testBuilder()
.sqlQuery("select * from TestFunctionsWithTypeExpoQueries_testViewShield1")
.ordered()
@@ -253,10 +256,9 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testMetaDataExposeType() throws Exception {
- final String root = FileUtils.getResourceAsFile("/typeExposure/metadata_caching").toURI().toString();
- final String query = String.format("select count(*) as col \n" +
- "from dfs_test.`%s` \n" +
- "where concat(a, 'asdf') = 'asdf'", root);
+ final String query = "select count(*) as col " +
+ "from dfs.`typeExposure/metadata_caching` " +
+ "where concat(a, 'asdf') = 'asdf'";
// Validate the plan
final String[] expectedPlan = {"Scan.*a.parquet.*numFiles = 1"};
@@ -275,8 +277,8 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testDate_Part() throws Exception {
- final String query = "select date_part('year', date '2008-2-23') as col \n" +
- "from cp.`tpch/region.parquet` \n" +
+ final String query = "select date_part('year', date '2008-2-23') as col " +
+ "from cp.`tpch/region.parquet` " +
"limit 0";
List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -295,7 +297,7 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testNegativeByInterpreter() throws Exception {
- final String query = "select * from cp.`tpch/region.parquet` \n" +
+ final String query = "select * from cp.`tpch/region.parquet` " +
"where r_regionkey = negative(-1)";
// Validate the plan
@@ -306,10 +308,10 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testSumRequiredType() throws Exception {
- final String query = "SELECT \n" +
- "SUM(CASE WHEN (CAST(n_regionkey AS INT) = 1) THEN 1 ELSE 0 END) AS col \n" +
- "FROM cp.`tpch/nation.parquet` \n" +
- "GROUP BY CAST(n_regionkey AS INT) \n" +
+ final String query = "SELECT " +
+ "SUM(CASE WHEN (CAST(n_regionkey AS INT) = 1) THEN 1 ELSE 0 END) AS col " +
+ "FROM cp.`tpch/nation.parquet` " +
+ "GROUP BY CAST(n_regionkey AS INT) " +
"limit 0";
List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -328,8 +330,8 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testSQRTDecimalLiteral() throws Exception {
- final String query = "SELECT sqrt(5.1) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String query = "SELECT sqrt(5.1) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -348,8 +350,8 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testSQRTIntegerLiteral() throws Exception {
- final String query = "SELECT sqrt(4) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String query = "SELECT sqrt(4) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -368,8 +370,8 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testTimestampDiff() throws Exception {
- final String query = "select to_timestamp('2014-02-13 00:30:30','YYYY-MM-dd HH:mm:ss') - to_timestamp('2014-02-13 00:30:30','YYYY-MM-dd HH:mm:ss') as col \n" +
- "from cp.`tpch/region.parquet` \n" +
+ final String query = "select to_timestamp('2014-02-13 00:30:30','YYYY-MM-dd HH:mm:ss') - to_timestamp('2014-02-13 00:30:30','YYYY-MM-dd HH:mm:ss') as col " +
+ "from cp.`tpch/region.parquet` " +
"limit 0";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -388,9 +390,9 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testEqualBetweenIntervalAndTimestampDiff() throws Exception {
- final String query = "select to_timestamp('2016-11-02 10:00:00','YYYY-MM-dd HH:mm:ss') + interval '10-11' year to month as col \n" +
- "from cp.`tpch/region.parquet` \n" +
- "where (to_timestamp('2016-11-02 10:00:00','YYYY-MM-dd HH:mm:ss') - to_timestamp('2016-01-01 10:00:00','YYYY-MM-dd HH:mm:ss') < interval '5 10:00:00' day to second) \n" +
+ final String query = "select to_timestamp('2016-11-02 10:00:00','YYYY-MM-dd HH:mm:ss') + interval '10-11' year to month as col " +
+ "from cp.`tpch/region.parquet` " +
+ "where (to_timestamp('2016-11-02 10:00:00','YYYY-MM-dd HH:mm:ss') - to_timestamp('2016-01-01 10:00:00','YYYY-MM-dd HH:mm:ss') < interval '5 10:00:00' day to second) " +
"limit 0";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -409,11 +411,11 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testAvgAndSUM() throws Exception {
- final String query = "SELECT AVG(cast(r_regionkey as float)) AS `col1`, \n" +
- "SUM(cast(r_regionkey as float)) AS `col2`, \n" +
- "SUM(1) AS `col3` \n" +
- "FROM cp.`tpch/region.parquet` \n" +
- "GROUP BY CAST(r_regionkey AS INTEGER) \n" +
+ final String query = "SELECT AVG(cast(r_regionkey as float)) AS `col1`, " +
+ "SUM(cast(r_regionkey as float)) AS `col2`, " +
+ "SUM(1) AS `col3` " +
+ "FROM cp.`tpch/region.parquet` " +
+ "GROUP BY CAST(r_regionkey AS INTEGER) " +
"LIMIT 0";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -445,10 +447,10 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testAvgCountStar() throws Exception {
- final String query = "select avg(distinct cast(r_regionkey as bigint)) + avg(cast(r_regionkey as integer)) as col1, \n" +
- "sum(distinct cast(r_regionkey as bigint)) + 100 as col2, count(*) as col3 \n" +
- "from cp.`tpch/region.parquet` alltypes_v \n" +
- "where cast(r_regionkey as bigint) = 100000000000000000 \n" +
+ final String query = "select avg(distinct cast(r_regionkey as bigint)) + avg(cast(r_regionkey as integer)) as col1, " +
+ "sum(distinct cast(r_regionkey as bigint)) + 100 as col2, count(*) as col3 " +
+ "from cp.`tpch/region.parquet` alltypes_v " +
+ "where cast(r_regionkey as bigint) = 100000000000000000 " +
"limit 0";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -479,15 +481,15 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testUDFInGroupBy() throws Exception {
- final String query = "select count(*) as col1, substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2) as col2, \n" +
- "char_length(substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2)) as col3 \n" +
- "from cp.`tpch/region.parquet` t1 \n" +
- "left outer join cp.`tpch/nation.parquet` t2 on cast(t1.r_regionkey as Integer) = cast(t2.n_nationkey as Integer) \n" +
- "left outer join cp.`employee.json` t3 on cast(t1.r_regionkey as Integer) = cast(t3.employee_id as Integer) \n" +
- "group by substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2), \n" +
- "char_length(substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2)) \n" +
- "order by substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2),\n" +
- "char_length(substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2)) \n" +
+ final String query = "select count(*) as col1, substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2) as col2, " +
+ "char_length(substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2)) as col3 " +
+ "from cp.`tpch/region.parquet` t1 " +
+ "left outer join cp.`tpch/nation.parquet` t2 on cast(t1.r_regionkey as Integer) = cast(t2.n_nationkey as Integer) " +
+ "left outer join cp.`employee.json` t3 on cast(t1.r_regionkey as Integer) = cast(t3.employee_id as Integer) " +
+ "group by substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2), " +
+ "char_length(substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2)) " +
+ "order by substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2)," +
+ "char_length(substr(lower(UPPER(cast(t3.full_name as varchar(100)))), 5, 2)) " +
"limit 0";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -518,12 +520,12 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testWindowSumAvg() throws Exception {
- final String query = "with query as ( \n" +
- "select sum(cast(employee_id as integer)) over w as col1, cast(avg(cast(employee_id as bigint)) over w as double precision) as col2, count(*) over w as col3 \n" +
- "from cp.`tpch/region.parquet` \n" +
- "window w as (partition by cast(full_name as varchar(10)) order by cast(full_name as varchar(10)) nulls first)) \n" +
- "select * \n" +
- "from query \n" +
+ final String query = "with query as ( " +
+ "select sum(cast(employee_id as integer)) over w as col1, cast(avg(cast(employee_id as bigint)) over w as double precision) as col2, count(*) over w as col3 " +
+ "from cp.`tpch/region.parquet` " +
+ "window w as (partition by cast(full_name as varchar(10)) order by cast(full_name as varchar(10)) nulls first)) " +
+ "select * " +
+ "from query " +
"limit 0";
final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = Lists.newArrayList();
@@ -554,24 +556,24 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testWindowRanking() throws Exception {
- final String queryCUME_DIST = "select CUME_DIST() over(order by n_nationkey) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String queryCUME_DIST = "select CUME_DIST() over(order by n_nationkey) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
- final String queryDENSE_RANK = "select DENSE_RANK() over(order by n_nationkey) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String queryDENSE_RANK = "select DENSE_RANK() over(order by n_nationkey) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
- final String queryPERCENT_RANK = "select PERCENT_RANK() over(order by n_nationkey) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String queryPERCENT_RANK = "select PERCENT_RANK() over(order by n_nationkey) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
- final String queryRANK = "select RANK() over(order by n_nationkey) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String queryRANK = "select RANK() over(order by n_nationkey) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
- final String queryROW_NUMBER = "select ROW_NUMBER() over(order by n_nationkey) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String queryROW_NUMBER = "select ROW_NUMBER() over(order by n_nationkey) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
final TypeProtos.MajorType majorTypeDouble = TypeProtos.MajorType.newBuilder()
@@ -632,8 +634,8 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testWindowNTILE() throws Exception {
- final String query = "select ntile(1) over(order by position_id) as col \n" +
- "from cp.`employee.json` \n" +
+ final String query = "select ntile(1) over(order by position_id) as col " +
+ "from cp.`employee.json` " +
"limit 0";
final TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
@@ -653,11 +655,11 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testLeadLag() throws Exception {
- final String queryLEAD = "select lead(cast(n_nationkey as BigInt)) over(order by n_nationkey) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String queryLEAD = "select lead(cast(n_nationkey as BigInt)) over(order by n_nationkey) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
- final String queryLAG = "select lag(cast(n_nationkey as BigInt)) over(order by n_nationkey) as col \n" +
- "from cp.`tpch/nation.parquet` \n" +
+ final String queryLAG = "select lag(cast(n_nationkey as BigInt)) over(order by n_nationkey) as col " +
+ "from cp.`tpch/nation.parquet` " +
"limit 0";
final TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
@@ -683,12 +685,12 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test
public void testFirst_Last_Value() throws Exception {
- final String queryFirst = "select first_value(cast(position_id as integer)) over(order by position_id) as col \n" +
- "from cp.`employee.json` \n" +
+ final String queryFirst = "select first_value(cast(position_id as integer)) over(order by position_id) as col " +
+ "from cp.`employee.json` " +
"limit 0";
- final String queryLast = "select first_value(cast(position_id as integer)) over(order by position_id) as col \n" +
- "from cp.`employee.json` \n" +
+ final String queryLast = "select first_value(cast(position_id as integer)) over(order by position_id) as col " +
+ "from cp.`employee.json` " +
"limit 0";
final TypeProtos.MajorType majorType = TypeProtos.MajorType.newBuilder()
@@ -714,8 +716,8 @@ public class TestFunctionsWithTypeExpoQueries extends BaseTestQuery {
@Test // DRILL-4529
public void testWindowSumConstant() throws Exception {
- final String query = "select sum(1) over w as col \n" +
- "from cp.`tpch/region.parquet` \n" +
+ final String query = "select sum(1) over w as col " +
+ "from cp.`tpch/region.parquet` " +
"window w as (partition by r_regionkey)";
final String[] expectedPlan = {"\\$SUM0"};
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestInList.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestInList.java b/exec/java-exec/src/test/java/org/apache/drill/TestInList.java
index 79dfa13..08f3d9a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestInList.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestInList.java
@@ -20,11 +20,12 @@ package org.apache.drill;
import static org.junit.Assert.assertEquals;
import org.apache.drill.categories.SqlTest;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(SqlTest.class)
-public class TestInList extends BaseTestQuery{
+public class TestInList extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestInList.class);
@Test
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java b/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java
index 690f793..89a9e9d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestJoinNullable.java
@@ -20,29 +20,25 @@ package org.apache.drill;
import static org.junit.Assert.assertEquals;
import org.apache.drill.categories.OperatorTest;
-import org.apache.drill.common.util.TestTools;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(OperatorTest.class)
-public class TestJoinNullable extends BaseTestQuery{
+public class TestJoinNullable extends BaseTestQuery {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJoinNullable.class);
- static final String WORKING_PATH = TestTools.getWorkingPath();
- static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
-
private static void enableJoin(boolean hj, boolean mj) throws Exception {
-
- test(String.format("alter session set `planner.enable_hashjoin` = %s", hj ? "true":"false"));
- test(String.format("alter session set `planner.enable_mergejoin` = %s", mj ? "true":"false"));
+ test("alter session set `planner.enable_hashjoin` = %s", hj);
+ test("alter session set `planner.enable_mergejoin` = %s", mj);
test("alter session set `planner.slice_target` = 1");
}
/** InnerJoin on nullable cols, HashJoin */
@Test
public void testHashInnerJoinOnNullableColumns() throws Exception {
- String query = String.format("select t1.a1, t1.b1, t2.a2, t2.b2 from dfs_test.`%s/jsoninput/nullable1.json` t1, " +
- " dfs_test.`%s/jsoninput/nullable2.json` t2 where t1.b1 = t2.b2", TEST_RES_PATH, TEST_RES_PATH);
+ String query = "select t1.a1, t1.b1, t2.a2, t2.b2 from cp.`jsoninput/nullable1.json` t1, " +
+ " cp.`jsoninput/nullable2.json` t2 where t1.b1 = t2.b2";
final int expectedRecordCount = 1;
enableJoin(true, false);
@@ -53,9 +49,9 @@ public class TestJoinNullable extends BaseTestQuery{
/** LeftOuterJoin on nullable cols, HashJoin */
@Test
public void testHashLOJOnNullableColumns() throws Exception {
- String query = String.format("select t1.a1, t1.b1, t2.a2, t2.b2 from dfs_test.`%s/jsoninput/nullable1.json` t1 " +
- " left outer join dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " on t1.b1 = t2.b2", TEST_RES_PATH, TEST_RES_PATH);
+ String query = "select t1.a1, t1.b1, t2.a2, t2.b2 from cp.`jsoninput/nullable1.json` t1 " +
+ " left outer join cp.`jsoninput/nullable2.json` t2 " +
+ " on t1.b1 = t2.b2";
final int expectedRecordCount = 2;
@@ -67,9 +63,9 @@ public class TestJoinNullable extends BaseTestQuery{
/** RightOuterJoin on nullable cols, HashJoin */
@Test
public void testHashROJOnNullableColumns() throws Exception {
- String query = String.format("select t1.a1, t1.b1, t2.a2, t2.b2 from dfs_test.`%s/jsoninput/nullable1.json` t1 " +
- " right outer join dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " on t1.b1 = t2.b2", TEST_RES_PATH, TEST_RES_PATH);
+ String query = "select t1.a1, t1.b1, t2.a2, t2.b2 from cp.`jsoninput/nullable1.json` t1 " +
+ " right outer join cp.`jsoninput/nullable2.json` t2 " +
+ " on t1.b1 = t2.b2";
final int expectedRecordCount = 4;
@@ -81,9 +77,9 @@ public class TestJoinNullable extends BaseTestQuery{
/** FullOuterJoin on nullable cols, HashJoin */
@Test
public void testHashFOJOnNullableColumns() throws Exception {
- String query = String.format("select t1.a1, t1.b1, t2.a2, t2.b2 from dfs_test.`%s/jsoninput/nullable1.json` t1 " +
- " full outer join dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " on t1.b1 = t2.b2", TEST_RES_PATH, TEST_RES_PATH);
+ String query = "select t1.a1, t1.b1, t2.a2, t2.b2 from cp.`jsoninput/nullable1.json` t1 " +
+ " full outer join cp.`jsoninput/nullable2.json` t2 " +
+ " on t1.b1 = t2.b2";
final int expectedRecordCount = +5;
@@ -96,11 +92,10 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeInnerJoinOnNullableColumns() throws Exception {
String query =
- String.format(
"select t1.a1, t1.b1, t2.a2, t2.b2 "
- + " from dfs_test.`%s/jsoninput/nullable1.json` t1, "
- + " dfs_test.`%s/jsoninput/nullable2.json` t2 "
- + " where t1.b1 = t2.b2", TEST_RES_PATH, TEST_RES_PATH);
+ + " from cp.`jsoninput/nullable1.json` t1, "
+ + " cp.`jsoninput/nullable2.json` t2 "
+ + " where t1.b1 = t2.b2";
final int expectedRecordCount = 1;
enableJoin(false, true);
@@ -111,9 +106,9 @@ public class TestJoinNullable extends BaseTestQuery{
/** LeftOuterJoin on nullable cols, MergeJoin */
@Test
public void testMergeLOJNullable() throws Exception {
- String query = String.format("select t1.a1, t1.b1, t2.a2, t2.b2 from dfs_test.`%s/jsoninput/nullable1.json` t1 " +
- " left outer join dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " on t1.b1 = t2.b2", TEST_RES_PATH, TEST_RES_PATH);
+ String query = "select t1.a1, t1.b1, t2.a2, t2.b2 from cp.`jsoninput/nullable1.json` t1 " +
+ " left outer join cp.`jsoninput/nullable2.json` t2 " +
+ " on t1.b1 = t2.b2";
final int expectedRecordCount = 2;
@@ -125,9 +120,9 @@ public class TestJoinNullable extends BaseTestQuery{
/** RightOuterJoin on nullable cols, MergeJoin */
@Test
public void testMergeROJOnNullableColumns() throws Exception {
- String query = String.format("select t1.a1, t1.b1, t2.a2, t2.b2 from dfs_test.`%s/jsoninput/nullable1.json` t1 " +
- " right outer join dfs_test.`%s/jsoninput/nullable2.json` t2 " +
- " on t1.b1 = t2.b2", TEST_RES_PATH, TEST_RES_PATH);
+ String query = "select t1.a1, t1.b1, t2.a2, t2.b2 from cp.`jsoninput/nullable1.json` t1 " +
+ " right outer join cp.`jsoninput/nullable2.json` t2 " +
+ " on t1.b1 = t2.b2";
final int expectedRecordCount = 4;
@@ -141,12 +136,10 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableNoOrderedInputs() throws Exception {
String query =
- String.format(
"SELECT * "
- + "FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` t1 "
- + " left outer join dfs_test.`%s/jsoninput/nullableOrdered2.json` t2 "
- + " using ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + "FROM cp.`jsoninput/nullableOrdered1.json` t1 "
+ + " left outer join cp.`jsoninput/nullableOrdered2.json` t2 "
+ + " using ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -158,15 +151,13 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableOneOrderedInputAscNullsFirst() throws Exception {
String query =
- String.format(
"SELECT * "
- + "from dfs_test.`%s/jsoninput/nullableOrdered1.json` t1 "
+ + "from cp.`jsoninput/nullableOrdered1.json` t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` t2 "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` t2 "
+ " ORDER BY 1 ASC NULLS FIRST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -178,15 +169,13 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableOneOrderedInputAscNullsLast() throws Exception {
String query =
- String.format(
"SELECT * "
- + "FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` t1 "
+ + "FROM cp.`jsoninput/nullableOrdered1.json` t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` t2 "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` t2 "
+ " ORDER BY 1 ASC NULLS LAST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -198,15 +187,13 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableOneOrderedInputDescNullsFirst() throws Exception {
String query =
- String.format(
"SELECT * "
- + "FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` t1 "
+ + "FROM cp.`jsoninput/nullableOrdered1.json` t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` t2 "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` t2 "
+ " ORDER BY 1 DESC NULLS FIRST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -218,15 +205,13 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableOneOrderedInputDescNullsLast() throws Exception {
String query =
- String.format(
"SELECT * "
- + "FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` t1 "
+ + "FROM cp.`jsoninput/nullableOrdered1.json` t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` t2 "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` t2 "
+ " ORDER BY 1 DESC NULLS LAST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -240,17 +225,15 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableBothInputsOrderedAscNullsFirstVsAscNullsFirst() throws Exception {
String query =
- String.format(
"SELECT * "
+ "from ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` "
+ + " FROM cp.`jsoninput/nullableOrdered1.json` "
+ " ORDER BY 1 ASC NULLS FIRST ) t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` "
+ " ORDER BY 1 ASC NULLS FIRST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -262,17 +245,15 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableBothInputsOrderedAscNullsLastVsAscNullsFirst() throws Exception {
String query =
- String.format(
"SELECT * "
+ "from ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` "
+ + " FROM cp.`jsoninput/nullableOrdered1.json` "
+ " ORDER BY 1 ASC NULLS LAST ) t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` "
+ " ORDER BY 1 ASC NULLS FIRST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -284,17 +265,15 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableBothInputsOrderedAscNullsFirstVsAscNullsLast() throws Exception {
String query =
- String.format(
"SELECT * "
+ "from ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` "
+ + " FROM cp.`jsoninput/nullableOrdered1.json` "
+ " ORDER BY 1 ASC NULLS FIRST ) t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` "
+ " ORDER BY 1 ASC NULLS LAST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -306,17 +285,15 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableBothInputsOrderedAscNullsLastVsAscNullsLast() throws Exception {
String query =
- String.format(
"SELECT * "
+ "from ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` "
+ + " FROM cp.`jsoninput/nullableOrdered1.json` "
+ " ORDER BY 1 ASC NULLS LAST ) t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` "
+ " ORDER BY 1 ASC NULLS LAST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -329,17 +306,15 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableBothInputsOrderedDescNullsFirstVsAscNullsFirst() throws Exception {
String query =
- String.format(
"SELECT * "
+ "from ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` "
+ + " FROM cp.`jsoninput/nullableOrdered1.json` "
+ " ORDER BY 1 DESC NULLS FIRST ) t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` "
+ " ORDER BY 1 ASC NULLS FIRST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -352,17 +327,15 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableBothInputsOrderedDescNullsLastVsAscNullsFirst() throws Exception {
String query =
- String.format(
"SELECT * "
+ "from ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` "
+ + " FROM cp.`jsoninput/nullableOrdered1.json` "
+ " ORDER BY 1 DESC NULLS LAST ) t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` "
+ " ORDER BY 1 ASC NULLS FIRST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -375,17 +348,15 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableBothInputsOrderedDescNullsFirstVsAscNullsLast() throws Exception {
String query =
- String.format(
"SELECT * "
+ "from ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` "
+ + " FROM cp.`jsoninput/nullableOrdered1.json` "
+ " ORDER BY 1 DESC NULLS FIRST ) t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` "
+ " ORDER BY 1 ASC NULLS LAST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
@@ -398,17 +369,15 @@ public class TestJoinNullable extends BaseTestQuery{
@Test
public void testMergeLOJNullableBothInputsOrderedDescNullsLastVsAscNullsLast() throws Exception {
String query =
- String.format(
"SELECT * "
+ "from ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered1.json` "
+ + " FROM cp.`jsoninput/nullableOrdered1.json` "
+ " ORDER BY 1 DESC NULLS LAST ) t1 "
+ " LEFT OUTER JOIN "
+ " ( SELECT key, data "
- + " FROM dfs_test.`%s/jsoninput/nullableOrdered2.json` "
+ + " FROM cp.`jsoninput/nullableOrdered2.json` "
+ " ORDER BY 1 ASC NULLS LAST ) t2 "
- + " USING ( key )",
- TEST_RES_PATH, TEST_RES_PATH);
+ + " USING ( key )";
final int expectedRecordCount = 6;
enableJoin(false, true);
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java b/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
index ab0acd2..4ac8580 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
@@ -27,46 +27,42 @@ public class TestMergeFilterPlan extends PlanTestBase {
@Test
public void testDRILL_FilterMerge() throws Exception {
- String viewDDL = "create or replace view MyViewWithFilter as " +
- " SELECT first_name, " +
- " last_name, " +
- " full_name, " +
- " salary, " +
- " employee_id, " +
- " store_id, " +
- " position_id, " +
- " position_title, " +
- " education_level " +
- " FROM cp.`employee.json` " +
- " WHERE position_id in (1, 2, 3 ) " ;
-
- String querySQL = " select dat.store_id\n" +
- " , sum(dat.store_cost) as total_cost\n" +
- " from (\n" +
- " select store_id, position_id\n" +
- " , sum( salary) as store_cost\n" +
- " from MyViewWithFilter \n" +
- " where full_name in ( select n_name\n" +
- " from cp.`tpch/nation.parquet`)\n" +
- " and education_level = 'GRADUATE DEGREE' " +
- " and position_id in ( select position_id \n" +
- " from MyViewWithFilter\n" +
- " where position_title like '%VP%'\n" +
- " )\n" +
- " group by store_id, position_id\n" +
- ") dat\n" +
- "group by dat.store_id\n" +
- "order by dat.store_id";
-
String expectedPattern1 = "Filter(condition=[AND(OR(=($0, 1), =($0, 2), =($0, 3)), =($4, 'GRADUATE DEGREE'))])";
String expectedPattern2 = "Filter(condition=[AND(OR(=($0, 1), =($0, 2), =($0, 3)), LIKE($1, '%VP%'))])";
String excludedPattern = "Filter(condition=[OR(=($0, 1), =($0, 2), =($0, 3))])";
- test("use dfs_test.tmp");
+ test("use dfs.tmp");
- test(viewDDL);
+ test("create or replace view MyViewWithFilter as " +
+ " SELECT first_name, " +
+ " last_name, " +
+ " full_name, " +
+ " salary, " +
+ " employee_id, " +
+ " store_id, " +
+ " position_id, " +
+ " position_title, " +
+ " education_level " +
+ " FROM cp.`employee.json` " +
+ " WHERE position_id in (1, 2, 3 )");
- testPlanSubstrPatterns(querySQL,
+ testPlanSubstrPatterns(" select dat.store_id " +
+ " , sum(dat.store_cost) as total_cost " +
+ " from ( " +
+ " select store_id, position_id " +
+ " , sum(salary) as store_cost " +
+ " from MyViewWithFilter " +
+ " where full_name in ( select n_name " +
+ " from cp.`tpch/nation.parquet`) " +
+ " and education_level = 'GRADUATE DEGREE' " +
+ " and position_id in ( select position_id " +
+ " from MyViewWithFilter " +
+ " where position_title like '%VP%' " +
+ " ) " +
+ " group by store_id, position_id " +
+ ") dat " +
+ "group by dat.store_id " +
+ "order by dat.store_id",
new String[]{expectedPattern1, expectedPattern2}, new String[]{excludedPattern});
test("drop view MyViewWithFilter ");
http://git-wip-us.apache.org/repos/asf/drill/blob/acc5ed92/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java b/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
index 9d78024..347c4ec 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestOutOfMemoryOutcome.java
@@ -22,6 +22,7 @@ import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import org.apache.drill.exec.testing.ControlsInjectionUtil;
+import org.apache.drill.test.BaseTestQuery;
import org.junit.Assert;
import org.junit.Test;
@@ -29,7 +30,7 @@ import org.junit.Test;
* Run several tpch queries and inject an OutOfMemoryException in ScanBatch that will cause an OUT_OF_MEMORY outcome to
* be propagated downstream. Make sure the proper "memory error" message is sent to the client.
*/
-public class TestOutOfMemoryOutcome extends BaseTestQuery{
+public class TestOutOfMemoryOutcome extends BaseTestQuery {
private static final String SINGLE_MODE = "ALTER SESSION SET `planner.disable_exchanges` = true";