You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2018/05/11 22:07:40 UTC
[drill] 01/04: DRILL-6249: Adding more unit testing documentation.
This is an automated email from the ASF dual-hosted git repository.
amansinha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit dc8d0107cb6a4a31e1beaa9c98d6e51cf7486a67
Author: Timothy Farkas <ti...@apache.org>
AuthorDate: Thu Mar 15 14:27:55 2018 -0700
DRILL-6249: Adding more unit testing documentation.
close apache/drill#1251
---
README.md | 5 +-
.../java/org/apache/drill/test/DirTestWatcher.java | 14 +-
.../test/java/org/apache/drill/test/TestTools.java | 7 +-
docs/dev/BaseTestQuery.md | 4 +
docs/dev/ClusterTest.md | 4 +
docs/dev/DevDocs.md | 6 +-
docs/dev/GeneratedCode.md | 51 +++++++
docs/dev/InstantiatingComponents.md | 107 ++++++++++++++
docs/dev/LicenseHeaders.md | 34 +++++
docs/dev/PhysicalOpUnitTestBase.md | 31 ++++
docs/dev/RowSetFramework.md | 112 +++++++++++++++
docs/dev/TempDirectories.md | 17 +++
docs/dev/TestDataSets.md | 159 +++++++++++++++++++++
docs/dev/Testing.md | 134 +++++++++++++++--
.../drill/exec/ops/ExchangeFragmentContext.java | 3 +
.../drill/exec/ops/ExecutorFragmentContext.java | 4 +
.../org/apache/drill/exec/ops/FragmentContext.java | 10 +-
.../apache/drill/exec/ops/FragmentContextImpl.java | 28 +++-
.../apache/drill/exec/ops/RootFragmentContext.java | 3 +
.../org/apache/drill/exec/record/BatchSchema.java | 4 +
.../org/apache/drill/TestDynamicUDFSupport.java | 2 +-
.../exec/expr/fn/FunctionInitializerTest.java | 2 +-
.../physical/impl/window/GenerateTestData.java | 4 +-
.../store/dfs/TestSchemaNotFoundException.java | 8 +-
.../org/apache/drill/test/BaseDirTestWatcher.java | 16 ++-
.../java/org/apache/drill/test/BaseTestQuery.java | 4 +
.../java/org/apache/drill/test/ExampleTest.java | 24 ++++
.../apache/drill/test/rowSet/RowSetBuilder.java | 7 +
.../apache/drill/test/rowSet/RowSetUtilities.java | 6 +
.../drill/test/rowSet/file/JsonFileBuilder.java | 26 ++++
30 files changed, 794 insertions(+), 42 deletions(-)
diff --git a/README.md b/README.md
index a30cf07..f2b5b9d 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,10 @@ Please see the [Apache Drill Website](http://drill.apache.org/) or the [Apache D
## Join the community!
-Apache Drill is an Apache Foundation project and is seeking all types of contributions. Please say hello on the Apache Drill mailing list or join our Google Hangouts for more information. (More information can be found at the Apache Drill website).
+Apache Drill is an Apache Foundation project and is seeking all types of contributions.
+Please say hello on the [Apache Drill mailing list](http://drill.apache.org/mailinglists/)
+or join our [Google Hangouts](http://drill.apache.org/community-resources/) for more information.
+(More information can be found at the [Apache Drill website](http://drill.apache.org/)).
## Export Control
This distribution includes cryptographic software. The country in which you currently reside may have restrictions on the import, possession, use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, please check your country's laws, regulations and policies concerning the import, possession, or use, and re-export of encryption software, to see if this is permitted. See <http://www.wassenaar.org/> for more information.
diff --git a/common/src/test/java/org/apache/drill/test/DirTestWatcher.java b/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
index 271ac94..6bdc951 100644
--- a/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
+++ b/common/src/test/java/org/apache/drill/test/DirTestWatcher.java
@@ -31,10 +31,10 @@ import java.nio.file.Paths;
* </p>
*
* <p>
- * A {@link DirTestWatcher} is added to a test by declaring it as a JUnit {@link org.junit.Rule}. A {@link org.junit.Rule} is
- * a piece of code that is run before and after every JUnit test marked with the {@link org.junit.Test} annotation. When the
+ * A {@link DirTestWatcher} is added to a test by declaring it as a JUnit {@link org.junit.Rule}. A {@link org.junit.Rule Rule} is
+ * a piece of code that is run before and after every JUnit test marked with the {@link org.junit.Test Test} annotation. When the
* {@link DirTestWatcher} is added to a test class the {@link DirTestWatcher} will create a temp directory before each of your
- * {@link org.junit.Test}s and optionally delete the temp directory after each of your {@link org.junit.Test}s. The temp directory
+ * {@link org.junit.Test Test}s and optionally delete the temp directory after each of your {@link org.junit.Test Test}s. The <b>base temp directory</b>
* created by the {@link DirTestWatcher} is in the <b>target</b> folder of the maven project and has the form
* <b>(my test class fully qualified name)/(my test method name)</b>. So in the context of the code example below, the temp directory created for
* each test in <b>target</b> will be <b>my.proj.MyTestClass/myTestMethod1</b> and <b>my.proj.MyTestClass/myTestMethod2</b> respectively.
@@ -46,7 +46,7 @@ import java.nio.file.Paths;
* </p>
*
* <p>
- * By default, the {@link DirTestWatcher} deletes the temp directory it creates at the end of each {@link org.junit.Test}. However, you can create a {@link DirTestWatcher}
+ * By default, the {@link DirTestWatcher} deletes the temp directory it creates at the end of each {@link org.junit.Test Test}. However, you can create a {@link DirTestWatcher}
* by doing {@code new DirTestWatcher(false)} to disable the deletion of temp directories after a test. This is useful if you want to examine files after a test runs.
* </p>
*
@@ -54,16 +54,16 @@ import java.nio.file.Paths;
* package my.proj;
*
* public class MyTestClass {
- * @org.junit.Rule
+ * @Rule
* public final DirTestWatcher dirTestWatcher = new DirTestWatcher();
*
- * @org.junit.Test
+ * @Test
* public void myTestMethod1() {
* File dir = dirTestWatcher.getDir();
* // Do stuff in the temp directory
* }
*
- * @org.junit.Test
+ * @Test
* public void myTestMethod2() {
* File dir = dirTestWatcher.getDir();
* // Do stuff in the temp directory
diff --git a/common/src/test/java/org/apache/drill/test/TestTools.java b/common/src/test/java/org/apache/drill/test/TestTools.java
index 661b978..8cf7ca7 100644
--- a/common/src/test/java/org/apache/drill/test/TestTools.java
+++ b/common/src/test/java/org/apache/drill/test/TestTools.java
@@ -34,10 +34,13 @@ public class TestTools {
PROJECT
}
- public static final Path TEST_RESOURCES = Paths.get("src", "test", "resources");
+ public static final Path TEST_RESOURCES_REL = Paths.get("src", "test", "resources");
public static final Path PROJECT_ROOT = Paths.get("..", "..");
public static final Path WORKING_PATH = new File(".").toPath();
+ public static final Path TEST_RESOURCES_ABS = WORKING_PATH.resolve(TEST_RESOURCES_REL);
+ public static final Path SAMPLE_DATA = PROJECT_ROOT.resolve("sample-data");
+
static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean()
.getInputArguments()
.toString()
@@ -56,7 +59,7 @@ public class TestTools {
public static File getResourceFile(Path relPath) {
return WORKING_PATH
- .resolve(TEST_RESOURCES)
+ .resolve(TEST_RESOURCES_REL)
.resolve(relPath)
.toFile();
}
diff --git a/docs/dev/BaseTestQuery.md b/docs/dev/BaseTestQuery.md
new file mode 100644
index 0000000..97b046d
--- /dev/null
+++ b/docs/dev/BaseTestQuery.md
@@ -0,0 +1,4 @@
+# BaseTestQuery Deprecated. Use [ClusterTest](ClusterTest.md) Instead.
+
+The [BaseTestQuery](../../exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java) class is useful if you want start a local Drill cluster and run several
+integration tests against the Drill cluster.
\ No newline at end of file
diff --git a/docs/dev/ClusterTest.md b/docs/dev/ClusterTest.md
new file mode 100644
index 0000000..d58ed3a
--- /dev/null
+++ b/docs/dev/ClusterTest.md
@@ -0,0 +1,4 @@
+# ClusterTest
+
+The [ClusterTest](../../exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java) class is useful if you want start a local Drill cluster and run several integration
+tests against the Drill cluster. Please read the [ClusterTest](../../exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java) java doc for more details.
\ No newline at end of file
diff --git a/docs/dev/DevDocs.md b/docs/dev/DevDocs.md
index 2b5b31f..3fb7615 100644
--- a/docs/dev/DevDocs.md
+++ b/docs/dev/DevDocs.md
@@ -6,4 +6,8 @@ For information about configuring your development enviornment see [Environment.
## Testing
-For information about how to do Integration and Unit Testing in Drill see [Testing.md](Testing.md).
\ No newline at end of file
+For information about how to do integration and unit testing in Drill see [Testing.md](Testing.md).
+
+## License Headers
+
+For more information about working with license headers see [LicenseHeaders.md](LicenseHeaders.md)
diff --git a/docs/dev/GeneratedCode.md b/docs/dev/GeneratedCode.md
new file mode 100644
index 0000000..aaa11b6
--- /dev/null
+++ b/docs/dev/GeneratedCode.md
@@ -0,0 +1,51 @@
+# Testing Generated Code
+
+## Writing Unit Tests For Generated Code
+
+An example of unit testing generated code without running all of Drill is the **priorityQueueOrderingTest()** test in
+[TopNBatchTest](../../exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TopN/TopNBatchTest.java). That test tests the
+[PriorityQueueTemplate](../../exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java) class separately from the rest of Drill.
+
+The testing of [PriorityQueueTemplate](../../exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/PriorityQueueTemplate.java) is mainly accomplished by creating
+instances of the following classes:
+
+ * [FunctionLookupContext](../../exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionLookupContext.java)
+ * [CodeCompiler](../../exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java)
+
+## Creating A [FunctionLookupContext](../../exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionLookupContext.java)
+
+```
+new FunctionImplementationRegistry(drillConfig)
+```
+
+## Creating A [CodeCompiler](../../exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java)
+
+ 1. Create an [OperatorFixture](OperatorFixture.md).
+ 1. Retrieve the [SystemOptionManager](../../exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java) from the
+ [OperatorFixture](OperatorFixture.md).
+ ```
+ operatorFixture.getOptionManager();
+ ```
+ 1. Create an instance of [CodeCompiler](../../exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java).
+ ```
+ new CodeCompiler(drillConfig, optionManager)
+ ```
+
+## Debugging Generated Code
+
+It is possible to set break points in generated code.
+
+### Instructions For IntelliJ
+
+ 1. File→Project structure…→Modules→distribution→Sources → Add content root
+ 1. Chose /tmp/drill/codegen
+ 1. Mark it as Sources directory.
+ 1. Set saveCodeForDebugging(true) for the code generator of interest
+ 1. Run the unit test of interest
+ 1. Now some generated classes should appear in Intellij under the distribution module
+ 1. Set a break point in a generated class and run the unit test in debug mode
+
+### Instructions For Eclipse
+
+ 1. To step into the generated code, set a breakpoint just before we call into the setup method.
+ 1. Step into that method which will step into doSetup. This opens the generated code file.
diff --git a/docs/dev/InstantiatingComponents.md b/docs/dev/InstantiatingComponents.md
new file mode 100644
index 0000000..80ba6b1
--- /dev/null
+++ b/docs/dev/InstantiatingComponents.md
@@ -0,0 +1,107 @@
+# Instantiating Components
+
+There are a few techniques for creating instances classes in unit tests:
+
+* Use a mocking library. **(Depracated)**
+* Provide a simple impementation of an interface
+* Build a real instance of class using the class's builders / constructors
+* Use the [ClusterFixture](ClusterFixture.md) or [OperatorFixture](OperatorFixture.md) classes to create instances of objects.
+
+## Mocking Libraries (Deprecated)
+
+Drill uses two mocking libraries in order to mock classes.
+
+* [Mockito](http://site.mockito.org)
+* [JMockit](http://jmockit.github.io/tutorial.html)
+
+These libraries were originally used to work around the lack of well defined interfaces and adequate testing tools. Drill has made significant improvements in these areas
+so using mocking libraries are no longer required. Existing tests that use these libraries will be refactored to remove them, and new tests should NOT use these libraries.
+
+## Instantiating Contexts
+
+There are several contexts used throughout Drill, for a complete description of each and how
+they are used please see [FragmentContextImpl](../../exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java).
+
+When doing tests you can use the following mock contexts:
+
+ * [MockFragmentContext](../../exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java) is a simple mock implementation of
+ the [FragmentContext](../../exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java). A mock instance of the class can also be retrieved from an
+ [OperatorFixture](OperatorFixture.md).
+ ```
+ FragmentContext context = operatorFixture.getFragmentContext();
+ ```
+
+## Creating An Instance of [QueryId](../../protocol/src/main/java/org/apache/drill/exec/proto/beans/QueryId.java)
+
+```
+UserBitShared.QueryId queryId = UserBitShared.QueryId.newBuilder()
+ .setPart1(1L)
+ .setPart2(2L)
+ .build();
+```
+
+## Creating [FragmentHandle](../../protocol/src/main/java/org/apache/drill/exec/proto/beans/FragmentHandle.java)
+
+```
+ExecProtos.FragmentHandle fragmentHandle = ExecProtos.FragmentHandle.newBuilder()
+ .setQueryId(queryId)
+ .setMinorFragmentId(1)
+ .setMajorFragmentId(2)
+ .build();
+```
+
+## Creating A [DrillConfig](../../common/src/main/java/org/apache/drill/common/config/DrillConfig.java)
+
+There are a few ways to create a [DrillConfig](../../common/src/main/java/org/apache/drill/common/config/DrillConfig.java). The simplest way is to
+ create a [ClusterFixture](ClusterFixture.md) or [OperatorFixture](OperatorFixture.md) and then do the following:
+
+```
+DrillConfig drillConfig = clusterFixture.config();
+```
+
+or
+
+```
+DrillConfig drillConfig = operatorFixture.config();
+```
+
+If you need a [DrillConfig](../../common/src/main/java/org/apache/drill/common/config/DrillConfig.java) and don't want all the extra things provided
+by [ClusterFixture](ClusterFixture.md) and [OperatorFixture](OperatorFixture.md), you can use
+[ConfigBuilder](../../exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java).
+
+## Creating A [SpillSet](../../exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/spill/SpillSet.java)
+
+ 1. Create a [PhysicalOperator](../../exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java).
+ ```
+ HashJoinPOP pop = new HashJoinPOP(null, null, null, JoinRelType.FULL);
+ ```
+ 1. Create a [DrillConfig](../../common/src/main/java/org/apache/drill/common/config/DrillConfig.java).
+ 1. Create a [FragmentHandle](../../protocol/src/main/java/org/apache/drill/exec/proto/beans/FragmentHandle.java) as described above.
+ 1. Create a [SpillSet](../../exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/spill/SpillSet.java).
+ ```
+ SpillSet spillSet = new SpillSet(config, fragmentHandle, pop);
+ ```
+
+## Creating A [PersistentStoreProvider](../../exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreProvider.java)
+
+```
+LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(drillConfig);
+provider.start();
+```
+
+## Creating A [LogicalPlanPersistence](../../logical/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java)
+
+```
+LogicalPlanPersistence logicalPlanPersistence = PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(drillConfig);
+```
+
+## Creating An Instance Of An Option Manager
+
+You can create an instance of the [SystemOptionManager](../../exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java) by leveraging the
+[OperatorFixture](OperatorFixture.md).
+
+ 1. Create an [OperatorFixture](OperatorFixture.md).
+ 1. Retrieve the [SystemOptionManager](../../exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java).
+ ```
+ operatorFixture.getOptionManager();
+ ```
diff --git a/docs/dev/LicenseHeaders.md b/docs/dev/LicenseHeaders.md
new file mode 100644
index 0000000..21634ed
--- /dev/null
+++ b/docs/dev/LicenseHeaders.md
@@ -0,0 +1,34 @@
+# License Headers
+
+Drill uses two license header checkers:
+
+* [Apache RAT Plugin](http://creadur.apache.org/rat/apache-rat-plugin/)
+* [License Maven Plugin](http://code.mycila.com/license-maven-plugin/)
+
+## Why Two?
+
+[Apache RAT Plugin](http://creadur.apache.org/rat/apache-rat-plugin/) is used because it is the standard license header
+checker for Apache projects.
+
+[License Maven Plugin](http://code.mycila.com/license-maven-plugin/) performs stricter license checks and supports disallowing license headers wrapped in `/**` and `**/`. This
+allows us to inforce requiring all license headers to be wrapped only in `/*` and `*/`.
+
+## Doing License Checks
+
+The license checks are disabled locally by default and are enabled on Travis. If you'd like to perform
+license checks locally you can do the following:
+
+```
+ mvn license:check -Dlicense.skip=false
+```
+
+## Auto Formatting Headers
+
+If the license checks fail and you can't figure out what's wrong with your headers, you can auto-format
+your license headers with the following command:
+
+```
+mvn license:format -Dlicense.skip=false
+```
+
+This command will also add license headers to files without them.
diff --git a/docs/dev/PhysicalOpUnitTestBase.md b/docs/dev/PhysicalOpUnitTestBase.md
new file mode 100644
index 0000000..e9b8bcd
--- /dev/null
+++ b/docs/dev/PhysicalOpUnitTestBase.md
@@ -0,0 +1,31 @@
+# Single Operator Unit Test
+
+It is possible to run an end to end test of an operator in isolation by extending
+[PhysicalOpUnitTestBase](../../exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java).
+
+A simple example of an operator level unit test is the following:
+
+```
+public class BasicPhysicalOpUnitTest extends PhysicalOpUnitTestBase {
+
+ @Test
+ public void testSimpleProject() {
+ Project projectConf = new Project(parseExprs("x+5", "x"), null);
+ List<String> jsonBatches = Lists.newArrayList(
+ "[{\"x\": 5 },{\"x\": 10 }]",
+ "[{\"x\": 20 },{\"x\": 30 },{\"x\": 40 }]");
+ opTestBuilder()
+ .physicalOperator(projectConf)
+ .inputDataStreamJson(jsonBatches)
+ .baselineColumns("x")
+ .baselineValues(10l)
+ .baselineValues(15l)
+ .baselineValues(25l)
+ .baselineValues(35l)
+ .baselineValues(45l)
+ .go();
+ }
+}
+
+```
+
diff --git a/docs/dev/RowSetFramework.md b/docs/dev/RowSetFramework.md
new file mode 100644
index 0000000..6df42fa
--- /dev/null
+++ b/docs/dev/RowSetFramework.md
@@ -0,0 +1,112 @@
+# RowSet Framework
+
+The RowSet Framework allows you to create custom instances of:
+
+ * [BatchSchema](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java)
+ * [VectorContainer](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java)
+ * [TupleMetadata](../../exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleMetadata.java)
+ * [RecordBatch](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java): This is effectively a fake instance of an upstream operator.
+
+It also allows the comparison of data container in [VectorContainer](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java)s through the use
+of the [RowSetComparison](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetComparison.java) and
+[RowSetUtilities](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java).
+
+## Creating A [BatchSchema](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java)
+
+The [SchemaBuilder](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaBuilder.java) class can be used
+to create an instance [BatchSchema](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java). An example
+of how to to this can be found the **secondTest()** method of [ExampleTest](../../exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java).
+
+**Note:** The [BatchSchema](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java) class has limited complex type support. When
+possible use [TupleMetadata](../../exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleMetadata.java) and
+[TupleSchema](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/TupleSchema.java) instead.
+
+## Creating [TupleMetadata](../../exec/vector/src/main/java/org/apache/drill/exec/record/metadata/TupleMetadata.java)
+
+```
+TupleMetadata schema = new SchemaBuilder()
+ .add(...)
+ .add(...)
+ .buildSchema();
+```
+
+## Creating Test [VectorContainer](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java)
+
+[VectorContainer](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java)s populated with data can be created with the
+[RowSetBuilder](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java). In order to use it do the following:
+
+ 1. Create an allocator
+ ```
+ BufferAllocator allocator = operatorFixture.allocator();
+ ```
+ 1. Create the desired BatchSchema using the [SchemaBuilder](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaBuilder.java).
+ ```
+ TupleMetadata schema = new SchemaBuilder()
+ .add(...)
+ .add(...)
+ .buildSchema();
+ ```
+ 1. Create a [RowSetBuilder](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java) and add
+ records to it. Then build a [RowSet](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java).
+ ```
+ RowSet rowSet = new RowSetBuilder(allocator, schema)
+ .addRow(110, "green", new floatArray(5.5f, 2.3f), strArray("1a", "1b"))
+ .addRow(109, "blue", new floatArray(1.5f), strArray("2a"))
+ .addRow(108, "red", new floatArray(-11.1f, 0.0f, .5f), strArray("3a", "3b", "3c"))
+ .build();
+ ```
+ 1. Retrieve the [VectorContainer](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java) wrapped by the
+ [RowSet](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java).
+ ```
+ VectorContainer container = rowSet.container();
+ ```
+
+## Creating A Mock Record Batch (Upstream Operator) With Data
+
+Create a [RowSet](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java) and then create a
+[RowSetBatch](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBatch)
+
+```
+RecordBatch batch = new RowSetBatch(rowSet);
+```
+
+## Comparison Of Results
+
+### Compare Two [RowSet](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java)s
+
+Use [RowSetUtilities](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java).
+
+```
+RowSetUtilities.verify(expectedRowSet, actualRowSet)
+```
+
+### Compare A [VectorContainer](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java) To A [RowSet](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java)
+
+You can convert a [VectorContainer](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java) into a [RowSet](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java)
+a few ways:
+
+ * If you are using an [OperatorFixture](OperatorFixture.md) the best way to do this is with:
+ ```
+ operatorFixture.wrap(container);
+ ```
+ * When there is no selection vector you can do the following:
+ ```
+ RowSet rowSet = DirectRowSet.fromContainer(container);
+ ```
+ * When there is a [SelectionVector2](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java).
+ ```
+ RowSet rowSet = IndirectRowSet.fromSv2(container, container.getSelectionVector2());
+ ```
+ * When there is a [SelectionVector4](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java).
+ ```
+ RowSet rowSet = HyperRowSetImpl.fromContainer(container, container.getSelectionVector4());
+ ```
+
+After the [VectorContainer](../../exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java) is wrapped in a [RowSet](../.
+./exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java) you can compare the two
+[RowSet](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java)s as usual.
+
+## End To End Example
+
+A good example of building a [RowSet](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSet.java) and comparing results can be found in the
+**testInitialSchema()** test in [TestResultSetLoaderProtocol](../../exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java).
diff --git a/docs/dev/TempDirectories.md b/docs/dev/TempDirectories.md
new file mode 100644
index 0000000..e853695
--- /dev/null
+++ b/docs/dev/TempDirectories.md
@@ -0,0 +1,17 @@
+# Temp Directory Utilities
+
+The two basic temp directory classes are:
+
+ - [DirTestWatcher](../exec/java-exec/src/test/java/org/apache/drill/test/DirTestWatcher.java)
+ - [BaseDirTestWatcher](../exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java)
+
+These classes are used to create temp directories for each of your unit tests. The advantage to using
+these temp directory classes are:
+
+ - All files are deleted after a unit test completes. This prevents a build machine being polluted with a
+ bunch of unit test files.
+ - Each unit test outputs its files to a unique well defined location. This makes it easy to find files
+ for debugging. Also since each temp directory is unique, it prevents multiple unit test runs from interferring
+ with one another on a build machine.
+
+For examples on how to use these classes, please read the javadoc for each class.
diff --git a/docs/dev/TestDataSets.md b/docs/dev/TestDataSets.md
new file mode 100644
index 0000000..4e66d5e
--- /dev/null
+++ b/docs/dev/TestDataSets.md
@@ -0,0 +1,159 @@
+# Data Sets
+
+Drill includes several data sets for testing, and also provides some tools for generating test data sets.
+
+## Bundled Data Sets
+
+There are three primary data sets bundled with drill for testing:
+
+ - **Sample Data:** These are parquet files in the [sample-data](../../sample-data) folder.
+ - **Resource Data:** These are data files in the [exec/java-exec/src/test/resources](../../exec/java-exec/src/test/resources) folder.
+ - **TPCH Data:** These are trimmed down versions of the tpch data sets. They are retrieved and bundled
+ in the [contrib/data](../../contrib/data) maven submodule. They are also accessible on [Apache Drill's S3 bucket](http://apache-drill.s3.amazonaws.com/files/sf-0
+ .01_tpc-h_parquet.tgz).
+ When unit tests are running all of the files in these data set are available from the classpath storage plugin. The tpch
+ files include:
+ - **customer.parquet**
+ - **lineitem.parquet**
+ - **nation.parquet**
+ - **orders.parquet**
+ - **part.parquet**
+ - **partsup.parquet**
+ - **region.parquet**
+ - **supplier.parquet**
+
+### Using Sample Data in Unit Tests
+
+#### [ClusterFixture](ClusterFixture.md)
+
+See **seventhTest()** in [ExampleTest](../../exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java) for an example of how to do this.
+
+#### BaseTestQuery (Deprecated Use [ClusterFixture](ClusterFixture.md) or [ClusterTest](ClusterTest.md) Instead)
+
+When using the [BaseDirTestWatcher](../../exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java) you
+can make [sample-data](../../sample-data) accessible from the ```dfs``` storage plugin by doing the following:
+
+```
+public class TestMyClass {
+ @ClassRule
+ public static final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data", "region.parquet"));
+ }
+
+ @Test
+ public void simpleTest() {
+ // dfs.root.`sample-data/region.parquet` will be accessible from my test
+ }
+}
+```
+
+Or if you are extending [BaseTestQuery](../../exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java)
+
+```
+public class TestMyClass extends BaseTestQuery {
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyFileToRoot(Paths.get("sample-data", "region.parquet"));
+ }
+
+ @Test
+ public void simpleTest() {
+ // dfs.root.`sample-data/region.parquet` will be accessible from my test
+ }
+}
+```
+
+### Using Resource Data in Unit Tests
+
+#### [ClusterFixture](ClusterFixture.md)
+
+See **sixthTest()** in [ExampleTest](../../exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java) for an example of how to do this.
+
+#### BaseTestQuery (Deprecated Use [ClusterFixture](ClusterFixture.md) or [ClusterTest](ClusterTest.md) Instead)
+
+When using the [BaseDirTestWatcher](../../exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java) you
+can make data from [exec/java-exec/src/test/resources](../../exec/java-exec/src/test/resources) accessible from the ```dfs``` storage plugin by doing the following:
+
+```
+public class TestMyClass {
+ @ClassRule
+ public static final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("join", "empty_part"));
+ }
+
+ @Test
+ public void simpleTest() {
+ // src/test/resources/join/empty_part is acessible at dfs.root.`join/empty_part` from my test
+ }
+}
+```
+
+Or if you are extending [BaseTestQuery](../../exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java)
+
+```
+public class TestMyClass extends BaseTestQuery {
+ @BeforeClass
+ public static void setupFiles() {
+ dirTestWatcher.copyResourceToRoot(Paths.get("join", "empty_part"));
+ }
+
+ @Test
+ public void simpleTest() {
+ // src/test/resources/join/empty_part is acessible at dfs.root.`join/empty_part` from my test
+ }
+}
+```
+
+### Using TPCH Data in Unit Tests
+
+TPCH data is accessible via the classpath storage plugin
+
+```
+cp.`tpch/customer.parquet`
+```
+
+## Generating Data Sets
+
+There are a few ways to generate data for testing:
+
+ * [JsonFileBuilder](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java)
+ * Inline Mock Scanner
+ * [MockRecordReader](./MockRecordReader.md)
+
+### Json
+
+The [JsonFileBuilder](../../exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java)
+can be used to create json data files. It's useful for creating data files for both integration and unit tests.
+An example of using the JsonFileBuilder can be found in secondTest() in
+[ExampleTest](../../exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java).
+
+### Inline Mock Scanner
+
+The MockScanner is a special scanner that generates data for a query. It can only be used for integration testing.
+
+#### Using In A Query
+
+The MockScanner can be used directly from sql queries. An example can be found in **thirdTest()** in
+[ExampleTest](../../exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java).
+
+The example uses the following query
+
+```
+SELECT id_i, name_s10 FROM `mock`.`employees_5`
+```
+
+The select columns encode the name and type of the columns to generate. The table name also encodes
+the number of records to generate.
+
+ - **Columns:** Columns in the sql query have the form **(name)_(type)**. **(name)** is the name that is assigned to a column. **(type)** is the data type
+ of the column. Valid types are:
+ - Use **i** for specifying an integer column.
+ - Use **s(n)** for specifying a string column. Replace **(n)** with an integer which specifies the number of character in each varchar record.
+ - **Table:** Tables in the sql query have the form **(name)_(n)**. Where **(name)** is the name of the source table. **(n)** specifies the number of records to
+ generate for the source table.
diff --git a/docs/dev/Testing.md b/docs/dev/Testing.md
index c7c2ee2..09343d6 100644
--- a/docs/dev/Testing.md
+++ b/docs/dev/Testing.md
@@ -2,27 +2,141 @@
Drill makes extensive use of [JUnit](http://junit.org/junit4/) and other libraries for testing. This page provides pointers to the information you need to work with Drill tests. We don't repeat that information; you will want to follow the links and read the original material to get a complete understanding of the libraries that Drill uses.
-Caveat: information here about Drill is "reverse engineered" from the code; this page has not yet had the benefit of insight from the developers who created Drill's test structure.
+# Writing Tests
-# Topics
-
-"Classic" Drill testing techniques
+## General Testing Tools
+* [Test Data Sets](TestDataSets.md)
+* [Temp Directory Utilities](TempDirectories.md)
* [Testing with JUnit](JUnit.md)
* [Test Logging](TestLogging.md)
+
+## Deprecated Drill Testing Techniques
+
+This is a list of old Drill testing machinery that we have cleaner machinery for now.
+
+* [BaseTestQuery](BaseTestQuery.md): Deprecated, use [ClusterTest](ClusterTest.md) instead.
+
+## Legacy Drill Testing Techniques
+
+This is a list of old Drill testing machinery for which there is no other alternative at the moment.
+
* [Testing with Physical Plans and Mock Data](LegacyTestingFrameworks.md)
-"Updated" Drill testing techniques
+## Latest Drill Testing Techniques
+
+These are all the latest Drill testing Techniques that you should use. If you find one of these approaches insufficient for your use, please enhance them or
+file a Jira so that they an be improved.
+* [RowSet Framework](RowSetFramework.md)
* [Cluster Fixture Framework](ClusterFixture.md)
* [Operator Fixture Framework](OperatorFixture.md)
-* [The Mock Record Reader](MockRecordReader.md)
+* [ClusterTest](ClusterTest.md)
+* [Single Operator Unit Test](PhysicalOpUnitTestBase.md)
+* [Instantiating Components](InstantiatingComponents.md)
+* [Generated Code](GeneratedCode.md)
+* [MiniPlanUnitTestBase](../../exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java)
+
+## Testing Guidelines
+
+When testing an operator or a specific internal mechanism in Drill you should use unit tests and should avoid depending on integration tests. This
+approach means that you will have to design your implementation to be decoupled from the rest of Drill. While this seems daunting, in reality it doesn't take
+much time and you will spend much less time debugging and stressing about customer bug fixes.
+
+Drill provides several tools to facilitate unit testing. See:
+
+ * [RowSet Framework](RowSetFramework.md)
+ * [Operator Fixture Framework](OperatorFixture.md)
+ * [Single Operator Unit Test](PhysicalOpUnitTestBase.md)
+ * [Instantiating Components](InstantiatingComponents.md)
+ * [Generated Code](GeneratedCode.md)
+
+The following are some examples of operators which have leveraged these techniques for unit testing:
+
+ * [Lateral Join Operator](../../exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestLateralJoinCorrectness.java)
+ * [External Store Operator](../../exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestExternalSortInternals.java)
+
+After the unit tests are implemented and passing, use integration tests to ensure the whole system works. The standard tools for accomplishing integration
+testing are:
+
+ * [ClusterFixture](ClusterFixture.md)
+
+The general rule is: test as close to your code as possible, refactoring and breaking dependencies where needed to accomplish this.
+On the other hand, if the test is more of a SQL or planner-level concept, then testing at the query level might be fine.
+
+## Categories
+
+Currently Drill uses Travis to run smoke tests for every PR and commit. All of Drill's unit tests cannot be run on Travis because Drill's tests take longer to run than the
+maximum allowed container time for the free tier of Travis. In order to decide which tests are run on Travis and which tests are not, tests are categorized using JUnit's
+`@Category` annotation. Currently the following categories are excluded from Travis:
+
+ - **SlowTest:** Tests that are slow.
+ - **UnlikelyTest:** Tests that cover parts of the code that are rarely or never touched.
+ - **SecurityTest:** Corner case tests for security features.
+
+To mark a test with a category you can do the following:
+
+```
+@Category(SlowTest.class)
+public class MyTest {
+ // Testing code
+}
+```
+
+To mark a test with multiple categories you can do the following:
+
+```
+@Category({SlowTest.class, SecurityTest.class})
+public class MyTest {
+ // Testing code
+}
+```
+
+# Running Tests
+
+Drill tests run in parallel. The model for parallel execution is to divide test classes between multiple
+forked test processes. Each test process then runs the test classes assigned to it sequentially.
+
+## Speeding Up Test Runs
+
+There are a couple knobs you can turn to make tests run faster on your machine.
+
+ * **Maven Build Threads**: `-T <num>`
+ * **Sure Fire Fork Count**: `-DforkCount=<num>`
+ * **Test Categories**
+
+### -T
+
+Maven allows you to use multiple threads to compile sub modules. Also when running tests each build
+thread forks its own surefire process, so the tests for different submodules are run in parallel. In order
+to leverage this use the `-T` flag. By default this option is effectively `1`, so there is only one build thread by default.
+
+Ex. In order to run the build using two maven threads use the following command.
+
+```
+mvn -T 2 clean install
+```
+
+### -DforkCount
+
+To run tests within a submodule in parallel you can use the `-DforkCount` option. By default this `2`, so two surefire processes are forked for each build thread.
+
+Ex. Run 4 test processes in parallel
+
+```
+mvn clean install -DforkCount=4
+```
+
+**Note:** The `-DforkCount` option interacts with `-T`. When used together each build thread (`-T`) gets
+`-DforkCount` test processes.
-# Mockito
+### Running Categories
-Drill depends on the [Mockito](http://site.mockito.org) framework. (Need to find & document usages.)
+You can leverage categories to run subsets of the tests. This is useful if you need to test something like a
+storage plugin on a remote jenkins server. See the java docs [here](../../common/src/test/java/org/apache/drill/categories/package-info.java) for examples. For a list of all
+the availabe categories go [here](../../common/src/test/java/org/apache/drill/categories).
-# Maven
+## Maven
It is often helpful to know the [set of properties](https://cwiki.apache.org/confluence/display/MAVEN/Maven+Properties+Guide) that Maven defines and are available for use in the POM file.
@@ -69,4 +183,4 @@ class DrillBuf ...
# IntelliJ
-TBA
\ No newline at end of file
+TBA
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExchangeFragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExchangeFragmentContext.java
index 2fb0745..ab4d6ec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExchangeFragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExchangeFragmentContext.java
@@ -21,6 +21,9 @@ import org.apache.drill.exec.proto.CoordinationProtos;
import org.apache.drill.exec.rpc.control.Controller;
import org.apache.drill.exec.work.batch.IncomingBuffers;
+/**
+ * This provides the resources required by an exchange operator.
+ */
public interface ExchangeFragmentContext extends FragmentContext {
void waitForSendComplete();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
index 8031a15..dbf440c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
@@ -31,6 +31,10 @@ import java.util.Collection;
import java.util.Map;
import java.util.Set;
+/**
+ * This interface represents the context that is used by a Drillbit in classes like the
+ * {@link org.apache.drill.exec.work.fragment.FragmentExecutor}.
+ */
public interface ExecutorFragmentContext extends RootFragmentContext {
/**
* Returns the root allocator for the Drillbit.
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 6a0e103..593e3d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -38,16 +38,8 @@ import org.apache.drill.exec.testing.ExecutionControls;
import io.netty.buffer.DrillBuf;
/**
- * Fragment context interface: separates implementation from definition.
- * Allows unit testing by mocking or reimplementing services with
- * test-time versions. The name is awkward, chosen to avoid renaming
- * the implementation class which is used in many places in legacy code.
- * New code should use this interface, and the names should eventually
- * be swapped with {@link FragmentContextImpl} becoming
- * <tt>FragmentContextImpl</tt> and this interface becoming
- * {@link FragmentContextImpl}.
+ * Provides the resources required by a non-exchange operator to execute.
*/
-
public interface FragmentContext extends UdfUtilities, AutoCloseable {
/**
* Returns the UDF registry.
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
index 01c7045..b192850 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContextImpl.java
@@ -69,9 +69,31 @@ import com.google.common.collect.Maps;
import io.netty.buffer.DrillBuf;
/**
- * Contextual objects required for execution of a particular fragment.
- * This is the implementation; use <tt>FragmentContext</tt>
- * in code to allow tests to use test-time implementations.
+ * <p>
+ * This is the core Context which implements all the Context interfaces:
+ *
+ * <ul>
+ * <li>{@link FragmentContext}: A context provided to non-exchange operators.</li>
+ * <li>{@link ExchangeFragmentContext}: A context provided to exchange operators.</li>
+ * <li>{@link RootFragmentContext}: A context provided to fragment roots.</li>
+ * <li>{@link ExecutorFragmentContext}: A context used by the Drillbit.</li>
+ * </ul>
+ *
+ * The interfaces above expose resources to varying degrees. They are ordered from most restrictive ({@link FragmentContext})
+ * to least restrictive ({@link ExecutorFragmentContext}).
+ * </p>
+ * <p>
+ * Since {@link FragmentContextImpl} implements all of the interfaces listed above, the facade pattern is used in order
+ * to cast a {@link FragmentContextImpl} object to the desired interface where-ever it is needed. The facade pattern
+ * is powerful since it allows us to easily create minimal context objects to be used in unit tests. Without
+ * the use of interfaces and the facade pattern we would have to create a complete {@link FragmentContextImpl} object
+ * to unit test any part of the code that depends on a context.
+ * </p>
+ * <p>
+ * <b>General guideline:</b> Use the most narrow interface for the task. For example, "internal" operators don't need visibility to the networking functionality.
+ * Using the narrow interface allows unit testing without using mocking libraries. Often, the surrounding structure already has exposed the most narrow interface. If there are
+ * opportunities to clean up older code, we can do so as needed to make testing easier.
+ * </p>
*/
public class FragmentContextImpl extends BaseFragmentContext implements ExecutorFragmentContext {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentContextImpl.class);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/RootFragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/RootFragmentContext.java
index a8dab2c..9aa6edc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/RootFragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/RootFragmentContext.java
@@ -17,6 +17,9 @@
*/
package org.apache.drill.exec.ops;
+/**
+ * Provides services needed by the {@link org.apache.drill.exec.work.fragment.FragmentExecutor}.
+ */
public interface RootFragmentContext extends ExchangeFragmentContext {
FragmentStats getStats();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
index 2d01ef4..67598e0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
@@ -25,6 +25,10 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.drill.common.types.TypeProtos.MajorType;
+/**
+ * Historically {@link BatchSchema} is used to represent the schema of a batch. However, it does not handle complex types well. If you have a choice, use
+ * {@link org.apache.drill.exec.record.metadata.TupleMetadata} instead.
+ */
public class BatchSchema implements Iterable<MaterializedField> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BatchSchema.class);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
index e1692eb..41da123 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
@@ -81,7 +81,7 @@ import static org.mockito.Mockito.verify;
public class TestDynamicUDFSupport extends BaseTestQuery {
private static final Path jars = TestTools.WORKING_PATH
- .resolve(TestTools.TEST_RESOURCES)
+ .resolve(TestTools.TEST_RESOURCES_REL)
.resolve("jars");
private static final String default_binary_name = "DrillUDF-1.0.jar";
private static final String UDF_SUB_DIR = "udf";
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
index c25162c..7c10bd3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java
@@ -59,7 +59,7 @@ public class FunctionInitializerTest {
@BeforeClass
public static void init() throws Exception {
Path jars = TestTools.WORKING_PATH
- .resolve(TestTools.TEST_RESOURCES)
+ .resolve(TestTools.TEST_RESOURCES_REL)
.resolve("jars");
String binaryName = "DrillUDF-1.0.jar";
String sourceName = JarUtil.getSourceName(binaryName);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
index f7f891e..7f8d614 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/window/GenerateTestData.java
@@ -357,7 +357,7 @@ public class GenerateTestData {
private static void generateData(final String tableName, final Partition[] pby_data, final Partition[] nopby_data,
final boolean addLineNo) throws FileNotFoundException {
final Path path = TestTools.WORKING_PATH
- .resolve(TestTools.TEST_RESOURCES)
+ .resolve(TestTools.TEST_RESOURCES_REL)
.resolve(Paths.get("window", tableName));
final File pathFolder = path.toFile();
@@ -382,4 +382,4 @@ public class GenerateTestData {
generateData("b4.p4", dataB4P4(true), dataB4P4(false), true);
}
-}
\ No newline at end of file
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java
index 2fe44c2..dcd25bc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestSchemaNotFoundException.java
@@ -27,7 +27,7 @@ public class TestSchemaNotFoundException extends BaseTestQuery {
@Test(expected = Exception.class)
public void testSchemaNotFoundForWrongStoragePlgn() throws Exception {
- final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES));
+ final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES_REL));
final String query = String.format("select * from dfs1.`%s`", table);
try {
testNoResult(query);
@@ -41,7 +41,7 @@ public class TestSchemaNotFoundException extends BaseTestQuery {
@Test(expected = Exception.class)
public void testSchemaNotFoundForWrongWorkspace() throws Exception {
- final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES));
+ final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES_REL));
final String query = String.format("select * from dfs.tmp1.`%s`", table);
try {
testNoResult(query);
@@ -55,7 +55,7 @@ public class TestSchemaNotFoundException extends BaseTestQuery {
@Test(expected = Exception.class)
public void testSchemaNotFoundForWrongWorkspaceUsingDefaultWorkspace() throws Exception {
- final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES));
+ final String table = String.format("%s/empty", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES_REL));
final String query = String.format("select * from tmp1.`%s`", table);
try {
testNoResult("use dfs");
@@ -70,7 +70,7 @@ public class TestSchemaNotFoundException extends BaseTestQuery {
@Test(expected = Exception.class)
public void testTableNotFoundException() throws Exception {
- final String table = String.format("%s/missing.parquet", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES));
+ final String table = String.format("%s/missing.parquet", TestTools.WORKING_PATH.resolve(TestTools.TEST_RESOURCES_REL));
final String query = String.format("select * from tmp.`%s`", table);
try {
testNoResult("use dfs");
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java b/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
index 51a4028..21b4a64 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/BaseDirTestWatcher.java
@@ -27,17 +27,31 @@ import java.nio.file.Path;
import java.nio.file.Paths;
/**
+ * <h4>Overview</h4>
* <p>
* This is a {@link DirTestWatcher} which creates all the temporary directories required by a Drillbit and the various <b>dfs.*</b> storage workspaces. It also
* provides convenience methods that do the following:
- * </p>
*
* <ol>
* <li>Copy project files to temp directories. This is useful for copying the sample data into a temp directory.</li>
* <li>Copy resource files to temp.</li>
* <li>Updating parquet metadata files.</li>
* </ol>
+ * </p>
+ *
+ * <p>
+ * The {@link BaseDirTestWatcher} creates the following directories in the <b>base temp directory</b> (for a description of where the <b>base temp directory</b>
+ * is located please read the docs for {@link DirTestWatcher}):
+ *
+ * <ul>
+ * <li><b>tmp:</b> {@link #getTmpDir()}</li>
+ * <li><b>store:</b> {@link #getStoreDir()}</li>
+ * <li><b>root:</b> {@link #getRootDir()}</li>
+ * <li><b>dfsTestTmp:</b> {@link #getDfsTestTmpDir()}</li>
+ * </ul>
+ * </p>
*
+ * <h4>Examples</h4>
* <p>
* The {@link BaseDirTestWatcher} is used in {@link BaseTestQuery} and an example of how it is used in conjunction with the {@link ClusterFixture} can be found in
* {@link ExampleTest}.
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
index 5b8846f..1f258b5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java
@@ -72,6 +72,10 @@ import com.google.common.io.Resources;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.ValueVector;
+/**
+ * @deprecated Use {@link ClusterTest} instead.
+ */
+@Deprecated
public class BaseTestQuery extends ExecTest {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
index 348c0a8..9200368 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
@@ -260,6 +260,30 @@ public class ExampleTest {
}
/**
+ * This example shows how to define a workspace that points to test files in src/main/resources.
+ */
+ @Test
+ public void sixthTest() throws Exception {
+ try (ClusterFixture cluster = ClusterFixture.standardCluster(dirTestWatcher);
+ ClientFixture client = cluster.clientFixture()) {
+ cluster.defineWorkspace("dfs", "resources", TestTools.TEST_RESOURCES_ABS.toFile().getAbsolutePath(), "tsv");
+ client.queryBuilder().sql("SELECT * from dfs.resources.`testframework/small_test_data.tsv`").printCsv();
+ }
+ }
+
+ /**
+ * This example shows how to define a workspace that points to test files in the sample-data folder.
+ */
+ @Test
+ public void seventhTest() throws Exception {
+ try (ClusterFixture cluster = ClusterFixture.standardCluster(dirTestWatcher);
+ ClientFixture client = cluster.clientFixture()) {
+ cluster.defineWorkspace("dfs", "sampledata", TestTools.SAMPLE_DATA.toFile().getAbsolutePath(), "parquet");
+ client.queryBuilder().sql("SELECT * from dfs.sampledata.`nation.parquet`").printCsv();
+ }
+ }
+
+ /**
* Example of running a specific test as Java program. Handy if you want to
* run the test from the command line, or if your test runs so long that JUnit
* would kill it with a timeout.
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java
index dfeaad1..953a6fe 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java
@@ -45,6 +45,13 @@ public final class RowSetBuilder {
private boolean withSv2;
private Set<Integer> skipIndices = Sets.newHashSet();
+ /**
+ * Creates a {@link RowSetBuilder}. Since {@link BatchSchema} does not handle complex types well, this has been deprecated in favor of the other constructors.
+ * @deprecated Use {@link #RowSetBuilder(BufferAllocator, TupleMetadata)} instead.
+ * @param allocator An allocator.
+ * @param schema A {@link BatchSchema}.
+ */
+ @Deprecated
public RowSetBuilder(BufferAllocator allocator, BatchSchema schema) {
this(allocator, MetadataUtils.fromFields(schema), 10);
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java
index 31109fe..7c332a1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java
@@ -235,6 +235,12 @@ public class RowSetUtilities {
return new Object[] {element};
}
+ /**
+ * Convenience method to verify the actual results, then free memory
+ * for both the expected and actual result sets.
+ * @param expected The expected results.
+ * @param actual the actual results to verify.
+ */
public static void verify(RowSet expected, RowSet actual) {
new RowSetComparison(expected).verifyAndClearAll(actual);
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
index 0b720e8..c8cad4b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
@@ -35,6 +35,16 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+/**
+ * <h4>Overview</h4>
+ * <p>
+ * Builds a json file containing the data in a {@link RowSet}.
+ * </p>
+ * <h4>Example</h4>
+ * <p>
+ * You can find an example of how to use {@link JsonFileBuilder} at {@link org.apache.drill.test.ExampleTest#secondTest()}.
+ * </p>
+ */
public class JsonFileBuilder
{
public static final String DEFAULT_DOUBLE_FORMATTER = "%f";
@@ -57,11 +67,22 @@ public class JsonFileBuilder
private final RowSet rowSet;
private final Map<String, String> customFormatters = Maps.newHashMap();
+ /**
+ * Creates a {@link JsonFileBuilder} that will write the given {@link RowSet} to a file.
+ *
+ * @param rowSet The {@link RowSet} to be written to a file.
+ */
public JsonFileBuilder(RowSet rowSet) {
this.rowSet = Preconditions.checkNotNull(rowSet);
Preconditions.checkArgument(rowSet.rowCount() > 0, "The given rowset is empty.");
}
+ /**
+ * Sets a custom formatter for a column using {@link String#format(String, Object...)} notation.
+ * @param columnName The name of the column to change the formatter for.
+ * @param columnFormatter The {@link String#format(String, Object...)} to use when writing a column value to the json file.
+ * @return The {@link JsonFileBuilder}.
+ */
public JsonFileBuilder setCustomFormatter(final String columnName, final String columnFormatter) {
Preconditions.checkNotNull(columnName);
Preconditions.checkNotNull(columnFormatter);
@@ -86,6 +107,11 @@ public class JsonFileBuilder
return this;
}
+ /**
+ * Writes the configured data to the given file in json format.
+ * @param tableFile The file to write the json data to.
+ * @throws IOException
+ */
public void build(File tableFile) throws IOException {
tableFile.getParentFile().mkdirs();
--
To stop receiving notification emails like this one, please contact
amansinha@apache.org.