You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2017/11/13 14:11:23 UTC
[1/2] drill git commit: DRILL-5832: Change OperatorFixture to use
system option manager
Repository: drill
Updated Branches:
refs/heads/master 3036d3700 -> 42fc11e53
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
index f6b3a8d..2694f64 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSortSpillWithException.java
@@ -32,7 +32,7 @@ import org.apache.drill.exec.testing.Controls;
import org.apache.drill.exec.testing.ControlsInjectionUtil;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
-import org.apache.drill.test.FixtureBuilder;
+import org.apache.drill.test.ClusterFixtureBuilder;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -51,7 +51,7 @@ public class TestSortSpillWithException extends ClusterTest {
@BeforeClass
public static void setup() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.configProperty(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD, 1) // Unmanaged
.configProperty(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE, 1) // Unmanaged
.sessionOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 60 * 1024 * 1024) // Spill early
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
index f525b2a..4c90769 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
@@ -62,6 +62,7 @@ public class SortTestUtilities {
return makeSchema(MinorType.INT, true);
}
+ @SuppressWarnings("resource")
public static PriorityQueueCopierWrapper makeCopier(OperatorFixture fixture, String sortOrder, String nullOrder) {
FieldReference expr = FieldReference.getWithQuotedRef("key");
Ordering ordering = new Ordering(sortOrder, expr, nullOrder);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
index 1c9d49e..ba5dfce 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestShortArrays.java
@@ -83,7 +83,7 @@ public class TestShortArrays extends SubOperatorTest {
assertEquals(bHint.elementCount, bCol.estElementCountPerArray, 0.001);
// Create a new batch, and new vector, using the sizer and
- // intializer inferred from the previoius batch.
+ // initializer inferred from the previous batch.
SingleRowSet empty = fixture.rowSet(schema);
vi.allocateBatch(empty.container(), 100);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
index bde6764..1548d7d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java
@@ -34,8 +34,8 @@ import org.apache.commons.math3.util.Pair;
import org.apache.drill.BaseTestQuery;
import org.apache.drill.QueryTestUtil;
import org.apache.drill.SingleRowListener;
-import org.apache.drill.common.concurrent.ExtendedLatch;
import org.apache.drill.common.DrillAutoCloseables;
+import org.apache.drill.common.concurrent.ExtendedLatch;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.types.TypeProtos.MinorType;
@@ -58,6 +58,7 @@ import org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch;
import org.apache.drill.exec.planner.sql.DrillSqlWorker;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
+import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import org.apache.drill.exec.proto.UserBitShared.ExceptionWrapper;
import org.apache.drill.exec.proto.UserBitShared.QueryData;
@@ -74,15 +75,14 @@ import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.rpc.user.UserResultsListener;
import org.apache.drill.exec.store.pojo.PojoRecordReader;
-import org.apache.drill.exec.testing.ControlsInjectionUtil;
import org.apache.drill.exec.testing.Controls;
+import org.apache.drill.exec.testing.ControlsInjectionUtil;
import org.apache.drill.exec.util.Pointer;
import org.apache.drill.exec.work.foreman.Foreman;
import org.apache.drill.exec.work.foreman.ForemanException;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.apache.drill.exec.work.fragment.FragmentExecutor;
import org.apache.drill.test.DrillTest;
-import org.apache.drill.test.OperatorFixture;
import org.apache.drill.categories.SlowTest;
import org.junit.After;
import org.junit.AfterClass;
@@ -138,7 +138,6 @@ public class TestDrillbitResilience extends DrillTest {
* @param name name of the drillbit
*/
private static void stopDrillbit(final String name) {
- @SuppressWarnings("resource")
final Drillbit drillbit = drillbits.get(name);
if (drillbit == null) {
throw new IllegalStateException("No Drillbit named \"" + name + "\" found");
@@ -321,6 +320,19 @@ public class TestDrillbitResilience extends DrillTest {
ControlsInjectionUtil.setSessionOption(drillClient, option, value);
}
+ private static void resetSessionOption(final String option) {
+ try {
+ final List<QueryDataBatch> results = drillClient.runQuery(
+ UserBitShared.QueryType.SQL, String.format("ALTER session RESET `%s`",
+ option));
+ for (final QueryDataBatch data : results) {
+ data.release();
+ }
+ } catch (final RpcException e) {
+ fail("Could not reset option: " + e.toString());
+ }
+ }
+
/**
* Check that the injected exception is what we were expecting.
*
@@ -812,9 +824,8 @@ public class TestDrillbitResilience extends DrillTest {
final String query = "SELECT sales_city, COUNT(*) cnt FROM cp.`region.json` GROUP BY sales_city";
assertCancelledWithoutException(control, new ListenerThatCancelsQueryAfterFirstBatchOfData(), query);
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- setSessionOption(SLICE_TARGET, Long.toString(SLICE_TARGET_DEFAULT));
- setSessionOption(HASHAGG.getOptionName(), testOptionSet.getDefault(HASHAGG.getOptionName()).bool_val.toString());
+ resetSessionOption(SLICE_TARGET);
+ resetSessionOption(HASHAGG.getOptionName());
}
}
@@ -844,11 +855,9 @@ public class TestDrillbitResilience extends DrillTest {
final long after = countAllocatedMemory();
assertEquals(String.format("We are leaking %d bytes", after - before), before, after);
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- setSessionOption(SLICE_TARGET, Long.toString(SLICE_TARGET_DEFAULT));
- setSessionOption(HASHAGG.getOptionName(), testOptionSet.getDefault(HASHAGG.getOptionName()).bool_val.toString());
- setSessionOption(PARTITION_SENDER_SET_THREADS.getOptionName(),
- Long.toString(testOptionSet.getDefault(PARTITION_SENDER_SET_THREADS.getOptionName()).num_val));
+ resetSessionOption(SLICE_TARGET);
+ resetSessionOption(HASHAGG.getOptionName());
+ resetSessionOption(PARTITION_SENDER_SET_THREADS.getOptionName());
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java
index bf7578f..4f4390f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/StatusResourcesTest.java
@@ -23,7 +23,7 @@ import org.apache.drill.exec.server.options.OptionDefinition;
import org.apache.drill.exec.server.options.OptionValidator;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
-import org.apache.drill.test.FixtureBuilder;
+import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.test.RestClientFixture;
import org.junit.Assert;
import org.junit.Test;
@@ -36,7 +36,7 @@ public class StatusResourcesTest {
public void testRetrieveInternalOption() throws Exception {
OptionDefinition optionDefinition = createMockPropOptionDefinition();
- FixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder().
configProperty(ExecConstants.HTTP_ENABLE, true).
configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
configProperty(ExecConstants.HTTP_PORT_HUNT, true).
@@ -60,7 +60,7 @@ public class StatusResourcesTest {
@Test
public void testRetrievePublicOption() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder().
configProperty(ExecConstants.HTTP_ENABLE, true).
configProperty(ExecConstants.HTTP_PORT_HUNT, true).
configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, false).
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java
index 60e466d..0f7c213 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetInternalsTest.java
@@ -23,7 +23,7 @@ import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
-import org.apache.drill.test.FixtureBuilder;
+import org.apache.drill.test.ClusterFixtureBuilder;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -34,7 +34,7 @@ public class ParquetInternalsTest extends ClusterTest {
@BeforeClass
public static void setup( ) throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
// Set options, etc.
;
startCluster(builder);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index 8f56c45..cb5000f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -292,9 +292,7 @@ public class TestParquetFilterPushDown extends PlanTestBase {
testParquetFilterPD(query1, 9, 3, false);
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- test("alter session set `" + PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY + "` = " +
- testOptionSet.getDefault(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY).bool_val);
+ resetSessionOption(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY);
deleteTableIfExists(tableName);
}
}
@@ -317,9 +315,7 @@ public class TestParquetFilterPushDown extends PlanTestBase {
testParquetFilterPD(query1, 9, 3, false);
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- test("alter session set `" + PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY + "` = " +
- testOptionSet.getDefault(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY).num_val);
+ resetSessionOption(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY);
deleteTableIfExists(tableName);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
index 79a0f45..e7b150c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
@@ -34,7 +34,7 @@ import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvid
import org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
-import org.apache.drill.test.FixtureBuilder;
+import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.testutils.DirTestWatcher;
import org.apache.zookeeper.CreateMode;
import org.junit.Assert;
@@ -85,7 +85,7 @@ public class TestPStoreProviders extends TestWithZookeeper {
org.apache.commons.io.FileUtils.copyDirectory(localOptionsResources, optionsDir);
- FixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder().
configProperty(ExecConstants.HTTP_ENABLE, false).
configProperty(ExecConstants.SYS_STORE_PROVIDER_CLASS, LocalPersistentStoreProvider.class.getCanonicalName()).
configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, String.format("file://%s", dirTestWatcher.getDirPath())).
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java
index 2476da7..7a62ef3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestQueryMemoryAlloc.java
@@ -33,8 +33,8 @@ public class TestQueryMemoryAlloc extends DrillTest {
@Test
public void testDefaultOptions() throws Exception {
OperatorFixtureBuilder builder = OperatorFixture.builder();
- builder.options().set(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
- builder.options().set(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
+ builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
+ builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
try (OperatorFixture fixture = builder.build()) {
@@ -58,8 +58,8 @@ public class TestQueryMemoryAlloc extends DrillTest {
@Test
public void testCustomFloor() throws Exception {
OperatorFixtureBuilder builder = OperatorFixture.builder();
- builder.options().set(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
- builder.options().set(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3 * ONE_GB);
+ builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.05);
+ builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 3 * ONE_GB);
try (OperatorFixture fixture = builder.build()) {
@@ -83,8 +83,8 @@ public class TestQueryMemoryAlloc extends DrillTest {
@Test
public void testCustomPercent() throws Exception {
OperatorFixtureBuilder builder = OperatorFixture.builder();
- builder.options().set(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.10);
- builder.options().set(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
+ builder.systemOption(ExecConstants.PERCENT_MEMORY_PER_QUERY_KEY, 0.10);
+ builder.systemOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY, 2 * ONE_GB);
try (OperatorFixture fixture = builder.build()) {
@@ -117,9 +117,9 @@ public class TestQueryMemoryAlloc extends DrillTest {
@Test
public void testOpMemory() throws Exception {
OperatorFixtureBuilder builder = OperatorFixture.builder();
- builder.options().set(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.7);
- builder.options().set(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 10);
- builder.options().set(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY, 40 * ONE_MB);
+ builder.systemOption(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.7);
+ builder.systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 10);
+ builder.systemOption(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP_KEY, 40 * ONE_MB);
try (OperatorFixture fixture = builder.build()) {
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
index 0710818..58793a2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestExtendedTypes.java
@@ -61,13 +61,8 @@ public class TestExtendedTypes extends BaseTestQuery {
+ "/0_0_0.json"));
assertEquals(new String(originalData), new String(newData));
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- testNoResult(String.format("ALTER SESSION SET `%s` = '%s'",
- ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName(),
- testOptionSet.getDefault(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName()).getValue()));
- testNoResult(String.format("ALTER SESSION SET `%s` = %s",
- ExecConstants.JSON_EXTENDED_TYPES.getOptionName(),
- testOptionSet.getDefault(ExecConstants.JSON_EXTENDED_TYPES.getOptionName()).getValue()));
+ resetSessionOption(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName());
+ resetSessionOption(ExecConstants.JSON_EXTENDED_TYPES.getOptionName());
}
}
@@ -92,13 +87,8 @@ public class TestExtendedTypes extends BaseTestQuery {
String expected = "drill_timestamp_millies,bin,bin1\n2015-07-07T03:59:43.488,drill,drill\n";
Assert.assertEquals(expected, actual);
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- testNoResult(String.format("ALTER SESSION SET `%s` = '%s'",
- ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName(),
- testOptionSet.getDefault(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName()).getValue()));
- testNoResult(String.format("ALTER SESSION SET `%s` = %s",
- ExecConstants.JSON_EXTENDED_TYPES.getOptionName(),
- testOptionSet.getDefault(ExecConstants.JSON_EXTENDED_TYPES.getOptionName()).getValue()));
+ resetSessionOption(ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName());
+ resetSessionOption(ExecConstants.JSON_EXTENDED_TYPES.getOptionName());
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java
index 8ad0fbb..a80d4f6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClientFixture.java
@@ -132,14 +132,12 @@ public class ClientFixture implements AutoCloseable {
* @param key
*/
- public void resetSystem(String key) {
- String sql = "ALTER SYSTEM RESET `" + key + "`";
- runSqlSilently(sql);
+ public void resetSession(String key) {
+ runSqlSilently("ALTER SESSION RESET `" + key + "`");
}
- public void resetSession(String key) {
- String sql = "ALTER SESSION RESET `" + key + "`";
- runSqlSilently(sql);
+ public void resetSystem(String key) {
+ runSqlSilently("ALTER SYSTEM RESET `" + key + "`");
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
index 5d53b59..f36c193 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
@@ -147,7 +147,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
private List<File> tempDirs = new ArrayList<>();
- ClusterFixture(FixtureBuilder builder) {
+ ClusterFixture(ClusterFixtureBuilder builder) {
setClientProps(builder);
configureZk(builder);
@@ -166,9 +166,9 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
/**
* Set the client properties to be used by client fixture.
- * @param builder {@link FixtureBuilder#clientProps}
+ * @param builder {@link ClusterFixtureBuilder#clientProps}
*/
- private void setClientProps(FixtureBuilder builder) {
+ private void setClientProps(ClusterFixtureBuilder builder) {
clientProps = builder.clientProps;
}
@@ -176,7 +176,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
return clientProps;
}
- private void configureZk(FixtureBuilder builder) {
+ private void configureZk(ClusterFixtureBuilder builder) {
// Start ZK if requested.
@@ -213,7 +213,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
}
}
- private void createConfig(FixtureBuilder builder) throws Exception {
+ private void createConfig(ClusterFixtureBuilder builder) throws Exception {
// Create a config
// Because of the way DrillConfig works, we can set the ZK
@@ -235,7 +235,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
}
}
- private void startDrillbits(FixtureBuilder builder) throws Exception {
+ private void startDrillbits(ClusterFixtureBuilder builder) throws Exception {
// // Ensure that Drill uses the log directory determined here rather than
// // it's hard-coded defaults. WIP: seems to be needed some times but
// // not others.
@@ -319,12 +319,12 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
((StoragePluginRegistryImpl) pluginRegistry).definePlugin(MockStorageEngineConfig.NAME, config, plugin);
}
- private void applyOptions(FixtureBuilder builder) throws Exception {
+ private void applyOptions(ClusterFixtureBuilder builder) throws Exception {
// Apply system options
if (builder.systemOptions != null) {
- for (FixtureBuilder.RuntimeOption option : builder.systemOptions) {
+ for (ClusterFixtureBuilder.RuntimeOption option : builder.systemOptions) {
clientFixture().alterSystem(option.key, option.value);
}
}
@@ -332,7 +332,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
// Apply session options.
if (builder.sessionOptions != null) {
- for (FixtureBuilder.RuntimeOption option : builder.sessionOptions) {
+ for (ClusterFixtureBuilder.RuntimeOption option : builder.sessionOptions) {
clientFixture().alterSession(option.key, option.value);
}
}
@@ -584,8 +584,8 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
public static final String EXPLAIN_PLAN_TEXT = "text";
public static final String EXPLAIN_PLAN_JSON = "json";
- public static FixtureBuilder builder() {
- FixtureBuilder builder = new FixtureBuilder()
+ public static ClusterFixtureBuilder builder() {
+ ClusterFixtureBuilder builder = new ClusterFixtureBuilder()
.sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, MAX_WIDTH_PER_NODE);
Properties props = new Properties();
props.putAll(ClusterFixture.TEST_CONFIGURATIONS);
@@ -604,8 +604,8 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
* @return a fixture builder with no default properties set
*/
- public static FixtureBuilder bareBuilder() {
- return new FixtureBuilder();
+ public static ClusterFixtureBuilder bareBuilder() {
+ return new ClusterFixtureBuilder();
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
new file mode 100644
index 0000000..8295554
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ZookeeperHelper;
+import org.apache.drill.exec.server.options.OptionDefinition;
+
+/**
+ * Build a Drillbit and client with the options provided. The simplest
+ * builder starts an embedded Drillbit, with the "dfs_test" name space,
+ * a max width (parallelization) of 2.
+ */
+
+public class ClusterFixtureBuilder {
+
+ public static class RuntimeOption {
+ public String key;
+ public Object value;
+
+ public RuntimeOption(String key, Object value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Values in the drill-module.conf file for values that are customized
+ // in the defaults.
+
+ public static final int DEFAULT_ZK_REFRESH = 500; // ms
+ public static final int DEFAULT_SERVER_RPC_THREADS = 10;
+ public static final int DEFAULT_SCAN_THREADS = 8;
+ public static final String OPTION_DEFAULTS_ROOT = "drill.exec.options.";
+
+ protected ConfigBuilder configBuilder = new ConfigBuilder();
+ protected List<RuntimeOption> sessionOptions;
+ protected List<RuntimeOption> systemOptions;
+ protected int bitCount = 1;
+ protected String bitNames[];
+ protected int localZkCount;
+ protected ZookeeperHelper zkHelper;
+ protected boolean usingZk;
+ protected File tempDir;
+ protected boolean preserveLocalFiles;
+ protected Properties clientProps;
+
+ /**
+ * The configuration builder which this fixture builder uses.
+ * @return the configuration builder for use in setting "advanced"
+ * configuration options.
+ */
+
+ public ConfigBuilder configBuilder() { return configBuilder; }
+
+ /**
+ * Use the given configuration file, stored as a resource, to start the
+ * embedded Drillbit. Note that the resource file should have the two
+ * following settings to work as a test:
+ * <pre><code>
+ * drill.exec.sys.store.provider.local.write : false,
+ * drill.exec.http.enabled : false
+ * </code></pre>
+ * It may be more convenient to add your settings to the default
+ * config settings with {@link #configProperty(String, Object)}.
+ * @param configResource path to the file that contains the
+ * config file to be read
+ * @return this builder
+ * @see {@link #configProperty(String, Object)}
+ */
+
+ public ClusterFixtureBuilder configResource(String configResource) {
+
+ // TypeSafe gets unhappy about a leading slash, but other functions
+ // require it. Silently discard the leading slash if given to
+ // preserve the test writer's sanity.
+
+ configBuilder.resource(ClusterFixture.trimSlash(configResource));
+ return this;
+ }
+
+ /**
+ *
+ */
+ public ClusterFixtureBuilder setOptionDefault(String key, Object value) {
+ String option_name = OPTION_DEFAULTS_ROOT + key;
+ configBuilder().put(option_name, value.toString());
+ return this;
+ }
+ /**
+ * Add an additional boot-time property for the embedded Drillbit.
+ * @param key config property name
+ * @param value property value
+ * @return this builder
+ */
+
+ public ClusterFixtureBuilder configProperty(String key, Object value) {
+ configBuilder.put(key, value.toString());
+ return this;
+ }
+
+ public ClusterFixtureBuilder putDefinition(OptionDefinition definition) {
+ configBuilder.putDefinition(definition);
+ return this;
+ }
+
+ /**
+ * Add an additional property for the client connection URL. Convert all the values into
+ * String type.
+ * @param key config property name
+ * @param value property value
+ * @return this builder
+ */
+ public ClusterFixtureBuilder configClientProperty(String key, Object value) {
+ if (clientProps == null) {
+ clientProps = new Properties();
+ }
+ clientProps.put(key, value.toString());
+ return this;
+ }
+
+ /**
+ * Provide a session option to be set once the Drillbit
+ * is started.
+ *
+ * @param key the name of the session option
+ * @param value the value of the session option
+ * @return this builder
+ * @see {@link ClusterFixture#alterSession(String, Object)}
+ */
+
+ public ClusterFixtureBuilder sessionOption(String key, Object value) {
+ if (sessionOptions == null) {
+ sessionOptions = new ArrayList<>();
+ }
+ sessionOptions.add(new RuntimeOption(key, value));
+ return this;
+ }
+
+ /**
+ * Provide a system option to be set once the Drillbit
+ * is started.
+ *
+ * @param key the name of the system option
+ * @param value the value of the system option
+ * @return this builder
+ * @see {@link ClusterFixture#alterSystem(String, Object)}
+ */
+
+ public ClusterFixtureBuilder systemOption(String key, Object value) {
+ if (systemOptions == null) {
+ systemOptions = new ArrayList<>();
+ }
+ systemOptions.add(new RuntimeOption(key, value));
+ return this;
+ }
+
+ /**
+ * Set the maximum parallelization (max width per node). Defaults
+ * to 2.
+ *
+ * @param n the "max width per node" parallelization option.
+ * @return this builder
+ */
+ public ClusterFixtureBuilder maxParallelization(int n) {
+ return sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, n);
+ }
+
+ /**
+ * The number of Drillbits to start in the cluster.
+ *
+ * @param n the desired cluster size
+ * @return this builder
+ */
+ public ClusterFixtureBuilder clusterSize(int n) {
+ bitCount = n;
+ bitNames = null;
+ return this;
+ }
+
+ /**
+ * Define a cluster by providing names to the Drillbits.
+ * The cluster size is the same as the number of names provided.
+ *
+ * @param bitNames array of (unique) Drillbit names
+ * @return this builder
+ */
+ public ClusterFixtureBuilder withBits(String bitNames[]) {
+ this.bitNames = bitNames;
+ bitCount = bitNames.length;
+ return this;
+ }
+
+ /**
+ * By default the embedded Drillbits use an in-memory cluster coordinator.
+ * Use this option to start an in-memory ZK instance to coordinate the
+ * Drillbits.
+ * @return this builder
+ */
+ public ClusterFixtureBuilder withLocalZk() {
+ return withLocalZk(1);
+ }
+
+ public ClusterFixtureBuilder withLocalZk(int count) {
+ localZkCount = count;
+ usingZk = true;
+
+ // Using ZK. Turn refresh wait back on.
+
+ return configProperty(ExecConstants.ZK_REFRESH, DEFAULT_ZK_REFRESH);
+ }
+
+ public ClusterFixtureBuilder withRemoteZk(String connStr) {
+ usingZk = true;
+ return configProperty(ExecConstants.ZK_CONNECTION, connStr);
+ }
+
+ /**
+ * Run the cluster using a Zookeeper started externally. Use this if
+ * multiple tests start a cluster: allows ZK to be started once for
+ * the entire suite rather than once per test case.
+ *
+ * @param zk the global Zookeeper to use
+ * @return this builder
+ */
+ public ClusterFixtureBuilder withZk(ZookeeperHelper zk) {
+ zkHelper = zk;
+ usingZk = true;
+
+ // Using ZK. Turn refresh wait back on.
+
+ configProperty(ExecConstants.ZK_REFRESH, DEFAULT_ZK_REFRESH);
+ return this;
+ }
+
+ public ClusterFixtureBuilder tempDir(File path) {
+ this.tempDir = path;
+ return this;
+ }
+
+ /**
+ * Starting with the addition of the CTTAS feature, a Drillbit will
+ * not restart unless we delete all local storage files before
+ * starting the Drillbit again. In particular, the stored copies
+ * of the storage plugin configs cause the temporary workspace
+ * check to fail. Normally the cluster fixture cleans up files
+ * both before starting and after shutting down the cluster. Set this
+ * option to preserve files after shutdown, perhaps to debug the
+ * contents.
+ * <p>
+ * This clean-up is needed only if we enable local storage writes
+ * (which we must do, unfortunately, to capture and analyze
+ * storage profiles.)
+ *
+ * @return this builder
+ */
+
+ public ClusterFixtureBuilder keepLocalFiles() {
+ preserveLocalFiles = true;
+ return this;
+ }
+
+ /**
+ * Enable saving of query profiles. The only way to save them is
+ * to enable local store provider writes, which also saves the
+ * storage plugin configs. Doing so causes the CTTAS feature to
+ * fail on the next run, so the test fixture deletes all local
+ * files on start and close, unless
+ * {@link #keepLocalFiles()} is set.
+ *
+ * @return this builder
+ */
+
+ public ClusterFixtureBuilder saveProfiles() {
+ configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true);
+ systemOption(ExecConstants.ENABLE_QUERY_PROFILE_OPTION, true);
+ systemOption(ExecConstants.QUERY_PROFILE_DEBUG_OPTION, true);
+ return this;
+ }
+
+ /**
+ * Create the embedded Drillbit and client, applying the options set
+ * in the builder. Best to use this in a try-with-resources block:
+ * <pre><code>
+ * FixtureBuilder builder = ClientFixture.newBuilder()
+ * .property(...)
+ * .sessionOption(...)
+ * ;
+ * try (ClusterFixture cluster = builder.build();
+ * ClientFixture client = cluster.clientFixture()) {
+ * // Do the test
+ * }
+ * </code></pre>
+ * Note that you use a single cluster fixture to create any number of
+ * drillbits in your cluster. If you want multiple clients, create the
+ * first as above, the others (or even the first) using the
+ * {@link ClusterFixture#clientBuilder()}. Using the client builder
+ * also lets you set client-side options in the rare cases that you
+ * need them.
+ *
+ * @return
+ */
+
+ public ClusterFixture build() {
+ return new ClusterFixture(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java
index e204fde..4a3823c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterTest.java
@@ -77,7 +77,7 @@ public class ClusterTest extends DrillTest {
protected static ClusterFixture cluster;
protected static ClientFixture client;
- protected static void startCluster(FixtureBuilder builder) throws Exception {
+ protected static void startCluster(ClusterFixtureBuilder builder) throws Exception {
cluster = builder.build();
client = cluster.clientFixture();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
index a770d3e..67ae4a3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java
@@ -88,7 +88,7 @@ public class ExampleTest {
@Test
public void secondTest() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.configProperty(ExecConstants.SLICE_TARGET, 10)
;
@@ -159,7 +159,7 @@ public class ExampleTest {
// And trace messages for one class.
.logger(ExternalSortBatch.class, Level.TRACE)
;
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
// Easy way to run single threaded for easy debugging
.maxParallelization(1)
// Set some session options
@@ -203,7 +203,7 @@ public class ExampleTest {
@Test
public void fifthTest() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.maxParallelization(1)
.configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true)
;
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
deleted file mode 100644
index e73c2ba..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.ZookeeperHelper;
-import org.apache.drill.exec.server.options.OptionDefinition;
-
-/**
- * Build a Drillbit and client with the options provided. The simplest
- * builder starts an embedded Drillbit, with the "dfs_test" name space,
- * a max width (parallelization) of 2.
- */
-
-public class FixtureBuilder {
-
- public static class RuntimeOption {
- public String key;
- public Object value;
-
- public RuntimeOption(String key, Object value) {
- this.key = key;
- this.value = value;
- }
- }
-
- // Values in the drill-module.conf file for values that are customized
- // in the defaults.
-
- public static final int DEFAULT_ZK_REFRESH = 500; // ms
- public static final int DEFAULT_SERVER_RPC_THREADS = 10;
- public static final int DEFAULT_SCAN_THREADS = 8;
- public static final String OPTION_DEFAULTS_ROOT = "drill.exec.options.";
-
- protected ConfigBuilder configBuilder = new ConfigBuilder();
- protected List<RuntimeOption> sessionOptions;
- protected List<RuntimeOption> systemOptions;
- protected int bitCount = 1;
- protected String bitNames[];
- protected int localZkCount;
- protected ZookeeperHelper zkHelper;
- protected boolean usingZk;
- protected File tempDir;
- protected boolean preserveLocalFiles;
- protected Properties clientProps;
-
- /**
- * The configuration builder which this fixture builder uses.
- * @return the configuration builder for use in setting "advanced"
- * configuration options.
- */
-
- public ConfigBuilder configBuilder() { return configBuilder; }
-
- /**
- * Use the given configuration file, stored as a resource, to start the
- * embedded Drillbit. Note that the resource file should have the two
- * following settings to work as a test:
- * <pre><code>
- * drill.exec.sys.store.provider.local.write : false,
- * drill.exec.http.enabled : false
- * </code></pre>
- * It may be more convenient to add your settings to the default
- * config settings with {@link #configProperty(String, Object)}.
- * @param configResource path to the file that contains the
- * config file to be read
- * @return this builder
- * @see {@link #configProperty(String, Object)}
- */
-
- public FixtureBuilder configResource(String configResource) {
-
- // TypeSafe gets unhappy about a leading slash, but other functions
- // require it. Silently discard the leading slash if given to
- // preserve the test writer's sanity.
-
- configBuilder.resource(ClusterFixture.trimSlash(configResource));
- return this;
- }
-
- /**
- *
- */
- public FixtureBuilder setOptionDefault(String key, Object value) {
- String option_name = OPTION_DEFAULTS_ROOT + key;
- configBuilder().put(option_name, value.toString());
- return this;
- }
- /**
- * Add an additional boot-time property for the embedded Drillbit.
- * @param key config property name
- * @param value property value
- * @return this builder
- */
-
- public FixtureBuilder configProperty(String key, Object value) {
- configBuilder.put(key, value.toString());
- return this;
- }
-
- public FixtureBuilder putDefinition(OptionDefinition definition) {
- configBuilder.putDefinition(definition);
- return this;
- }
-
- /**
- * Add an additional property for the client connection URL. Convert all the values into
- * String type.
- * @param key config property name
- * @param value property value
- * @return this builder
- */
- public FixtureBuilder configClientProperty(String key, Object value) {
- if (clientProps == null) {
- clientProps = new Properties();
- }
- clientProps.put(key, value.toString());
- return this;
- }
-
- /**
- * Provide a session option to be set once the Drillbit
- * is started.
- *
- * @param key the name of the session option
- * @param value the value of the session option
- * @return this builder
- * @see {@link ClusterFixture#alterSession(String, Object)}
- */
-
- public FixtureBuilder sessionOption(String key, Object value) {
- if (sessionOptions == null) {
- sessionOptions = new ArrayList<>();
- }
- sessionOptions.add(new RuntimeOption(key, value));
- return this;
- }
-
- /**
- * Provide a system option to be set once the Drillbit
- * is started.
- *
- * @param key the name of the system option
- * @param value the value of the system option
- * @return this builder
- * @see {@link ClusterFixture#alterSystem(String, Object)}
- */
-
- public FixtureBuilder systemOption(String key, Object value) {
- if (systemOptions == null) {
- systemOptions = new ArrayList<>();
- }
- systemOptions.add(new RuntimeOption(key, value));
- return this;
- }
-
- /**
- * Set the maximum parallelization (max width per node). Defaults
- * to 2.
- *
- * @param n the "max width per node" parallelization option.
- * @return this builder
- */
- public FixtureBuilder maxParallelization(int n) {
- return sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, n);
- }
-
- /**
- * The number of Drillbits to start in the cluster.
- *
- * @param n the desired cluster size
- * @return this builder
- */
- public FixtureBuilder clusterSize(int n) {
- bitCount = n;
- bitNames = null;
- return this;
- }
-
- /**
- * Define a cluster by providing names to the Drillbits.
- * The cluster size is the same as the number of names provided.
- *
- * @param bitNames array of (unique) Drillbit names
- * @return this builder
- */
- public FixtureBuilder withBits(String bitNames[]) {
- this.bitNames = bitNames;
- bitCount = bitNames.length;
- return this;
- }
-
- /**
- * By default the embedded Drillbits use an in-memory cluster coordinator.
- * Use this option to start an in-memory ZK instance to coordinate the
- * Drillbits.
- * @return this builder
- */
- public FixtureBuilder withLocalZk() {
- return withLocalZk(1);
- }
-
- public FixtureBuilder withLocalZk(int count) {
- localZkCount = count;
- usingZk = true;
-
- // Using ZK. Turn refresh wait back on.
-
- return configProperty(ExecConstants.ZK_REFRESH, DEFAULT_ZK_REFRESH);
- }
-
- public FixtureBuilder withRemoteZk(String connStr) {
- usingZk = true;
- return configProperty(ExecConstants.ZK_CONNECTION, connStr);
- }
-
- /**
- * Run the cluster using a Zookeeper started externally. Use this if
- * multiple tests start a cluster: allows ZK to be started once for
- * the entire suite rather than once per test case.
- *
- * @param zk the global Zookeeper to use
- * @return this builder
- */
- public FixtureBuilder withZk(ZookeeperHelper zk) {
- zkHelper = zk;
- usingZk = true;
-
- // Using ZK. Turn refresh wait back on.
-
- configProperty(ExecConstants.ZK_REFRESH, DEFAULT_ZK_REFRESH);
- return this;
- }
-
- public FixtureBuilder tempDir(File path) {
- this.tempDir = path;
- return this;
- }
-
- /**
- * Starting with the addition of the CTTAS feature, a Drillbit will
- * not restart unless we delete all local storage files before
- * starting the Drillbit again. In particular, the stored copies
- * of the storage plugin configs cause the temporary workspace
- * check to fail. Normally the cluster fixture cleans up files
- * both before starting and after shutting down the cluster. Set this
- * option to preserve files after shutdown, perhaps to debug the
- * contents.
- * <p>
- * This clean-up is needed only if we enable local storage writes
- * (which we must do, unfortunately, to capture and analyze
- * storage profiles.)
- *
- * @return this builder
- */
-
- public FixtureBuilder keepLocalFiles() {
- preserveLocalFiles = true;
- return this;
- }
-
- /**
- * Enable saving of query profiles. The only way to save them is
- * to enable local store provider writes, which also saves the
- * storage plugin configs. Doing so causes the CTTAS feature to
- * fail on the next run, so the test fixture deletes all local
- * files on start and close, unless
- * {@link #keepLocalFiles()} is set.
- *
- * @return this builder
- */
-
- public FixtureBuilder saveProfiles() {
- configProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true);
- systemOption(ExecConstants.ENABLE_QUERY_PROFILE_OPTION, true);
- systemOption(ExecConstants.QUERY_PROFILE_DEBUG_OPTION, true);
- return this;
- }
-
- /**
- * Create the embedded Drillbit and client, applying the options set
- * in the builder. Best to use this in a try-with-resources block:
- * <pre><code>
- * FixtureBuilder builder = ClientFixture.newBuilder()
- * .property(...)
- * .sessionOption(...)
- * ;
- * try (ClusterFixture cluster = builder.build();
- * ClientFixture client = cluster.clientFixture()) {
- * // Do the test
- * }
- * </code></pre>
- * Note that you use a single cluster fixture to create any number of
- * drillbits in your cluster. If you want multiple clients, create the
- * first as above, the others (or even the first) using the
- * {@link ClusterFixture#clientBuilder()}. Using the client builder
- * also lets you set client-side options in the rare cases that you
- * need them.
- *
- * @return
- */
-
- public ClusterFixture build() {
- return new ClusterFixture(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
index 39b0dc7..09abf12 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
@@ -18,12 +18,12 @@
package org.apache.drill.test;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.map.CaseInsensitiveMap;
import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.common.scanner.persistence.ScanResult;
import org.apache.drill.exec.ExecConstants;
@@ -43,14 +43,10 @@ import org.apache.drill.exec.ops.OperatorStatReceiver;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.VectorContainer;
-import org.apache.drill.exec.server.options.BaseOptionSet;
-import org.apache.drill.exec.server.options.OptionDefinition;
import org.apache.drill.exec.server.options.OptionSet;
-import org.apache.drill.exec.server.options.OptionValue;
-import org.apache.drill.exec.server.options.OptionValue.AccessibleScopes;
-import org.apache.drill.exec.server.options.OptionValue.OptionScope;
import org.apache.drill.exec.server.options.SystemOptionManager;
import org.apache.drill.exec.testing.ExecutionControls;
+import org.apache.drill.test.ClusterFixtureBuilder.RuntimeOption;
import org.apache.drill.test.rowSet.DirectRowSet;
import org.apache.drill.test.rowSet.HyperRowSetImpl;
import org.apache.drill.test.rowSet.IndirectRowSet;
@@ -92,84 +88,29 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable {
public static class OperatorFixtureBuilder
{
- ConfigBuilder configBuilder = new ConfigBuilder();
- TestOptionSet options = new TestOptionSet();
+ protected ConfigBuilder configBuilder = new ConfigBuilder();
+ protected List<RuntimeOption> systemOptions;
+ protected ExecutionControls controls;
public ConfigBuilder configBuilder() {
return configBuilder;
}
- public TestOptionSet options() {
- return options;
- }
-
- public OperatorFixture build() {
- return new OperatorFixture(this);
- }
- }
-
- /**
- * Test-time implementation of the system and session options. Provides
- * a simple storage and a simple set interface, then implements the standard
- * system and session option read interface.
- */
-
- public static class TestOptionSet extends BaseOptionSet {
-
- private Map<String,OptionValue> values = CaseInsensitiveMap.newHashMap();
- private Map<String,OptionValue> defaults = CaseInsensitiveMap.newHashMap();
-
- public TestOptionSet() {
- DrillConfig defaultConfig = DrillConfig.create();
- Map<String, OptionDefinition> definitions = SystemOptionManager.createDefaultOptionDefinitions();
- CaseInsensitiveMap<OptionValue> defaultValues = SystemOptionManager.populateDefaultValues(definitions, defaultConfig);
-
- for (Map.Entry<String, OptionValue> entry: defaultValues.entrySet()) {
- String optionName = entry.getKey();
- OptionValue optionValue = entry.getValue();
- defaults.put(optionName, optionValue);
+ public OperatorFixtureBuilder systemOption(String key, Object value) {
+ if (systemOptions == null) {
+ systemOptions = new ArrayList<>();
}
-
- // Crashes in FunctionImplementationRegistry if not set
- set(ExecConstants.CAST_TO_NULLABLE_NUMERIC, false);
- // Crashes in the Dynamic UDF code if not disabled
- set(ExecConstants.USE_DYNAMIC_UDFS_KEY, false);
- }
-
- public void set(String key, int value) {
- set(key, (long) value);
- }
-
- public void set(String key, long value) {
- values.put(key, OptionValue.create(AccessibleScopes.SYSTEM, key, value, OptionScope.SYSTEM));
+ systemOptions.add(new RuntimeOption(key, value));
+ return this;
}
- public void set(String key, boolean value) {
- values.put(key, OptionValue.create(AccessibleScopes.SYSTEM, key, value, OptionScope.SYSTEM));
- }
-
- public void set(String key, double value) {
- values.put(key, OptionValue.create(AccessibleScopes.SYSTEM, key, value, OptionScope.SYSTEM));
- }
-
- public void set(String key, String value) {
- values.put(key, OptionValue.create(AccessibleScopes.SYSTEM, key, value, OptionScope.SYSTEM));
- }
-
- @Override
- public OptionValue getOption(String name) {
- final OptionValue value = values.get(name);
-
- if (value != null) {
- return value;
- }
-
- return getDefault(name);
+ public OperatorFixtureBuilder setControls(ExecutionControls controls) {
+ this.controls = controls;
+ return this;
}
- @Override
- public OptionValue getDefault(String optionName) {
- return defaults.get(optionName);
+ public OperatorFixture build() {
+ return new OperatorFixture(this);
}
}
@@ -255,7 +196,7 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable {
public static class MockStats implements OperatorStatReceiver {
- public Map<Integer,Double> stats = new HashMap<>();
+ public Map<Integer, Double> stats = new HashMap<>();
@Override
public void addLongStat(MetricDef metric, long value) {
@@ -295,26 +236,39 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable {
}
}
- private final TestOptionSet options;
+ private final SystemOptionManager options;
private final TestCodeGenContext context;
private final OperatorStatReceiver stats;
protected OperatorFixture(OperatorFixtureBuilder builder) {
config = builder.configBuilder().build();
allocator = RootAllocatorFactory.newRoot(config);
- options = builder.options();
+ options = new SystemOptionManager(config);
+ try {
+ options.init();
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to initialize the system option manager", e);
+ }
+ if (builder.systemOptions != null) {
+ applySystemOptions(builder.systemOptions);
+ }
context = new TestCodeGenContext(config, options);
stats = new MockStats();
}
- public TestOptionSet options() { return options; }
-
+ private void applySystemOptions(List<RuntimeOption> systemOptions) {
+ for (RuntimeOption option : systemOptions) {
+ options.setLocalOption(option.key, option.value);
+ }
+ }
- public FragmentExecContext codeGenContext() { return context; }
+ public SystemOptionManager options() { return options; }
+ public FragmentExecContext fragmentExecContext() { return context; }
@Override
public void close() throws Exception {
allocator.close();
+ options.close();
}
public static OperatorFixtureBuilder builder() {
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
index 19b8ed5..83c60b3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
@@ -52,7 +52,7 @@ public class TestConfigLinkage {
public void testDefaultInternalValue() throws Exception {
OptionDefinition optionDefinition = createMockPropOptionDefinition();
- FixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder().
configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
putDefinition(optionDefinition);
@@ -72,7 +72,7 @@ public class TestConfigLinkage {
public void testDefaultValidatorInternalValue() throws Exception {
OptionDefinition optionDefinition = createMockPropOptionDefinition();
- FixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder().
putDefinition(optionDefinition).
configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a");
@@ -91,7 +91,7 @@ public class TestConfigLinkage {
/* Test if session option takes precedence */
@Test
public void testSessionOption() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder().sessionOption(ExecConstants.SLICE_TARGET, 10);
+ ClusterFixtureBuilder builder = ClusterFixture.builder().sessionOption(ExecConstants.SLICE_TARGET, 10);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SESSION'", SystemTable.OPTION_VAL
@@ -104,7 +104,7 @@ public class TestConfigLinkage {
/* Test if system option takes precedence over the boot option */
@Test
public void testSystemOption() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 10000);
+ ClusterFixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 10000);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String slice_target = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.slice_target' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName())
@@ -117,7 +117,7 @@ public class TestConfigLinkage {
public void testInternalSystemOption() throws Exception {
OptionDefinition optionDefinition = createMockPropOptionDefinition();
- FixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder().
putDefinition(optionDefinition).
configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
systemOption(MOCK_PROPERTY, "blah");
@@ -139,7 +139,7 @@ public class TestConfigLinkage {
/* Test if config option takes precedence if config option is not set */
@Test
public void testConfigOption() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.setOptionDefault(ExecConstants.SLICE_TARGET, 1000);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
@@ -164,7 +164,7 @@ public class TestConfigLinkage {
/* Test if altering session option takes precedence over system option */
@Test
public void testSessionPrecedence() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 100000);
+ ClusterFixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 100000);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
client.queryBuilder().sql("ALTER SESSION SET `planner.slice_target` = 10000").run();
@@ -177,7 +177,7 @@ public class TestConfigLinkage {
/* Test if setting maxwidth option through config takes effect */
@Test
public void testMaxWidthPerNodeConfig() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'BOOT'", SystemTable.OPTION_VAL.getTableName())
@@ -189,7 +189,7 @@ public class TestConfigLinkage {
/* Test if setting maxwidth at system level takes precedence */
@Test
public void testMaxWidthPerNodeSystem() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3);
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'SYSTEM'", SystemTable.OPTION_VAL.getTableName())
@@ -201,7 +201,7 @@ public class TestConfigLinkage {
/* Test if setting maxwidth at session level takes precedence */
@Test
public void testMaxWidthPerNodeSession() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.%s where name='planner.width.max_per_node' and optionScope = 'SESSION'", SystemTable.OPTION_VAL.getTableName())
@@ -215,7 +215,7 @@ public class TestConfigLinkage {
*/
@Test
public void testMaxWidthPerNodeDefault() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70);
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
long maxWidth = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(0.70, 0);
@@ -228,7 +228,7 @@ public class TestConfigLinkage {
/* Test if the scope is set during BOOT time and scope is actually BOOT */
@Test
public void testScope() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.SLICE_TARGET, 100000);
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.SLICE_TARGET, 100000);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String scope = client.queryBuilder()
@@ -241,7 +241,7 @@ public class TestConfigLinkage {
/* Test if the option is set at SYSTEM scope and the scope is actually SYSTEM */
@Test
public void testScopeSystem() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.SLICE_TARGET, 10000);
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.SLICE_TARGET, 10000);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String scope = client.queryBuilder()
@@ -254,7 +254,7 @@ public class TestConfigLinkage {
/* Test if the option is set at SESSION scope and the scope is actually SESSION */
@Test
public void testScopeSession() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.SLICE_TARGET, 100000);
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.SLICE_TARGET, 100000);
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
String scope = client.queryBuilder()
@@ -267,7 +267,7 @@ public class TestConfigLinkage {
/* Test if the option is altered at SYSTEM scope and the scope is actually SYSTEM */
@Test
public void testScopeAlterSystem() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder();
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder();
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
client.queryBuilder().sql("ALTER SYSTEM set `planner.slice_target`= 10000").run();
@@ -281,7 +281,7 @@ public class TestConfigLinkage {
/* Test if the option is altered at SESSION scope and the scope is actually SESSION */
@Test
public void testScopeAlterSession() throws Exception {
- FixtureBuilder builder = ClusterFixture.bareBuilder();
+ ClusterFixtureBuilder builder = ClusterFixture.bareBuilder();
try (ClusterFixture cluster = builder.build();
ClientFixture client = cluster.clientFixture()) {
client.queryBuilder().sql("ALTER SESSION set `planner.slice_target`= 10000").run();
@@ -296,7 +296,7 @@ public class TestConfigLinkage {
public void testAlterInternalSystemOption() throws Exception {
OptionDefinition optionDefinition = createMockPropOptionDefinition();
- FixtureBuilder builder = ClusterFixture.builder().
+ ClusterFixtureBuilder builder = ClusterFixture.builder().
configProperty(ExecConstants.bootDefaultFor(MOCK_PROPERTY), "a").
putDefinition(optionDefinition);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java b/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java
index 9f62478..dac71ec 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/package-info.java
@@ -20,7 +20,7 @@
* tests and ad-hoc tests created while developing features. Key components
* include:
* <ul>
- * <li>{@link FixtureBuilder}: Builder pattern to create an embedded Drillbit,
+ * <li>{@link ClusterFixtureBuilder}: Builder pattern to create an embedded Drillbit,
* or cluster of Drillbits, using a specified set of configuration, session
* and system options.</li>
* <li>{@link ClusterFixture}: The cluster created by the builder.</li>
[2/2] drill git commit: DRILL-5832: Change OperatorFixture to use
system option manager
Posted by ar...@apache.org.
DRILL-5832: Change OperatorFixture to use system option manager
- Rename FixtureBuilder to ClusterFixtureBuilder
- Provide alternative way to reset system/session options
- Fix for DRILL-5833: random failure in TestParquetWriter
- Provide strict, but clear, errors for missing options
closes #970
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/42fc11e5
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/42fc11e5
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/42fc11e5
Branch: refs/heads/master
Commit: 42fc11e53557477ac01c7dd31c3aa93e22fb4384
Parents: 3036d37
Author: Paul Rogers <pr...@maprtech.com>
Authored: Mon Oct 2 15:45:21 2017 -0700
Committer: Arina Ielchiieva <ar...@gmail.com>
Committed: Mon Nov 13 14:59:19 2017 +0200
----------------------------------------------------------------------
.../drill/common/util/CoreDecimalUtility.java | 9 +-
.../apache/drill/exec/hive/TestHiveStorage.java | 7 +-
.../drill/exec/ops/BufferManagerImpl.java | 5 +-
.../drill/exec/physical/impl/ImplCreator.java | 3 +
.../physical/impl/MergingReceiverCreator.java | 5 +-
.../drill/exec/physical/impl/ScanBatch.java | 24 +-
.../drill/exec/physical/impl/ScreenCreator.java | 6 +-
.../drill/exec/physical/impl/TraceInjector.java | 6 +-
.../exec/physical/impl/WriterRecordBatch.java | 2 +
.../exec/server/options/BaseOptionManager.java | 76 ++++-
.../exec/server/options/BaseOptionSet.java | 57 ----
.../drill/exec/server/options/OptionSet.java | 44 +++
.../exec/server/options/OptionValidator.java | 1 -
.../drill/exec/server/options/OptionValue.java | 4 +
.../server/options/PersistedOptionValue.java | 1 +
.../server/options/SystemOptionManager.java | 26 +-
.../drill/exec/store/dfs/FileSelection.java | 12 -
.../drill/exec/store/dfs/FileSystemConfig.java | 3 +-
.../exec/store/dfs/FileSystemSchemaFactory.java | 3 +-
.../drill/exec/store/dfs/ReadEntryFromHDFS.java | 4 +-
.../store/parquet/ParquetScanBatchCreator.java | 3 +-
.../parquet/columnreaders/ParquetSchema.java | 2 +-
.../parquet2/DrillParquetGroupConverter.java | 33 +-
.../exec/store/parquet2/DrillParquetReader.java | 39 +--
.../DrillParquetRecordMaterializer.java | 6 +-
.../store/provider/InMemoryStoreProvider.java | 11 +-
.../java/org/apache/drill/BaseTestQuery.java | 35 +-
.../physical/impl/agg/TestHashAggrSpill.java | 4 +-
.../impl/limit/TestLimitWithExchanges.java | 9 +-
.../impl/validate/TestValidationOptions.java | 8 +-
.../physical/impl/writer/TestParquetWriter.java | 113 ++++---
.../writer/TestParquetWriterEmptyFiles.java | 3 +-
.../impl/xsort/TestSimpleExternalSort.java | 10 +-
.../impl/xsort/TestSortSpillWithException.java | 4 +-
.../impl/xsort/managed/SortTestUtilities.java | 1 +
.../impl/xsort/managed/TestShortArrays.java | 2 +-
.../exec/server/TestDrillbitResilience.java | 33 +-
.../exec/server/rest/StatusResourcesTest.java | 6 +-
.../store/parquet/ParquetInternalsTest.java | 4 +-
.../parquet/TestParquetFilterPushDown.java | 8 +-
.../exec/store/sys/TestPStoreProviders.java | 4 +-
.../drill/exec/util/TestQueryMemoryAlloc.java | 18 +-
.../complex/writer/TestExtendedTypes.java | 18 +-
.../org/apache/drill/test/ClientFixture.java | 10 +-
.../org/apache/drill/test/ClusterFixture.java | 26 +-
.../drill/test/ClusterFixtureBuilder.java | 326 +++++++++++++++++++
.../java/org/apache/drill/test/ClusterTest.java | 2 +-
.../java/org/apache/drill/test/ExampleTest.java | 6 +-
.../org/apache/drill/test/FixtureBuilder.java | 326 -------------------
.../org/apache/drill/test/OperatorFixture.java | 114 ++-----
.../apache/drill/test/TestConfigLinkage.java | 34 +-
.../org/apache/drill/test/package-info.java | 2 +-
52 files changed, 761 insertions(+), 757 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
index 9f61ccf..bf12cfb 100644
--- a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
+++ b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
@@ -82,10 +82,9 @@ public class CoreDecimalUtility {
}
public static boolean isDecimalType(TypeProtos.MinorType minorType) {
- if (minorType == TypeProtos.MinorType.DECIMAL9 || minorType == TypeProtos.MinorType.DECIMAL18 ||
- minorType == TypeProtos.MinorType.DECIMAL28SPARSE || minorType == TypeProtos.MinorType.DECIMAL38SPARSE) {
- return true;
- }
- return false;
+ return minorType == TypeProtos.MinorType.DECIMAL9 ||
+ minorType == TypeProtos.MinorType.DECIMAL18 ||
+ minorType == TypeProtos.MinorType.DECIMAL28SPARSE ||
+ minorType == TypeProtos.MinorType.DECIMAL38SPARSE;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index efd26ff..cf98797 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -26,7 +26,6 @@ import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.test.OperatorFixture;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.joda.time.DateTime;
@@ -73,10 +72,8 @@ public class TestHiveStorage extends HiveTestBase {
.baselineValues(200L)
.go();
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- test(String.format("alter session set `%s` = %s",
- ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS,
- Boolean.toString(testOptionSet.getDefault(ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS).bool_val)));
+ test("alter session reset `%s`",
+ ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
index aa00b1c..d0dcace 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -44,6 +44,7 @@ public class BufferManagerImpl implements BufferManager {
managedBuffers.clear();
}
+ @Override
public DrillBuf replace(DrillBuf old, int newSize) {
if (managedBuffers.remove(old.memoryAddress()) == null) {
throw new IllegalStateException("Tried to remove unmanaged buffer.");
@@ -52,10 +53,12 @@ public class BufferManagerImpl implements BufferManager {
return getManagedBuffer(newSize);
}
+ @Override
public DrillBuf getManagedBuffer() {
return getManagedBuffer(256);
}
+ @Override
public DrillBuf getManagedBuffer(int size) {
DrillBuf newBuf = allocator.buffer(size, this);
managedBuffers.put(newBuf.memoryAddress(), newBuf);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
index 58bf383..0871621 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ImplCreator.java
@@ -105,6 +105,7 @@ public class ImplCreator {
}
/** Create RootExec and its children (RecordBatches) for given FragmentRoot */
+ @SuppressWarnings("unchecked")
private RootExec getRootExec(final FragmentRoot root, final FragmentContext context) throws ExecutionSetupException {
final List<RecordBatch> childRecordBatches = getChildren(root, context);
@@ -141,6 +142,7 @@ public class ImplCreator {
return proxyUgi.doAs(new PrivilegedExceptionAction<RecordBatch>() {
@Override
public RecordBatch run() throws Exception {
+ @SuppressWarnings("unchecked")
final CloseableRecordBatch batch = ((BatchCreator<PhysicalOperator>) getOpCreator(op, context)).getBatch(
context, op, childRecordBatches);
operators.addFirst(batch);
@@ -153,6 +155,7 @@ public class ImplCreator {
throw new ExecutionSetupException(errMsg, e);
}
} else {
+ @SuppressWarnings("unchecked")
final CloseableRecordBatch batch = ((BatchCreator<PhysicalOperator>) getOpCreator(op, context)).getBatch(context,
op, childRecordBatches);
operators.addFirst(batch);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java
index 59e7448..690a662 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/MergingReceiverCreator.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -31,6 +31,7 @@ import org.apache.drill.exec.work.batch.RawBatchBuffer;
public class MergingReceiverCreator implements BatchCreator<MergingReceiverPOP> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MergingReceiverCreator.class);
+ @SuppressWarnings("resource")
@Override
public MergingRecordBatch getBatch(FragmentContext context,
MergingReceiverPOP receiver,
@@ -45,6 +46,4 @@ public class MergingReceiverCreator implements BatchCreator<MergingReceiverPOP>
return new MergingRecordBatch(context, receiver, buffers);
}
-
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 64be129..6c2c171 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -165,9 +165,9 @@ public class ScanBatch implements CloseableRecordBatch {
try {
while (true) {
if (currentReader == null && !getNextReaderIfHas()) {
- releaseAssets(); // All data has been read. Release resource.
- done = true;
- return IterOutcome.NONE;
+ releaseAssets(); // All data has been read. Release resource.
+ done = true;
+ return IterOutcome.NONE;
}
injector.injectChecked(context.getExecutionControls(), "next-allocate", OutOfMemoryException.class);
currentReader.allocate(mutator.fieldVectorMap());
@@ -235,15 +235,14 @@ public class ScanBatch implements CloseableRecordBatch {
}
private boolean getNextReaderIfHas() throws ExecutionSetupException {
- if (readers.hasNext()) {
- currentReader = readers.next();
- implicitValues = implicitColumns.hasNext() ? implicitColumns.next() : null;
- currentReader.setup(oContext, mutator);
- currentReaderClassName = currentReader.getClass().getSimpleName();
- return true;
- } else {
+ if (!readers.hasNext()) {
return false;
}
+ currentReader = readers.next();
+ implicitValues = implicitColumns.hasNext() ? implicitColumns.next() : null;
+ currentReader.setup(oContext, mutator);
+ currentReaderClassName = currentReader.getClass().getSimpleName();
+ return true;
}
private void addImplicitVectors() {
@@ -251,8 +250,7 @@ public class ScanBatch implements CloseableRecordBatch {
if (!implicitColumnList.isEmpty()) {
for (String column : implicitColumnList.get(0).keySet()) {
final MaterializedField field = MaterializedField.create(column, Types.optional(MinorType.VARCHAR));
- @SuppressWarnings("resource")
- final ValueVector v = mutator.addField(field, NullableVarCharVector.class, true /*implicit field*/);
+ mutator.addField(field, NullableVarCharVector.class, true /*implicit field*/);
}
}
} catch(SchemaChangeException e) {
@@ -338,7 +336,6 @@ public class ScanBatch implements CloseableRecordBatch {
return implicitFieldVectorMap;
}
- @SuppressWarnings("resource")
@Override
public <T extends ValueVector> T addField(MaterializedField field,
Class<T> clazz) throws SchemaChangeException {
@@ -388,6 +385,7 @@ public class ScanBatch implements CloseableRecordBatch {
schemaChanged = false;
}
+ @SuppressWarnings("resource")
private <T extends ValueVector> T addField(MaterializedField field,
Class<T> clazz, boolean isImplicitField) throws SchemaChangeException {
Map<String, ValueVector> fieldVectorMap;
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
index 60355fb..d9abf40 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.exception.OutOfMemoryException;
-import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.ops.AccountingUserConnection;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.MetricDef;
@@ -33,12 +32,11 @@ import org.apache.drill.exec.proto.UserBitShared.QueryData;
import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatch.IterOutcome;
-
-import com.google.common.base.Preconditions;
-
import org.apache.drill.exec.testing.ControlsInjector;
import org.apache.drill.exec.testing.ControlsInjectorFactory;
+import com.google.common.base.Preconditions;
+
public class ScreenCreator implements RootCreator<Screen> {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScreenCreator.class);
private static final ControlsInjector injector = ControlsInjectorFactory.getInjector(ScreenCreator.class);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java
index 92d1882..0aac91e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TraceInjector.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -35,10 +35,7 @@ public class TraceInjector extends AbstractPhysicalVisitor<PhysicalOperator, Fra
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TraceInjector.class);
static int traceTagCount = 0;
-
-
RootExec root = null;
- private ScreenCreator sc = new ScreenCreator();
public static PhysicalOperator getExec(FragmentContext context, FragmentRoot root) throws ExecutionSetupException {
TraceInjector tI = new TraceInjector();
@@ -89,5 +86,4 @@ public class TraceInjector extends AbstractPhysicalVisitor<PhysicalOperator, Fra
return newOp;
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
index 939832b..0ea17d6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
@@ -131,11 +131,13 @@ public class WriterRecordBatch extends AbstractRecordBatch<Writer> {
}
private void addOutputContainerData() {
+ @SuppressWarnings("resource")
final VarCharVector fragmentIdVector = (VarCharVector) container.getValueAccessorById(
VarCharVector.class,
container.getValueVectorId(SchemaPath.getSimplePath("Fragment")).getFieldIds())
.getValueVector();
AllocationHelper.allocate(fragmentIdVector, 1, 50);
+ @SuppressWarnings("resource")
final BigIntVector summaryVector = (BigIntVector) container.getValueAccessorById(BigIntVector.class,
container.getValueVectorId(SchemaPath.getSimplePath("Number of records written")).getFieldIds())
.getValueVector();
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java
index 5a0fa2d..a502e07 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java
@@ -18,15 +18,87 @@
package org.apache.drill.exec.server.options;
import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.server.options.OptionValue.Kind;
import java.util.Iterator;
/**
- * This {@link OptionManager} implements some the basic methods and should be extended by concrete implementations.
+ * This {@link OptionManager} implements some the basic methods and should be
+ * extended by concrete implementations.
*/
-public abstract class BaseOptionManager extends BaseOptionSet implements OptionManager {
+
+public abstract class BaseOptionManager implements OptionManager {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseOptionManager.class);
+ /**
+ * Gets the current option value given a validator.
+ *
+ * @param validator the validator
+ * @return option value
+ * @throws IllegalArgumentException - if the validator is not found
+ */
+ private OptionValue getOptionSafe(OptionValidator validator) {
+ final String optionName = validator.getOptionName();
+ OptionValue value = getOption(optionName);
+ return value == null ? getDefault(optionName) : value;
+ }
+
+ @Override
+ public boolean getOption(TypeValidators.BooleanValidator validator) {
+ return getOptionSafe(validator).bool_val;
+ }
+
+ @Override
+ public double getOption(TypeValidators.DoubleValidator validator) {
+ return getOptionSafe(validator).float_val;
+ }
+
+ @Override
+ public long getOption(TypeValidators.LongValidator validator) {
+ return getOptionSafe(validator).num_val;
+ }
+
+ @Override
+ public String getOption(TypeValidators.StringValidator validator) {
+ return getOptionSafe(validator).string_val;
+ }
+
+ @Override
+ public boolean getBoolean(String name) {
+ return getByType(name, Kind.BOOLEAN).bool_val;
+ }
+
+ @Override
+ public long getLong(String name) {
+ return getByType(name, Kind.LONG).num_val;
+ }
+
+ @Override
+ public double getDouble(String name) {
+ return getByType(name, Kind.DOUBLE).float_val;
+ }
+
+ @Override
+ public String getString(String name) {
+ return getByType(name, Kind.STRING).string_val;
+ }
+
+ private OptionValue getByType(String name, Kind dataType) {
+ OptionValue value = getOption(name);
+ if (value == null) {
+ throw UserException.systemError(null)
+ .addContext("Undefined option: " + name)
+ .build(logger);
+ }
+ if (value.kind != dataType) {
+ throw UserException.systemError(null)
+ .addContext("Option " + name + " is of data type " +
+ value.kind + " but was requested as " + dataType)
+ .build(logger);
+ }
+ return value;
+ }
+
@Override
public OptionList getInternalOptionList() {
return getAllOptionList(true);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionSet.java
deleted file mode 100644
index f664401..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionSet.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.server.options;
-
-/**
- * A basic implementation of an {@link OptionSet}.
- */
-public abstract class BaseOptionSet implements OptionSet {
- /**
- * Gets the current option value given a validator.
- *
- * @param validator the validator
- * @return option value
- * @throws IllegalArgumentException - if the validator is not found
- */
- private OptionValue getOptionSafe(OptionValidator validator) {
- final String optionName = validator.getOptionName();
- OptionValue value = getOption(optionName);
- return value == null ? getDefault(optionName) : value;
- }
-
- @Override
- public boolean getOption(TypeValidators.BooleanValidator validator) {
- return getOptionSafe(validator).bool_val;
- }
-
- @Override
- public double getOption(TypeValidators.DoubleValidator validator) {
- return getOptionSafe(validator).float_val;
- }
-
- @Override
- public long getOption(TypeValidators.LongValidator validator) {
- return getOptionSafe(validator).num_val;
- }
-
- @Override
- public String getOption(TypeValidators.StringValidator validator) {
- return getOptionSafe(validator).string_val;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java
index 702d7df..e96d571 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionSet.java
@@ -74,4 +74,48 @@ public interface OptionSet {
* @return the string value
*/
String getOption(TypeValidators.StringValidator validator);
+
+ /**
+ * Return the value of a Boolean option.
+ *
+ * @param name option name
+ * @return the Boolean value
+ * @throws IllegalArgumentException if the option is undefined or
+ * is not of the correct data type
+ */
+
+ boolean getBoolean(String name);
+
+ /**
+ * Return the value of a long option.
+ *
+ * @param name option name
+ * @return the long value
+ * @throws IllegalArgumentException if the option is undefined or
+ * is not of the correct data type
+ */
+
+ long getLong(String name);
+
+ /**
+ * Return the value of a double option.
+ *
+ * @param name option name
+ * @return the double value
+ * @throws IllegalArgumentException if the option is undefined or
+ * is not of the correct data type
+ */
+
+ double getDouble(String name);
+
+ /**
+ * Return the value of a String option.
+ *
+ * @param name option name
+ * @return the String value
+ * @throws IllegalArgumentException if the option is undefined or
+ * is not of the correct data type
+ */
+
+ String getString(String name);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java
index 5418408..9143927 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.server.options;
-import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.server.options.OptionValue.Kind;
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
index e14a846..fc4a9f0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
@@ -133,10 +133,14 @@ public class OptionValue implements Comparable<OptionValue> {
return create(type, name, ((Boolean) val).booleanValue(), scope);
} else if (val instanceof Long) {
return create(type, name, ((Long) val).longValue(), scope);
+ } else if (val instanceof Integer) {
+ return create(type, name, ((Integer) val).longValue(), scope);
} else if (val instanceof String) {
return create(type, name, (String) val, scope);
} else if (val instanceof Double) {
return create(type, name, ((Double) val).doubleValue(), scope);
+ } else if (val instanceof Float) {
+ return create(type, name, ((Float) val).doubleValue(), scope);
}
throw new IllegalArgumentException(String.format("Unsupported type %s", val.getClass()));
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java
index 685799c..720a040 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/PersistedOptionValue.java
@@ -269,6 +269,7 @@ public class PersistedOptionValue {
* value of an option. This deserializer is essentially future proof since it only requires a value
* to be stored for an option.
*/
+ @SuppressWarnings("serial")
public static class Deserializer extends StdDeserializer<PersistedOptionValue> {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Deserializer.class);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index e81ed0b..1c45547 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -36,8 +36,10 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.store.sys.PersistentStore;
import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.apache.drill.exec.store.sys.PersistentStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.InMemoryStoreProvider;
import org.apache.drill.exec.util.AssertionUtil;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -223,7 +225,6 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
private final PersistentStoreProvider provider;
- private final DrillConfig bootConfig;
/**
* Persistent store for options that have been changed from default.
* NOTE: CRUD operations must use lowercase keys.
@@ -243,12 +244,25 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), PersistedOptionValue.class)
.name("sys.options")
.build();
- this.bootConfig = bootConfig;
this.definitions = definitions;
this.defaults = populateDefaultValues(definitions, bootConfig);
}
/**
+ * Test-only, in-memory version of the system option manager.
+ *
+ * @param bootConfig Drill config
+ */
+
+ @VisibleForTesting
+ public SystemOptionManager(final DrillConfig bootConfig) {
+ this.provider = new InMemoryStoreProvider(100);
+ this.config = null;
+ this.definitions = SystemOptionManager.createDefaultOptionDefinitions();
+ this.defaults = populateDefaultValues(definitions, bootConfig);
+ }
+
+ /**
* Initializes this option manager.
*
* @return this option manager
@@ -315,7 +329,13 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
@Override
public OptionValue getDefault(String optionName) {
- return defaults.get(optionName);
+ OptionValue value = defaults.get(optionName);
+ if (value == null) {
+ throw UserException.systemError(null)
+ .addContext("Undefined default value for option: " + optionName)
+ .build(logger);
+ }
+ return value;
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
index 81c7ad2..293259f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -208,18 +208,6 @@ public class FileSelection {
return this.wasAllPartitionsPruned;
}
- private static String commonPath(final List<FileStatus> statuses) {
- if (statuses == null || statuses.isEmpty()) {
- return "";
- }
-
- final List<String> files = Lists.newArrayList();
- for (final FileStatus status : statuses) {
- files.add(status.getPath().toString());
- }
- return commonPathForFiles(files);
- }
-
/**
* Returns longest common path for the given list of files.
*
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
index adc3909..3c8f3a7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemConfig.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -19,7 +19,6 @@ package org.apache.drill.exec.store.dfs;
import java.util.Map;
-import com.fasterxml.jackson.annotation.JsonInclude;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
index cf30162..5d99377 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.fs.Path;
* This is the top level schema that responds to root level path requests. Also supports
*/
public class FileSystemSchemaFactory implements SchemaFactory{
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchemaFactory.class);
public static final String DEFAULT_WS_NAME = "default";
@@ -61,6 +60,7 @@ public class FileSystemSchemaFactory implements SchemaFactory{
@Override
public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException {
+ @SuppressWarnings("resource")
FileSystemSchema schema = new FileSystemSchema(schemaName, schemaConfig);
SchemaPlus plusOfThis = parent.add(schema.getName(), schema);
schema.setPlus(plusOfThis);
@@ -75,6 +75,7 @@ public class FileSystemSchemaFactory implements SchemaFactory{
super(ImmutableList.<String>of(), name);
for(WorkspaceSchemaFactory f : factories){
if (f.accessible(schemaConfig.getUserName())) {
+ @SuppressWarnings("resource")
WorkspaceSchema s = f.createSchema(getSchemaPath(), schemaConfig);
schemaMap.put(s.getName(), s);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java
index 97124bc..e02d841 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/ReadEntryFromHDFS.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -34,10 +34,12 @@ public class ReadEntryFromHDFS extends ReadEntryWithPath implements FileWork{
this.length = length;
}
+ @Override
public long getStart() {
return start;
}
+ @Override
public long getLength() {
return length;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index 6017948..bb0b65f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -57,6 +57,7 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
private static final String ENABLE_BYTES_TOTAL_COUNTER = "parquet.benchmark.bytes.total";
private static final String ENABLE_TIME_READ_COUNTER = "parquet.benchmark.time.read";
+ @SuppressWarnings("resource")
@Override
public ScanBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan, List<RecordBatch> children)
throws ExecutionSetupException {
@@ -119,7 +120,7 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
if (logger.isDebugEnabled()) {
logger.debug(containsCorruptDates.toString());
}
- if (!context.getOptions().getOption(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val && !isComplex(footers.get(e.getPath()))) {
+ if (!context.getOptions().getBoolean(ExecConstants.PARQUET_NEW_RECORD_READER) && !isComplex(footers.get(e.getPath()))) {
readers.add(
new ParquetRecordReader(
context, e.getPath(), e.getRowGroupIndex(), e.getNumRecordsToRead(), fs,
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
index 10187b7..773f3d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java
@@ -118,7 +118,7 @@ public class ParquetSchema {
loadParquetSchema();
computeFixedPart();
- if (! selectedColumnMetadata.isEmpty() && allFieldsFixedLength) {
+ if (! selectedColumnMetadata.isEmpty() && allFieldsFixedLength) {
recordsPerBatch = (int) Math.min(Math.min(batchSize / bitWidthAllFixedFields,
footer.getBlocks().get(0).getColumns().get(0).getValueCount()), ParquetRecordReader.DEFAULT_RECORDS_TO_READ_IF_FIXED_WIDTH);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
index 5c8db91..5c7c8e1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
@@ -17,7 +17,7 @@
*/
package org.apache.drill.exec.store.parquet2;
-import io.netty.buffer.DrillBuf;
+import static org.apache.drill.exec.store.parquet.ParquetReaderUtility.NanoTimeUtils.getDateTimeValueFromBinary;
import java.math.BigDecimal;
import java.util.ArrayList;
@@ -48,7 +48,6 @@ import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.store.parquet.ParquetReaderUtility;
import org.apache.drill.exec.store.parquet.columnreaders.ParquetColumnMetadata;
-import org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader;
import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
@@ -67,8 +66,6 @@ import org.apache.drill.exec.vector.complex.writer.TimeStampWriter;
import org.apache.drill.exec.vector.complex.writer.TimeWriter;
import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter;
import org.apache.drill.exec.vector.complex.writer.VarCharWriter;
-import org.joda.time.DateTimeConstants;
-
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.Converter;
import org.apache.parquet.io.api.GroupConverter;
@@ -80,10 +77,11 @@ import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Type;
import org.apache.parquet.schema.Type.Repetition;
+import org.joda.time.DateTimeConstants;
import com.google.common.collect.Lists;
-import static org.apache.drill.exec.store.parquet.ParquetReaderUtility.NanoTimeUtils.getDateTimeValueFromBinary;
+import io.netty.buffer.DrillBuf;
public class DrillParquetGroupConverter extends GroupConverter {
@@ -182,7 +180,9 @@ public class DrillParquetGroupConverter extends GroupConverter {
switch(type.getOriginalType()) {
case DECIMAL: {
ParquetReaderUtility.checkDecimalTypeEnabled(options);
- Decimal9Writer writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal9() : mapWriter.decimal9(name);
+ Decimal9Writer writer = type.getRepetition() == Repetition.REPEATED
+ ? mapWriter.list(name).decimal9()
+ : mapWriter.decimal9(name, type.getDecimalMetadata().getScale(), type.getDecimalMetadata().getPrecision());
return new DrillDecimal9Converter(writer, type.getDecimalMetadata().getPrecision(), type.getDecimalMetadata().getScale());
}
case DATE: {
@@ -218,7 +218,9 @@ public class DrillParquetGroupConverter extends GroupConverter {
switch(type.getOriginalType()) {
case DECIMAL: {
ParquetReaderUtility.checkDecimalTypeEnabled(options);
- Decimal18Writer writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal18() : mapWriter.decimal18(name);
+ Decimal18Writer writer = type.getRepetition() == Repetition.REPEATED
+ ? mapWriter.list(name).decimal18()
+ : mapWriter.decimal18(name, type.getDecimalMetadata().getScale(), type.getDecimalMetadata().getPrecision());
return new DrillDecimal18Converter(writer, type.getDecimalMetadata().getPrecision(), type.getDecimalMetadata().getScale());
}
case TIMESTAMP_MILLIS: {
@@ -270,10 +272,14 @@ public class DrillParquetGroupConverter extends GroupConverter {
ParquetReaderUtility.checkDecimalTypeEnabled(options);
DecimalMetadata metadata = type.getDecimalMetadata();
if (metadata.getPrecision() <= 28) {
- Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision());
+ Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED
+ ? mapWriter.list(name).decimal28Sparse()
+ : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision());
return new DrillBinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer());
} else {
- Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal38Sparse() : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision());
+ Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED
+ ? mapWriter.list(name).decimal38Sparse()
+ : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision());
return new DrillBinaryToDecimal38Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer());
}
}
@@ -287,10 +293,14 @@ public class DrillParquetGroupConverter extends GroupConverter {
ParquetReaderUtility.checkDecimalTypeEnabled(options);
DecimalMetadata metadata = type.getDecimalMetadata();
if (metadata.getPrecision() <= 28) {
- Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision());
+ Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED
+ ? mapWriter.list(name).decimal28Sparse()
+ : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision());
return new DrillBinaryToDecimal28Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer());
} else {
- Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal38Sparse() : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision());
+ Decimal38SparseWriter writer = type.getRepetition() == Repetition.REPEATED
+ ? mapWriter.list(name).decimal38Sparse()
+ : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision());
return new DrillBinaryToDecimal38Converter(writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer());
}
} else if (type.getOriginalType() == OriginalType.INTERVAL) {
@@ -620,6 +630,7 @@ public class DrillParquetGroupConverter extends GroupConverter {
private VarBinaryWriter writer;
private VarBinaryHolder holder = new VarBinaryHolder();
+ @SuppressWarnings("resource")
public DrillFixedBinaryToVarbinaryConverter(VarBinaryWriter writer, int length, DrillBuf buf) {
this.writer = writer;
holder.buffer = buf = buf.reallocIfNeeded(length);
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 9e66f6d..e74c621 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -32,7 +32,6 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.OutOfMemoryException;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.ops.FragmentContext;
@@ -47,19 +46,18 @@ import org.apache.drill.exec.store.parquet.RowGroupReadEntry;
import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.NullableIntVector;
import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.hadoop.CodecFactory;
-import org.apache.parquet.hadoop.metadata.ColumnPath;
-import org.apache.parquet.io.RecordReader;
import org.apache.parquet.hadoop.ColumnChunkIncReadStore;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.ColumnIOFactory;
import org.apache.parquet.io.MessageColumnIO;
+import org.apache.parquet.io.RecordReader;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.Type;
@@ -84,14 +82,7 @@ public class DrillParquetReader extends AbstractRecordReader {
private RecordReader<Void> recordReader;
private DrillParquetRecordMaterializer recordMaterializer;
private int recordCount;
- private List<ValueVector> primitiveVectors;
private OperatorContext operatorContext;
- // The interface for the parquet-mr library does not allow re-winding, to enable us to write into our
- // fixed size value vectors, we must check how full the vectors are after some number of reads, for performance
- // we avoid doing this every record. These values are populated with system/session settings to allow users to optimize
- // for performance or allow a wider record size to be suported
- private final int fillLevelCheckFrequency;
- private final int fillLevelCheckThreshold;
private FragmentContext fragmentContext;
// For columns not found in the file, we need to return a schema element with the correct number of values
@@ -102,8 +93,8 @@ public class DrillParquetReader extends AbstractRecordReader {
// No actual data needs to be read out of the file, we only need to return batches until we have 'read' the number of
// records specified in the row group metadata
long mockRecordsRead=0;
- private List<SchemaPath> columnsNotFound=null;
- boolean noColumnsFound = false; // true if none of the columns in the projection list is found in the schema
+ private List<SchemaPath> columnsNotFound;
+ boolean noColumnsFound; // true if none of the columns in the projection list is found in the schema
// See DRILL-4203
private final ParquetReaderUtility.DateCorruptionStatus containsCorruptedDates;
@@ -116,8 +107,6 @@ public class DrillParquetReader extends AbstractRecordReader {
this.entry = entry;
setColumns(columns);
this.fragmentContext = fragmentContext;
- fillLevelCheckFrequency = this.fragmentContext.getOptions().getOption(ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD).num_val.intValue();
- fillLevelCheckThreshold = this.fragmentContext.getOptions().getOption(ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD).num_val.intValue();
}
public static MessageType getProjection(MessageType schema,
@@ -263,13 +252,13 @@ public class DrillParquetReader extends AbstractRecordReader {
}
}
- if(!noColumnsFound) {
+ if (!noColumnsFound) {
writer = new VectorContainerWriter(output);
// Discard the columns not found in the schema when create DrillParquetRecordMaterializer, since they have been added to output already.
+ @SuppressWarnings("unchecked")
final Collection<SchemaPath> columns = columnsNotFound == null || columnsNotFound.size() == 0 ? getColumns(): CollectionUtils.subtract(getColumns(), columnsNotFound);
recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection, columns,
fragmentContext.getOptions(), containsCorruptedDates);
- primitiveVectors = writer.getMapVector().getPrimitiveVectors();
recordReader = columnIO.getRecordReader(pageReadStore, recordMaterializer);
}
} catch (Exception e) {
@@ -331,22 +320,6 @@ public class DrillParquetReader extends AbstractRecordReader {
return count;
}
- private int getPercentFilled() {
- int filled = 0;
- for (final ValueVector v : primitiveVectors) {
- filled = Math.max(filled, v.getAccessor().getValueCount() * 100 / v.getValueCapacity());
- if (v instanceof VariableWidthVector) {
- filled = Math.max(filled, ((VariableWidthVector) v).getCurrentSizeInBytes() * 100 / ((VariableWidthVector) v).getByteCapacity());
- }
- // TODO - need to re-enable this
-// if (v instanceof RepeatedFixedWidthVector) {
-// filled = Math.max(filled, ((RepeatedFixedWidthVector) v).getAccessor().getGroupCount() * 100)
-// }
- }
- logger.debug("Percent filled: {}", filled);
- return filled;
- }
-
@Override
public void close() {
try {
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
index 2d778bd..f6ffdbd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
@@ -17,19 +17,17 @@
*/
package org.apache.drill.exec.store.parquet2;
+import java.util.Collection;
+
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.store.parquet.ParquetReaderUtility;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
-
import org.apache.parquet.io.api.GroupConverter;
import org.apache.parquet.io.api.RecordMaterializer;
import org.apache.parquet.schema.MessageType;
-import java.util.Collection;
-import java.util.List;
-
public class DrillParquetRecordMaterializer extends RecordMaterializer<Void> {
public DrillParquetGroupConverter root;
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java
index ffe7b18..3ab85ec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/InMemoryStoreProvider.java
@@ -32,10 +32,7 @@ public class InMemoryStoreProvider implements PersistentStoreProvider {
}
@Override
- public void close() throws Exception {
- // TODO Auto-generated method stub
-
- }
+ public void close() throws Exception { }
@Override
public <V> PersistentStore<V> getOrCreateStore(PersistentStoreConfig<V> config) throws StoreException {
@@ -43,9 +40,5 @@ public class InMemoryStoreProvider implements PersistentStoreProvider {
}
@Override
- public void start() throws Exception {
- // TODO Auto-generated method stub
-
- }
-
+ public void start() throws Exception { }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index a094f51..05ca6ea 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -78,6 +78,7 @@ import java.util.regex.Pattern;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.test.ClusterFixture;
public class BaseTestQuery extends ExecTest {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class);
@@ -215,6 +216,7 @@ public class BaseTestQuery extends ExecTest {
bits[i] = new Drillbit(config, serviceSet, classpathScan);
bits[i].run();
+ @SuppressWarnings("resource")
final StoragePluginRegistry pluginRegistry = bits[i].getContext().getStorage();
TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTmpSchemaLocation);
TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry);
@@ -377,6 +379,25 @@ public class BaseTestQuery extends ExecTest {
testNoResult(1, query, args);
}
+ public static void alterSession(String option, Object value) {
+ String valueStr = ClusterFixture.stringify(value);
+ try {
+ test("ALTER SESSION SET `%s` = %s", option, valueStr);
+ } catch(final Exception e) {
+ fail(String.format("Failed to set session option `%s` = %s, Error: %s",
+ option, valueStr, e.toString()));
+ }
+ }
+
+ public static void resetSessionOption(String option) {
+ try {
+ test("ALTER SESSION RESET `%s`", option);
+ } catch(final Exception e) {
+ fail(String.format("Failed to reset session option `%s`, Error: %s",
+ option, e.toString()));
+ }
+ }
+
protected static void testNoResult(int interation, String query, Object... args) throws Exception {
query = String.format(query, args);
logger.debug("Running query:\n--------------\n" + query);
@@ -440,9 +461,9 @@ public class BaseTestQuery extends ExecTest {
} catch (AssertionError e) {
e.addSuppressed(actualException);
throw e;
+ }
}
}
- }
/**
* Utility method which tests given query produces a {@link UserException}
@@ -479,23 +500,19 @@ public class BaseTestQuery extends ExecTest {
}
protected static void setSessionOption(final String option, final boolean value) {
- setSessionOption(option, Boolean.toString(value));
+ alterSession(option, value);
}
protected static void setSessionOption(final String option, final long value) {
- setSessionOption(option, Long.toString(value));
+ alterSession(option, value);
}
protected static void setSessionOption(final String option, final double value) {
- setSessionOption(option, Double.toString(value));
+ alterSession(option, value);
}
protected static void setSessionOption(final String option, final String value) {
- try {
- runSQL(String.format("alter session set `%s` = %s", option, value));
- } catch(final Exception e) {
- fail(String.format("Failed to set session option `%s` = %s, Error: %s", option, value, e.toString()));
- }
+ alterSession(option, value);
}
public static class SilentListener implements UserResultsListener {
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java
index 2003857..2e81acd 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggrSpill.java
@@ -28,7 +28,7 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
-import org.apache.drill.test.FixtureBuilder;
+import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.test.LogFixture;
import org.apache.drill.test.ProfileParser;
import org.apache.drill.test.QueryBuilder;
@@ -94,7 +94,7 @@ public class TestHashAggrSpill {
.logger("org.apache.drill", Level.WARN)
;
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.sessionOption(ExecConstants.HASHAGG_MAX_MEMORY_KEY,maxMem)
.sessionOption(ExecConstants.HASHAGG_NUM_PARTITIONS_KEY,numPartitions)
.sessionOption(ExecConstants.HASHAGG_MIN_BATCHES_PER_PARTITION_KEY,minBatches)
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java
index 8b75f3a..b49a12e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimitWithExchanges.java
@@ -74,8 +74,7 @@ public class TestLimitWithExchanges extends BaseTestQuery {
final String[] expectedPlan5 = {"(?s)Limit\\(fetch=\\[1\\].*UnionExchange.*Limit\\(fetch=\\[1\\]\\).*Join"};
testLimitHelper(sql5, expectedPlan5, excludedPlan, 1);
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- test("alter session set `%s` = %s", ExecConstants.SLICE_TARGET, testOptionSet.getDefault(ExecConstants.SLICE_TARGET).getValue());
+ resetSessionOption(ExecConstants.SLICE_TARGET);
}
}
@@ -96,8 +95,7 @@ public class TestLimitWithExchanges extends BaseTestQuery {
testLimitHelper(sql2, expectedPlan, excludedPlan2, 5);
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- test("alter session set `planner.slice_target` = " + testOptionSet.getDefault(ExecConstants.SLICE_TARGET).getValue());
+ resetSessionOption(ExecConstants.SLICE_TARGET);
}
}
@@ -126,8 +124,7 @@ public class TestLimitWithExchanges extends BaseTestQuery {
testLimitHelper(sql3, expectedPlan2, excludedPlan2, 10);
testLimitHelper(sql4, expectedPlan2, excludedPlan2, 10);
} finally {
- final OperatorFixture.TestOptionSet testOptionSet = new OperatorFixture.TestOptionSet();
- test("alter session set `planner.slice_target` = " + testOptionSet.getDefault(ExecConstants.SLICE_TARGET).getValue());
+ resetSessionOption(ExecConstants.SLICE_TARGET);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java
index d4e33b0..4b3cbff 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/validate/TestValidationOptions.java
@@ -24,7 +24,7 @@ import org.apache.drill.exec.store.easy.text.compliant.CompliantTextRecordReader
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.DrillTest;
-import org.apache.drill.test.FixtureBuilder;
+import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.test.LogFixture;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -59,7 +59,7 @@ public class TestValidationOptions extends DrillTest {
@Test
public void testOptions() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.maxParallelization(1)
.configProperty(ExecConstants.ENABLE_ITERATOR_VALIDATION, false)
.configProperty(ExecConstants.ENABLE_VECTOR_VALIDATION, false)
@@ -93,7 +93,7 @@ public class TestValidationOptions extends DrillTest {
@Test
public void testConfig() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.maxParallelization(1)
.configProperty(ExecConstants.ENABLE_ITERATOR_VALIDATION, true)
.configProperty(ExecConstants.ENABLE_VECTOR_VALIDATION, true)
@@ -119,7 +119,7 @@ public class TestValidationOptions extends DrillTest {
@Test
public void testDefaults() throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.maxParallelization(1)
;
try (ClusterFixture cluster = builder.build();
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 41be0ba..9715d14 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -17,8 +17,8 @@
*/
package org.apache.drill.exec.physical.impl.writer;
-import static org.apache.drill.exec.store.parquet.ParquetRecordWriter.DRILL_VERSION_PROPERTY;
import static org.apache.drill.TestBuilder.convertToLocalTimestamp;
+import static org.apache.drill.exec.store.parquet.ParquetRecordWriter.DRILL_VERSION_PROPERTY;
import static org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS;
import static org.junit.Assert.assertEquals;
@@ -26,14 +26,13 @@ import java.io.File;
import java.io.FileWriter;
import java.math.BigDecimal;
import java.sql.Date;
-import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.google.common.base.Joiner;
import org.apache.drill.BaseTestQuery;
import org.apache.drill.categories.ParquetTest;
import org.apache.drill.categories.SlowTest;
@@ -44,7 +43,6 @@ import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.fn.interp.TestConstantFolding;
import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.apache.drill.test.OperatorFixture;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -63,10 +61,11 @@ import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import com.google.common.base.Joiner;
+
@RunWith(Parameterized.class)
@Category({SlowTest.class, ParquetTest.class})
public class TestParquetWriter extends BaseTestQuery {
-// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetWriter.class);
@Parameterized.Parameters
public static Collection<Object[]> data() {
@@ -126,12 +125,12 @@ public class TestParquetWriter extends BaseTestQuery {
@BeforeClass
public static void initFs() throws Exception {
fs = getLocalFileSystem();
- test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ alterSession(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY, true);
}
@AfterClass
public static void disableDecimalDataType() throws Exception {
- test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ alterSession(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY, false);
}
@Test
@@ -188,25 +187,26 @@ public class TestParquetWriter extends BaseTestQuery {
try {
// read all of the types with the complex reader
- test(String.format("alter session set %s = true", ExecConstants.PARQUET_NEW_RECORD_READER));
+ alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true);
runTestAndValidate(allTypesSelection, "*", allTypesTable, "donuts_json");
} finally {
- test(String.format("alter session set %s = false", ExecConstants.PARQUET_NEW_RECORD_READER));
+ resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER);
}
}
@Test
public void testAllScalarTypesDictionary() throws Exception {
try {
- test(String.format("alter session set %s = true", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING));
+ alterSession(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, true);
/// read once with the flat reader
runTestAndValidate(allTypesSelection, "*", allTypesTable, "donuts_json");
// read all of the types with the complex reader
- test(String.format("alter session set %s = true", ExecConstants.PARQUET_NEW_RECORD_READER));
+ alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true);
runTestAndValidate(allTypesSelection, "*", allTypesTable, "donuts_json");
} finally {
- test(String.format("alter session set %s = false", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING));
+ resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER);
+ resetSessionOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING);
}
}
@@ -221,10 +221,10 @@ public class TestParquetWriter extends BaseTestQuery {
String selection = "type";
String inputTable = "cp.`donuts.json`";
try {
- test(String.format("alter session set %s = true", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING));
+ alterSession(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, true);
runTestAndValidate(selection, selection, inputTable, "donuts_json");
} finally {
- test(String.format("alter session set %s = false", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING));
+ resetSessionOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING);
}
}
@@ -262,7 +262,7 @@ public class TestParquetWriter extends BaseTestQuery {
@Test
public void testTPCHReadWrite1_date_convertedType() throws Exception {
try {
- test("alter session set `%s` = false", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING);
+ alterSession(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, false);
String selection = "L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, " +
"L_RETURNFLAG, L_LINESTATUS, L_SHIPDATE, cast(L_COMMITDATE as DATE) as L_COMMITDATE, cast(L_RECEIPTDATE as DATE) AS L_RECEIPTDATE, L_SHIPINSTRUCT, L_SHIPMODE, L_COMMENT";
String validationSelection = "L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER, L_QUANTITY, L_EXTENDEDPRICE, L_DISCOUNT, L_TAX, " +
@@ -270,9 +270,7 @@ public class TestParquetWriter extends BaseTestQuery {
String inputTable = "cp.`tpch/lineitem.parquet`";
runTestAndValidate(selection, validationSelection, inputTable, "lineitem_parquet_converted");
} finally {
- OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet();
- test("alter session set `%s` = %b", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING,
- optionSet.getDefault(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING).bool_val);
+ resetSessionOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING);
}
}
@@ -321,29 +319,24 @@ public class TestParquetWriter extends BaseTestQuery {
@Test
public void testTPCHReadWriteNoDictUncompressed() throws Exception {
try {
- test(String.format("alter session set `%s` = false", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING));
- test(String.format("alter session set `%s` = 'none'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE));
+ alterSession(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, false);
+ alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "none");
String inputTable = "cp.`tpch/supplier.parquet`";
runTestAndValidate("*", "*", inputTable, "supplier_parquet_no_dict_uncompressed");
} finally {
- final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet();
- test(String.format("alter session set `%s` = %b", ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING,
- optionSet.getDefault(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING).bool_val));
- test(String.format("alter session set `%s` = '%s'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE,
- optionSet.getDefault(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val));
+ resetSessionOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING);
+ resetSessionOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE);
}
}
@Test
public void testTPCHReadWriteDictGzip() throws Exception {
try {
- test(String.format("alter session set `%s` = 'gzip'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE));
+ alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "gzip");
String inputTable = "cp.`tpch/supplier.parquet`";
runTestAndValidate("*", "*", inputTable, "supplier_parquet_dict_gzip");
} finally {
- final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet();
- test(String.format("alter session set `%s` = '%s'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE,
- optionSet.getDefault(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val));
+ resetSessionOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE);
}
}
@@ -393,22 +386,33 @@ public class TestParquetWriter extends BaseTestQuery {
"cast(salary as decimal(24,2)) as decimal24, cast(salary as decimal(38,2)) as decimal38";
String validateSelection = "decimal8, decimal15, decimal24, decimal38";
String inputTable = "cp.`employee.json`";
- runTestAndValidate(selection, validateSelection, inputTable, "parquet_decimal");
+
+ // DRILL-5833: The "old" writer had a decimal bug, but the new one
+ // did not. The one used was random. Force the test to run both
+ // the old and new readers.
+
+ try {
+ alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true);
+ runTestAndValidate(selection, validateSelection, inputTable, "parquet_decimal");
+ alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, false);
+ runTestAndValidate(selection, validateSelection, inputTable, "parquet_decimal");
+ } finally {
+ resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER);
+ }
}
@Test
public void testMulipleRowGroups() throws Exception {
try {
- test(String.format("ALTER SESSION SET `%s` = %d", ExecConstants.PARQUET_BLOCK_SIZE, 1024*1024));
+ alterSession(ExecConstants.PARQUET_BLOCK_SIZE, 1024*1024);
String selection = "mi";
String inputTable = "cp.`customer.json`";
runTestAndValidate(selection, selection, inputTable, "foodmart_customer_parquet");
} finally {
- test(String.format("ALTER SESSION SET `%s` = %d", ExecConstants.PARQUET_BLOCK_SIZE, 512*1024*1024));
+ resetSessionOption(ExecConstants.PARQUET_BLOCK_SIZE);
}
}
-
@Test
public void testDate() throws Exception {
String selection = "cast(hire_date as DATE) as hire_date";
@@ -465,9 +469,7 @@ public class TestParquetWriter extends BaseTestQuery {
.optionSettingQueriesForBaseline("alter system set `store.parquet.use_new_reader` = true")
.build().run();
} finally {
- final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet();
- test("alter system set `%s` = %b", ExecConstants.PARQUET_NEW_RECORD_READER,
- optionSet.getDefault(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val);
+ resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER);
}
}
@@ -486,10 +488,7 @@ public class TestParquetWriter extends BaseTestQuery {
"alter system set `store.parquet.use_new_reader` = true")
.build().run();
} finally {
- final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet();
- test("alter system set `%s` = %b",
- ExecConstants.PARQUET_NEW_RECORD_READER,
- optionSet.getDefault(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val);
+ resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER);
}
}
@@ -730,7 +729,6 @@ public class TestParquetWriter extends BaseTestQuery {
try {
deleteTableIfExists(outputFile);
test("use dfs_test.tmp");
- // test("ALTER SESSION SET `planner.add_producer_consumer` = false");
String query = String.format("SELECT %s FROM %s", selection, inputTable);
String create = "CREATE TABLE " + outputFile + " AS " + query;
String validateQuery = String.format("SELECT %s FROM " + outputFile, validationSelection);
@@ -738,13 +736,16 @@ public class TestParquetWriter extends BaseTestQuery {
testBuilder()
.unOrdered()
- .sqlQuery(query)
- .sqlBaselineQuery(validateQuery)
+ // Validate query is the query on the output file (the one to validate)
+ .sqlQuery(validateQuery)
+ // The basline query is the query on the input file (the expected values)
+ .sqlBaselineQuery(query)
.go();
Configuration hadoopConf = new Configuration();
hadoopConf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
Path output = new Path(getDfsTestTmpSchemaLocation(), outputFile);
+ @SuppressWarnings("resource")
FileSystem fs = output.getFileSystem(hadoopConf);
for (FileStatus file : fs.listStatus(output)) {
ParquetMetadata footer = ParquetFileReader.readFooter(hadoopConf, file, SKIP_ROW_GROUPS);
@@ -765,10 +766,10 @@ public class TestParquetWriter extends BaseTestQuery {
public void testImpalaParquetInt96() throws Exception {
compareParquetReadersColumnar("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`");
try {
- test("alter session set %s = true", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
+ alterSession(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP, true);
compareParquetReadersColumnar("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`");
} finally {
- test("alter session reset %s", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
+ resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
}
}
@@ -799,7 +800,7 @@ public class TestParquetWriter extends BaseTestQuery {
.baselineColumns("int96_ts")
.build().run();
} finally {
- test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
+ resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
}
}
@@ -818,12 +819,12 @@ public class TestParquetWriter extends BaseTestQuery {
@Test
public void testImpalaParquetTimestampInt96AsTimeStamp() throws Exception {
try {
- test("alter session set %s = false", ExecConstants.PARQUET_NEW_RECORD_READER);
+ alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, false);
compareParquetInt96Converters("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`");
- test("alter session set %s = true", ExecConstants.PARQUET_NEW_RECORD_READER);
+ alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true);
compareParquetInt96Converters("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`");
} finally {
- test("alter session reset `%s`", ExecConstants.PARQUET_NEW_RECORD_READER);
+ resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER);
}
}
@@ -939,7 +940,7 @@ public class TestParquetWriter extends BaseTestQuery {
.build()
.run();
} finally {
- test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
+ resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
}
}
@@ -958,24 +959,22 @@ public class TestParquetWriter extends BaseTestQuery {
@Test
public void testTPCHReadWriteGzip() throws Exception {
try {
- test(String.format("alter session set `%s` = 'gzip'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE));
+ alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "gzip");
String inputTable = "cp.`tpch/supplier.parquet`";
runTestAndValidate("*", "*", inputTable, "suppkey_parquet_dict_gzip");
} finally {
- final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet();
- test(String.format("alter session set `%s` = '%s'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, optionSet.getDefault(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val));
+ resetSessionOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE);
}
}
@Test
public void testTPCHReadWriteSnappy() throws Exception {
try {
- test(String.format("alter session set `%s` = 'snappy'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE));
+ alterSession(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, "snappy");
String inputTable = "cp.`supplier_snappy.parquet`";
runTestAndValidate("*", "*", inputTable, "suppkey_parquet_dict_snappy");
} finally {
- final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet();
- test(String.format("alter session set `%s` = '%s'", ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, optionSet.getDefault(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val));
+ resetSessionOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE);
}
}
@@ -997,7 +996,7 @@ public class TestParquetWriter extends BaseTestQuery {
.build()
.run();
} finally {
- test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
+ resetSessionOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java
index c9d4300..2170f41 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java
@@ -97,8 +97,7 @@ public class TestParquetWriterEmptyFiles extends BaseTestQuery {
.go();
} finally {
// restore the session option
- final OperatorFixture.TestOptionSet optionSet = new OperatorFixture.TestOptionSet();
- test("ALTER SESSION SET `store.parquet.block-size` = %d", optionSet.getDefault(ExecConstants.PARQUET_BLOCK_SIZE).num_val);
+ resetSessionOption(ExecConstants.PARQUET_BLOCK_SIZE);
deleteTableIfExists(outputFile);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/42fc11e5/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
index d2d7b0c..aa0d4ee 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
@@ -34,7 +34,7 @@ import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.test.ClientFixture;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.DrillTest;
-import org.apache.drill.test.FixtureBuilder;
+import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.categories.SlowTest;
import org.junit.Rule;
import org.junit.Test;
@@ -69,7 +69,7 @@ public class TestSimpleExternalSort extends DrillTest {
*/
private void mergeSortWithSv2(boolean testLegacy) throws Exception {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false)
;
try (ClusterFixture cluster = builder.build();
@@ -96,7 +96,7 @@ public class TestSimpleExternalSort extends DrillTest {
}
private void sortOneKeyDescendingMergeSort(boolean testLegacy) throws Throwable {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false)
;
try (ClusterFixture cluster = builder.build();
@@ -147,7 +147,7 @@ public class TestSimpleExternalSort extends DrillTest {
}
private void sortOneKeyDescendingExternalSort(boolean testLegacy) throws Throwable {
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
.configProperty(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD, 4)
.configProperty(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE, 4)
.configProperty(ExecConstants.EXTERNAL_SORT_BATCH_LIMIT, 4)
@@ -173,7 +173,7 @@ public class TestSimpleExternalSort extends DrillTest {
}
private void outOfMemoryExternalSort(boolean testLegacy) throws Throwable{
- FixtureBuilder builder = ClusterFixture.builder()
+ ClusterFixtureBuilder builder = ClusterFixture.builder()
// Probably do nothing in modern Drill
.configProperty("drill.memory.fragment.max", 50_000_000)
.configProperty("drill.memory.fragment.initial", 2_000_000)