You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/12 18:34:52 UTC
[01/24] git commit: DRILL-660: Fix errors when querying a hive table
with no data.
Repository: incubator-drill
Updated Branches:
refs/heads/master 27a9c98a5 -> 4198a17a8
DRILL-660: Fix errors when querying a hive table with no data.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ce15e931
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ce15e931
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ce15e931
Branch: refs/heads/master
Commit: ce15e931ff0aa1969fa7c75d2ce184ca59458004
Parents: dd650cc
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Jun 10 11:51:41 2014 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Tue Jun 10 22:41:12 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/hive/HiveRecordReader.java | 43 ++++++++++++--------
.../exec/store/hive/HiveScanBatchCreator.java | 7 ++++
.../exec/store/hive/HiveTestDataGenerator.java | 2 +
.../apache/drill/jdbc/test/TestJdbcQuery.java | 13 +++---
.../apache/drill/jdbc/test/TestMetadataDDL.java | 2 +
5 files changed, 42 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
index 4361262..edd79e6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
@@ -108,6 +108,7 @@ public class HiveRecordReader implements RecordReader {
protected List<ValueVector> pVectors = Lists.newArrayList();
protected Object redoRecord;
List<Object> partitionValues = Lists.newArrayList();
+ protected boolean empty;
protected static final int TARGET_RECORD_COUNT = 4000;
@@ -117,6 +118,7 @@ public class HiveRecordReader implements RecordReader {
this.inputSplit = inputSplit;
this.context = context;
this.columns = columns;
+ this.empty = (inputSplit == null && partition == null);
init();
}
@@ -144,11 +146,9 @@ public class HiveRecordReader implements RecordReader {
}
job.setInputFormat(format.getClass());
- if (partition != null) {
- List<FieldSchema> partitionKeys = table.getPartitionKeys();
- for (FieldSchema field : partitionKeys) {
- partitionNames.add(field.getName());
- }
+ List<FieldSchema> partitionKeys = table.getPartitionKeys();
+ for (FieldSchema field : partitionKeys) {
+ partitionNames.add(field.getName());
}
try {
@@ -168,7 +168,7 @@ public class HiveRecordReader implements RecordReader {
for (SchemaPath field : columns) {
String columnName = field.getRootSegment().getPath(); //TODO?
if (!tableColumns.contains(columnName)) {
- if (partition != null && partitionNames.contains(columnName)) {
+ if (partitionNames.contains(columnName)) {
selectedPartitionNames.add(columnName);
} else {
throw new ExecutionSetupException(String.format("Column %s does not exist", columnName));
@@ -195,11 +195,11 @@ public class HiveRecordReader implements RecordReader {
selectedPartitionNames = partitionNames;
}
- if (partition != null) {
- for (int i = 0; i < table.getPartitionKeys().size(); i++) {
- FieldSchema field = table.getPartitionKeys().get(i);
- if (selectedPartitionNames.contains(field.getName())) {
- selectedPartitionTypes.add(field.getType());
+ for (int i = 0; i < table.getPartitionKeys().size(); i++) {
+ FieldSchema field = table.getPartitionKeys().get(i);
+ if (selectedPartitionNames.contains(field.getName())) {
+ selectedPartitionTypes.add(field.getType());
+ if (partition != null) {
partitionValues.add(convertPartitionType(field.getType(), partition.getValues().get(i)));
}
}
@@ -207,13 +207,16 @@ public class HiveRecordReader implements RecordReader {
} catch (SerDeException e) {
throw new ExecutionSetupException(e);
}
- try {
- reader = format.getRecordReader(inputSplit, job, Reporter.NULL);
- } catch (IOException e) {
- throw new ExecutionSetupException("Failed to get Recordreader", e);
+
+ if (!empty) {
+ try {
+ reader = format.getRecordReader(inputSplit, job, Reporter.NULL);
+ } catch (IOException e) {
+ throw new ExecutionSetupException("Failed to get Recordreader", e);
+ }
+ key = reader.createKey();
+ value = reader.createValue();
}
- key = reader.createKey();
- value = reader.createValue();
}
@Override
@@ -228,7 +231,7 @@ public class HiveRecordReader implements RecordReader {
}
for (int i = 0; i < selectedPartitionNames.size(); i++) {
String type = selectedPartitionTypes.get(i);
- MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(columnNames.get(i)), Types.getMajorTypeFromName(type));
+ MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(selectedPartitionNames.get(i)), Types.getMajorTypeFromName(type));
Class vvClass = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getDataMode());
pVectors.add(output.addField(field, vvClass));
}
@@ -439,6 +442,10 @@ public class HiveRecordReader implements RecordReader {
@Override
public int next() {
+ if (empty) {
+ return 0;
+ }
+
for (ValueVector vv : vectors) {
VectorAllocator.getAllocator(vv, 50).alloc(TARGET_RECORD_COUNT);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
index a0837bc..8914db2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java
@@ -66,6 +66,13 @@ public class HiveScanBatchCreator implements BatchCreator<HiveSubScan> {
}
}
}
+
+ // If there are no readers created (which is possible when the table is empty), create an empty RecordReader to
+ // output the schema
+ if (readers.size() == 0) {
+ readers.add(new HiveRecordReader(table, null, null, config.getColumns(), context));
+ }
+
return new ScanBatch(config, context, readers.iterator());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index f1565d9..5a511c0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -85,6 +85,8 @@ public class HiveTestDataGenerator {
"ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE");
executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE default.foodate", testDateDataFile));
+ // create a table with no data
+ executeQuery("CREATE TABLE IF NOT EXISTS default.empty_table(a INT, b STRING)");
ss.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index f486cc9..1b83148 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -50,7 +50,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
- public class TestJdbcQuery extends JdbcTest{
+public class TestJdbcQuery extends JdbcTest{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class);
@@ -77,12 +77,6 @@ import static org.junit.Assert.fail;
}
@Test
- @Ignore
- public void testHiveRead() throws Exception{
- testQuery("select * from hive.kv");
- }
-
- @Test
public void testHiveReadWithDb() throws Exception{
testQuery("select * from hive.`default`.kv");
testQuery("select key from hive.`default`.kv group by key");
@@ -95,6 +89,11 @@ import static org.junit.Assert.fail;
}
@Test
+ public void testQueryEmptyHiveTable() throws Exception {
+ testQuery("SELECT * FROM hive.`default`.empty_table");
+ }
+
+ @Test
@Ignore
public void testJsonQuery() throws Exception{
testQuery("select * from cp.`employee.json`");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 95af9f8..5299bb5 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -49,6 +49,7 @@ public class TestMetadataDDL extends TestJdbcQuery {
JdbcAssert.withFull("hive.default")
.sql("SHOW TABLES")
.returns(
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n"
);
@@ -69,6 +70,7 @@ public class TestMetadataDDL extends TestJdbcQuery {
JdbcAssert.withFull("dfs.tmp")
.sql("SHOW TABLES IN hive.`default`")
.returns(
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n");
}
[04/24] git commit: DRILL-720: Add secondary cast rules to be able to
implicitly cast to VARBINARY (from VARCHAR, INT, BIGINT, FLOAT4, FLOAT8)
Posted by ja...@apache.org.
DRILL-720: Add secondary cast rules to be able to implicitly cast to VARBINARY (from VARCHAR, INT, BIGINT, FLOAT4, FLOAT8)
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/564f9690
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/564f9690
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/564f9690
Branch: refs/heads/master
Commit: 564f9690c9f41b456751b4c3fcda4e547195b375
Parents: fc54e8e
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Jun 10 16:04:12 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 09:03:24 2014 -0700
----------------------------------------------------------------------
.../drill/exec/resolver/ResolverTypePrecedence.java | 12 +++++++++++-
.../org/apache/drill/exec/resolver/TypeCastRules.java | 1 +
.../java/org/apache/drill/jdbc/test/TestJdbcQuery.java | 10 +++++++++-
3 files changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/564f9690/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
index 9d83941..68a1643 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
@@ -125,8 +125,18 @@ public class ResolverTypePrecedence {
rule.add(MinorType.INTERVAL);
rule.add(MinorType.INTERVALYEAR);
rule.add(MinorType.INTERVALDAY);
-
secondaryImplicitCastRules.put(MinorType.VARCHAR, rule);
+
+ rule = new HashSet<>();
+
+ // Be able to implicitly cast to VARBINARY
+ rule.add(MinorType.INT);
+ rule.add(MinorType.BIGINT);
+ rule.add(MinorType.FLOAT4);
+ rule.add(MinorType.FLOAT8);
+ rule.add(MinorType.VARCHAR);
+ secondaryImplicitCastRules.put(MinorType.VARBINARY, rule);
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/564f9690/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
index 3c20555..5c5ff80 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
@@ -761,6 +761,7 @@ public class TypeCastRules {
rule.add(MinorType.FLOAT4);
rule.add(MinorType.FLOAT8);
rule.add(MinorType.BIT);
+ rule.add(MinorType.VARCHAR);
rule.add(MinorType.VARBINARY);
rule.add(MinorType.FIXEDBINARY);
rules.put(MinorType.VARBINARY, rule);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/564f9690/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 501927c..0412f00 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -414,5 +414,13 @@ public class TestJdbcQuery extends JdbcTest{
);
}
-
+ @Test
+ public void testLengthUTF8VarCharInput() throws Exception {
+ JdbcAssert.withNoDefaultSchema()
+ .sql("select length('Sheri', 'UTF8') as L_UTF8 " +
+ "from cp.`employee.json` where employee_id = 1")
+ .returns(
+ "L_UTF8=5\n"
+ );
+ }
}
[13/24] git commit: DRILL-963: Fix Hive test
[TestViews.testInfoSchemaWithView()] on Windows
Posted by ja...@apache.org.
DRILL-963: Fix Hive test [TestViews.testInfoSchemaWithView()] on Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2903ed3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2903ed3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2903ed3a
Branch: refs/heads/master
Commit: 2903ed3a0c129396fb50a5e60a79f632a06fc944
Parents: 4773b57
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue Jun 10 15:52:28 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 16:08:16 2014 -0700
----------------------------------------------------------------------
.../jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2903ed3a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
index 6591752..7dc9c30 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
@@ -30,6 +30,8 @@ import static org.junit.Assert.assertTrue;
/** Contains tests for creating/droping and using views in Drill. */
public class TestViews extends TestJdbcQuery {
+ private final static String NEW_LINE = System.getProperty("line.separator");
+
/** Helper test method for view tests */
private void testViewHelper(final String viewCreate, final String viewName,
final String viewQuery, final String queryResult) throws Exception{
@@ -265,7 +267,7 @@ public class TestViews extends TestJdbcQuery {
"WHERE TABLE_NAME = 'testview3'");
result = JdbcAssert.toString(resultSet).trim();
resultSet.close();
- expected = "TABLE_CATALOG=DRILL; TABLE_SCHEMA=dfs.tmp; TABLE_NAME=testview3; VIEW_DEFINITION=SELECT *\nFROM `hive`.`kv`";
+ expected = "TABLE_CATALOG=DRILL; TABLE_SCHEMA=dfs.tmp; TABLE_NAME=testview3; VIEW_DEFINITION=SELECT *"+NEW_LINE+"FROM `hive`.`kv`";
assertTrue(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected),
expected.equals(result));
[06/24] git commit: DRILL-502: Fix running sum type for average
function.
Posted by ja...@apache.org.
DRILL-502: Fix running sum type for average function.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b900be20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b900be20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b900be20
Branch: refs/heads/master
Commit: b900be20d666009b07c486253748e8ff3a5a23e6
Parents: ce15e93
Author: Mehant Baid <me...@gmail.com>
Authored: Mon Jun 9 23:57:45 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 09:03:24 2014 -0700
----------------------------------------------------------------------
exec/java-exec/src/main/codegen/data/AggrTypes2.tdd | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b900be20/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd b/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd
index 7c2643a..c6655af 100644
--- a/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd
+++ b/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd
@@ -21,10 +21,10 @@
{inputType: "BigInt", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Numeric"},
{inputType: "NullableInt", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Numeric"},
{inputType: "NullableBigInt", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Numeric"},
- {inputType: "Float4", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Numeric"},
- {inputType: "Float8", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Numeric"},
- {inputType: "NullableFloat4", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Numeric"},
- {inputType: "NullableFloat8", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Numeric"},
+ {inputType: "Float4", outputType: "Float8", sumRunningType: "Float8", countRunningType: "BigInt", major: "Numeric"},
+ {inputType: "Float8", outputType: "Float8", sumRunningType: "Float8", countRunningType: "BigInt", major: "Numeric"},
+ {inputType: "NullableFloat4", outputType: "Float8", sumRunningType: "Float8", countRunningType: "BigInt", major: "Numeric"},
+ {inputType: "NullableFloat8", outputType: "Float8", sumRunningType: "Float8", countRunningType: "BigInt", major: "Numeric"},
{inputType: "IntervalDay", outputType: "Interval", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Date"},
{inputType: "NullableIntervalDay", outputType: "Interval", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Date"},
{inputType: "IntervalYear", outputType: "Interval", sumRunningType: "BigInt", countRunningType: "BigInt", major: "Date"},
[19/24] git commit: DRILL-548: Re-enable tpch13
Posted by ja...@apache.org.
DRILL-548: Re-enable tpch13
I am no longer seeing the IOOB exception.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/7c1ee01c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/7c1ee01c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/7c1ee01c
Branch: refs/heads/master
Commit: 7c1ee01caa30758a88e78143c068ec126c8fb42a
Parents: 930bc0f
Author: Steven Phillips <sp...@maprtech.com>
Authored: Wed Jun 11 02:17:38 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:25:16 2014 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/drill/TestTpchDistributed.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7c1ee01c/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
index 4b1cf2a..c388b3b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributed.java
@@ -92,7 +92,6 @@ public class TestTpchDistributed extends BaseTestQuery{
}
@Test
- @Ignore // flapping ioob
public void tpch13() throws Exception{
testDistributed("queries/tpch/13.sql");
}
[21/24] git commit: DRILL-909: Handle OOM in UnlimitedRawBatchBuffer
Posted by ja...@apache.org.
DRILL-909: Handle OOM in UnlimitedRawBatchBuffer
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d9a2f1c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d9a2f1c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d9a2f1c9
Branch: refs/heads/master
Commit: d9a2f1c9ac45ff31742282b6319891bec08de745
Parents: 7c1ee01
Author: Steven Phillips <sp...@maprtech.com>
Authored: Wed Jun 11 18:57:08 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:25:16 2014 -0700
----------------------------------------------------------------------
.../physical/impl/xsort/ExternalSortBatch.java | 2 +-
.../work/batch/UnlimitedRawBatchBuffer.java | 23 ++++++++++++++++++++
.../impl/xsort/TestSimpleExternalSort.java | 1 -
3 files changed, 24 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d9a2f1c9/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index d4c0b25..5582742 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -243,7 +243,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
// logger.debug("Took {} us to sort {} records", t, count);
break;
case OUT_OF_MEMORY:
- mergeAndSpill();
+ if (batchesSinceLastSpill > 2) mergeAndSpill();
batchesSinceLastSpill = 0;
break;
default:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d9a2f1c9/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
index a726a82..d14e50c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/UnlimitedRawBatchBuffer.java
@@ -34,6 +34,7 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
private final int softlimit;
private final int startlimit;
private final AtomicBoolean overlimit = new AtomicBoolean(false);
+ private final AtomicBoolean outOfMemory = new AtomicBoolean(false);
private final ReadController readController;
private final boolean multiFragment;
@@ -57,6 +58,15 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
@Override
public void enqueue(RawFragmentBatch batch) {
+ if (batch.getHeader().getIsOutOfMemory()) {
+ logger.debug("Setting autoread false");
+ readController.setAutoRead(false);
+ if (!outOfMemory.get() && !buffer.peekFirst().getHeader().getIsOutOfMemory()) {
+ buffer.addFirst(batch);
+ }
+ outOfMemory.set(true);
+ return;
+ }
buffer.add(batch);
if(buffer.size() == softlimit){
overlimit.set(true);
@@ -85,6 +95,12 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
@Override
public RawFragmentBatch getNext(){
+ if (outOfMemory.get() && buffer.size() < 10) {
+ logger.debug("Setting autoread true");
+ outOfMemory.set(false);
+ readController.setAutoRead(true);
+ }
+
RawFragmentBatch b = null;
b = buffer.poll();
@@ -98,6 +114,13 @@ public class UnlimitedRawBatchBuffer implements RawBatchBuffer{
}
}
+ if (b != null && b.getHeader().getIsOutOfMemory()) {
+ outOfMemory.set(true);
+ readController.setAutoRead(false);
+ return b;
+ }
+
+
// if we are in the overlimit condition and aren't finished, check if we've passed the start limit. If so, turn off the overlimit condition and set auto read to true (start reading from socket again).
if(!finished && overlimit.get()){
if(buffer.size() == startlimit){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d9a2f1c9/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
index b263d2f..47f0342 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
@@ -178,7 +178,6 @@ public class TestSimpleExternalSort extends BaseTestQuery {
}
@Test
- @Ignore
public void outOfMemoryExternalSort() throws Throwable{
RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
[16/24] git commit: DRILL-946: Fix minor bug in casting decimal to
double
Posted by ja...@apache.org.
DRILL-946: Fix minor bug in casting decimal to double
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/734f9a8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/734f9a8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/734f9a8d
Branch: refs/heads/master
Commit: 734f9a8dc6f24c9121877fa841b4815dc80fe346
Parents: 60a429f
Author: Mehant Baid <me...@gmail.com>
Authored: Wed Jun 11 19:10:06 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:13:45 2014 -0700
----------------------------------------------------------------------
.../main/codegen/templates/Decimal/CastDecimalFloat.java | 2 +-
.../org/apache/drill/jdbc/test/TestFunctionsQuery.java | 11 +++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/734f9a8d/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java
index b259cb3..92cd576 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastDecimalFloat.java
@@ -52,7 +52,7 @@ public class Cast${type.from}${type.to} implements DrillSimpleFunc {
public void eval() {
// Divide the decimal with the scale to get the floating point value
- out.value = (${type.javatype}) (org.apache.drill.common.util.DecimalUtility.adjustScaleDivide(in.value, (int) in.scale));
+ out.value = ((${type.javatype}) (in.value)) / (org.apache.drill.common.util.DecimalUtility.getPowerOfTen((int) in.scale));
}
}
<#elseif type.major == "DecimalComplexFloat" || type.major == "DecimalComplexDouble"> <#-- Cast function template for conversion from Decimal9, Decimal18 to Float4 -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/734f9a8d/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
index 89224d4..60b8f82 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestFunctionsQuery.java
@@ -512,4 +512,15 @@ public class TestFunctionsQuery {
"TS_EXT=20.123; " +
"TM_EXT=30.303\n");
}
+
+ @Test
+ public void testCastDecimalDouble() throws Exception {
+ String query = "select cast((cast('1.0001' as decimal(18, 9))) as double) DECIMAL_DOUBLE_CAST " +
+ "from cp.`employee.json` where employee_id = 1";
+
+ JdbcAssert.withNoDefaultSchema()
+ .sql(query)
+ .returns(
+ "DECIMAL_DOUBLE_CAST=1.0001\n");
+ }
}
[23/24] git commit: Fix regression caused by DRILL-521
Posted by ja...@apache.org.
Fix regression caused by DRILL-521
DRILL-521 moved to metastore.api.Table from ql.metadata.Table which caused
failures when reading Hive tables that have schema given in SerDe properties
such as avro. Revert to using ql.metadata.Table.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/fcd5988d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/fcd5988d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/fcd5988d
Branch: refs/heads/master
Commit: fcd5988de7edcb1d2b72377b1637e7c7b88a3d77
Parents: 21d9fb7
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Jun 11 23:05:52 2014 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Wed Jun 11 23:05:52 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/store/hive/schema/DrillHiveTable.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fcd5988d/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
index 547c0bb..949fa06 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -26,7 +26,7 @@ import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.store.hive.HiveReadEntry;
import org.apache.drill.exec.store.hive.HiveStoragePlugin;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -45,7 +45,7 @@ public class DrillHiveTable extends DrillTable{
public DrillHiveTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) {
super(storageEngineName, plugin, readEntry);
- this.hiveTable = readEntry.getTable();
+ this.hiveTable = new Table(readEntry.getTable());
}
@Override
@@ -53,7 +53,7 @@ public class DrillHiveTable extends DrillTable{
List<RelDataType> typeList = Lists.newArrayList();
List<String> fieldNameList = Lists.newArrayList();
- List<FieldSchema> hiveFields = hiveTable.getSd().getCols();
+ List<FieldSchema> hiveFields = hiveTable.getCols();
for(FieldSchema hiveField : hiveFields) {
fieldNameList.add(hiveField.getName());
typeList.add(getNullableRelDataTypeFromHiveType(
[03/24] git commit: DRILL-944: Fix overflow while adding Time +
Interval data type
Posted by ja...@apache.org.
DRILL-944: Fix overflow while adding Time + Interval data type
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/72084e32
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/72084e32
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/72084e32
Branch: refs/heads/master
Commit: 72084e32c08d3136de4d6940c0965c0c8ed6b04a
Parents: 564f969
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Jun 10 17:40:42 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 09:03:24 2014 -0700
----------------------------------------------------------------------
.../DateIntervalArithmeticFunctions.java | 23 +++++++-------------
.../apache/drill/jdbc/test/TestJdbcQuery.java | 14 ++++++++++--
2 files changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/72084e32/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
index 4fbfdf9..634e41a 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/DateIntervalArithmeticFunctions.java
@@ -176,24 +176,17 @@ import org.apache.drill.exec.expr.fn.impl.DateUtility;
public class ${datetype}${intervaltype}Functions {
<#macro timeIntervalArithmeticBlock left right temp op output intervaltype>
- <#if intervaltype == "Interval">
- if (${right}.months != 0 || ${right}.days != 0) {
- throw new UnsupportedOperationException("Cannot add interval type with months or days to TIME");
- }
- ${output} = ${left}.value ${op} ${right}.milliSeconds;
- <#elseif intervaltype == "IntervalYear">
- if (1 == 1) {
- throw new UnsupportedOperationException("Cannot add IntervalYear to TIME");
- }
- <#elseif intervaltype == "IntervalDay">
- if (${right}.days != 0) {
- throw new UnsupportedOperationException("Cannot add days to TIME");
- }
- ${output} = ${left}.value ${op} ${right}.milliSeconds;
- <#elseif intervaltype == "Int" || intervaltype == "BigInt">
+ <#if intervaltype == "Int" || intervaltype == "BigInt">
if (1 == 1) {
throw new UnsupportedOperationException("Cannot add integer to TIME, cast it to specific interval");
}
+ <#elseif intervaltype == "IntervalYear">
+ // Needn't add anything to time from interval year data type. Output is same as input
+ ${output} = ${left}.value;
+ <#else>
+ ${output} = ${left}.value ${op} ${right}.milliSeconds;
+ // Wrap around 24 hour clock if we exceeded it while adding the time component
+ ${output} = ${output} % org.apache.drill.exec.expr.fn.impl.DateUtility.daysToStandardMillis;
</#if>
</#macro>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/72084e32/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 0412f00..2a69469 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -409,8 +409,8 @@ public class TestJdbcQuery extends JdbcTest{
"from cp.`employee.json` limit 1")
.returns(
"LEFT_STR=ab; " +
- "RIGHT_STR=ef; " +
- "REPLACE_STR=zzcdef\n"
+ "RIGHT_STR=ef; " +
+ "REPLACE_STR=zzcdef\n"
);
}
@@ -421,6 +421,16 @@ public class TestJdbcQuery extends JdbcTest{
"from cp.`employee.json` where employee_id = 1")
.returns(
"L_UTF8=5\n"
+ );
+ }
+
+ @Test
+ public void testTimeIntervalAddOverflow() throws Exception {
+ JdbcAssert.withNoDefaultSchema()
+ .sql("select extract(hour from (interval '10 20' day to hour + time '10:00:00')) as TIME_INT_ADD " +
+ "from cp.`employee.json` where employee_id = 1")
+ .returns(
+ "TIME_INT_ADD=6\n"
);
}
}
[17/24] git commit: DRILL-969: Improvements to ValueVector allocations
Posted by ja...@apache.org.
DRILL-969: Improvements to ValueVector allocations
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2712c3c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2712c3c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2712c3c3
Branch: refs/heads/master
Commit: 2712c3c35f8fc1e5bd7443c7e40d6b76fad9d49d
Parents: 734f9a8
Author: Steven Phillips <sp...@maprtech.com>
Authored: Mon Jun 9 16:39:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:24:55 2014 -0700
----------------------------------------------------------------------
.../codegen/templates/FixedValueVectors.java | 15 ++++---
.../codegen/templates/RepeatedValueVectors.java | 9 ++--
.../templates/VariableLengthVectors.java | 23 +++++-----
.../impl/svremover/RemovingRecordBatch.java | 46 +++++++++++++++-----
.../exec/record/AbstractSingleRecordBatch.java | 7 +++
.../exec/store/text/DrillTextRecordReader.java | 13 ++++--
6 files changed, 78 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2712c3c3/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
index af31f64..a83ec97 100644
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -53,7 +53,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
- private int allocationValueCount = 4000;
+ private int allocationValueCount = 4096;
private int allocationMonitor = 0;
public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
@@ -81,11 +81,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
public boolean allocateNewSafe() {
clear();
- if (allocationMonitor > 5) {
- allocationValueCount = Math.max(2, (int) (allocationValueCount * 0.9));
+ if (allocationMonitor > 10) {
+ allocationValueCount = Math.max(8, (int) (allocationValueCount / 2));
allocationMonitor = 0;
- } else if (allocationMonitor < -5) {
- allocationValueCount = (int) (allocationValueCount * 1.1);
+ } else if (allocationMonitor < -2) {
+ allocationValueCount = (int) (allocationValueCount * 2);
allocationMonitor = 0;
}
this.data = allocator.buffer(allocationValueCount * ${type.width});
@@ -102,6 +102,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
clear();
this.data = allocator.buffer(valueCount * ${type.width});
this.data.readerIndex(0);
+ this.allocationValueCount = valueCount;
}
/**
@@ -815,8 +816,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
int currentValueCapacity = getValueCapacity();
${minor.class}Vector.this.valueCount = valueCount;
int idx = (${type.width} * valueCount);
- if (((float) currentValueCapacity) / idx > 1.1) {
+ if (valueCount > 0 && currentValueCapacity > idx * 2) {
allocationMonitor++;
+ } else if (allocationMonitor > 0) {
+ allocationMonitor--;
}
data.writerIndex(idx);
if (data instanceof AccountingByteBuf) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2712c3c3/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 48efc16..7bf84f2 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -98,6 +98,8 @@ package org.apache.drill.exec.vector;
int endValue = offsets.getAccessor().get(startIndex + length);
values.splitAndTransferTo(startValue, endValue - startValue, target.values);
offsets.splitAndTransferTo(startIndex, length, target.offsets);
+ target.parentValueCount = parentValueCount;
+ target.childValueCount = childValueCount;
sliceOffset = startIndex;
}
@@ -369,8 +371,7 @@ package org.apache.drill.exec.vector;
if(getValueCapacity() <= index){
return false;
}
- offsets.getMutator().set(index+1, offsets.getAccessor().get(index));
- return true;
+ return offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
}
/**
@@ -392,7 +393,9 @@ package org.apache.drill.exec.vector;
}
public boolean addSafe(int index, byte[] bytes, int start, int length) {
- if(offsets.getValueCapacity() <= index+1) return false;
+ if(offsets.getValueCapacity() <= index+1) {
+ return false;
+ }
int nextOffset = offsets.getAccessor().get(index+1);
boolean b1 = values.getMutator().setSafe(nextOffset, bytes, start, length);
boolean b2 = offsets.getMutator().setSafe(index+1, nextOffset+1);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2712c3c3/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index 6a2dfd3..22a668d 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -49,7 +49,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
- private int allocationTotalByteCount = 40000;
+ private int allocationTotalByteCount = 32768;
private int allocationMonitor = 0;
public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
@@ -229,11 +229,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
@Override
public boolean allocateNewSafe() {
clear();
- if (allocationMonitor > 5) {
- allocationTotalByteCount = Math.max(1, (int) (allocationTotalByteCount * 0.9));
+ if (allocationMonitor > 10) {
+ allocationTotalByteCount = Math.max(8, (int) (allocationTotalByteCount / 2));
allocationMonitor = 0;
- } else if (allocationMonitor < -5) {
- allocationTotalByteCount = (int) (allocationTotalByteCount * 1.1);
+ } else if (allocationMonitor < -2) {
+ allocationTotalByteCount = (int) (allocationTotalByteCount * 2);
allocationMonitor = 0;
}
data = allocator.buffer(allocationTotalByteCount);
@@ -254,6 +254,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
assert totalBytes >= 0;
data = allocator.buffer(totalBytes);
data.readerIndex(0);
+ allocationTotalByteCount = totalBytes;
offsetVector.allocateNew(valueCount+1);
offsetVector.zeroVector();
}
@@ -359,7 +360,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
public boolean setSafe(int index, byte[] bytes) {
assert index >= 0;
- if(index >= getValueCapacity()) return false;
int currentOffset = offsetVector.getAccessor().get(index);
if (data.capacity() < currentOffset + bytes.length) {
@@ -391,7 +391,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
public boolean setSafe(int index, byte[] bytes, int start, int length) {
assert index >= 0;
- if(index >= getValueCapacity()) return false;
int currentOffset = offsetVector.getAccessor().get(index);
@@ -409,8 +408,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
public boolean setSafe(int index, Nullable${minor.class}Holder holder){
assert holder.isSet == 1;
- if(index >= getValueCapacity()) return false;
-
+
int start = holder.start;
int end = holder.end;
int len = end - start;
@@ -433,8 +431,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
}
public boolean setSafe(int index, ${minor.class}Holder holder){
- if(index >= getValueCapacity()) return false;
-
+
int start = holder.start;
int end = holder.end;
int len = end - start;
@@ -483,8 +480,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
${minor.class}Vector.this.valueCount = valueCount;
int idx = offsetVector.getAccessor().get(valueCount);
data.writerIndex(idx);
- if (((float) currentByteCapacity) / idx > 1.1) {
+ if (valueCount > 0 && currentByteCapacity > idx * 2) {
allocationMonitor++;
+ } else if (allocationMonitor > 0) {
+ allocationMonitor--;
}
if (data instanceof AccountingByteBuf) {
data.capacity(idx);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2712c3c3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
index f3388dc..77582c3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
@@ -49,6 +49,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
private int recordCount;
private boolean hasRemainder;
private int remainderIndex;
+ private boolean first;
public RemovingRecordBatch(SelectionVectorRemover popConfig, FragmentContext context, RecordBatch incoming) throws OutOfMemoryException {
super(popConfig, context, incoming);
@@ -93,9 +94,10 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
@Override
protected void doWork() {
- recordCount = incoming.getRecordCount();
- int copiedRecords = copier.copyRecords(0, recordCount);
- if (copiedRecords < recordCount) {
+ int incomingRecordCount = incoming.getRecordCount();
+ int copiedRecords = copier.copyRecords(0, incomingRecordCount);
+
+ if (copiedRecords < incomingRecordCount) {
for(VectorWrapper<?> v : container){
ValueVector.Mutator m = v.getValueVector().getMutator();
m.setValueCount(copiedRecords);
@@ -104,6 +106,7 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
remainderIndex = copiedRecords;
this.recordCount = remainderIndex;
} else {
+ recordCount = copiedRecords;
for(VectorWrapper<?> v : container){
ValueVector.Mutator m = v.getValueVector().getMutator();
m.setValueCount(recordCount);
@@ -118,16 +121,37 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
}
}
- logger.debug(String.format("doWork(): %s records copied for out of %s, remaining: %s, incoming schema %s ",
+ assert recordCount >= copiedRecords;
+ logger.debug("doWork(): {} records copied out of {}, remaining: {}, incoming schema {} ",
copiedRecords,
- incoming.getRecordCount(),
- incoming.getRecordCount() - remainderIndex,
- incoming.getSchema()));
+ incomingRecordCount,
+ incomingRecordCount - remainderIndex,
+ incoming.getSchema());
}
private void handleRemainder() {
+ int recordCount = incoming.getRecordCount();
int remainingRecordCount = incoming.getRecordCount() - remainderIndex;
- int copiedRecords = copier.copyRecords(remainderIndex, remainingRecordCount);
+ int copiedRecords;
+ while((copiedRecords = copier.copyRecords(remainderIndex, remainingRecordCount)) == 0) {
+ logger.debug("Copied zero records. Retrying");
+ container.zeroVectors();
+ }
+
+ /*
+ StringBuilder builder = new StringBuilder();
+ for (VectorWrapper w : container) {
+ builder.append(w.getField().getPath());
+ builder.append(" Value capacity: ");
+ builder.append(w.getValueVector().getValueCapacity());
+ if (w.getValueVector() instanceof VariableWidthVector) {
+ builder.append(" Byte capacity: ");
+ builder.append(((VariableWidthVector) w.getValueVector()).getByteCapacity());
+ builder.append("\n");
+ }
+ }
+ logger.debug(builder.toString());
+ */
if (copiedRecords < remainingRecordCount) {
for(VectorWrapper<?> v : container){
@@ -150,10 +174,10 @@ public class RemovingRecordBatch extends AbstractSingleRecordBatch<SelectionVect
remainderIndex = 0;
hasRemainder = false;
}
- if(logger.isDebugEnabled()) logger.debug(String.format("handleRemainder(): %s records copied for out of %s, remaining: %s, incoming schema %s ",
+ logger.debug(String.format("handleRemainder(): %s records copied out of %s, remaining: %s, incoming schema %s ",
copiedRecords,
- incoming.getRecordCount(),
- incoming.getRecordCount() - remainderIndex,
+ recordCount,
+ recordCount - remainderIndex,
incoming.getSchema()));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2712c3c3/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
index c5fdaeb..9473945 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
@@ -44,6 +44,13 @@ public abstract class AbstractSingleRecordBatch<T extends PhysicalOperator> exte
public IterOutcome innerNext() {
IterOutcome upstream = next(incoming);
if(first && upstream == IterOutcome.OK) upstream = IterOutcome.OK_NEW_SCHEMA;
+ if (!first && upstream == IterOutcome.OK && incoming.getRecordCount() == 0) {
+ do {
+ for (VectorWrapper w : incoming) {
+ w.clear();
+ }
+ } while ((upstream = next(incoming)) == IterOutcome.OK && incoming.getRecordCount() == 0);
+ }
first = false;
switch(upstream){
case NONE:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2712c3c3/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
index 5c3d381..20f458f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
@@ -63,6 +63,7 @@ public class DrillTextRecordReader implements RecordReader {
private Text value;
private int numCols = 0;
private boolean redoRecord = false;
+ private boolean first = true;
public DrillTextRecordReader(FileSplit split, FragmentContext context, char delimiter, List<SchemaPath> columns) {
this.context = context;
@@ -106,10 +107,12 @@ public class DrillTextRecordReader implements RecordReader {
@Override
public int next() {
- AllocationHelper.allocate(vector, targetRecordCount, 50);
+ logger.debug("vector value capacity {}", vector.getValueCapacity());
+ logger.debug("vector byte capacity {}", vector.getByteCapacity());
+ int batchSize = 0;
try {
int recordCount = 0;
- while (redoRecord || (recordCount < targetRecordCount && reader.next(key, value))) {
+ while (redoRecord || (batchSize < 200*1000 && reader.next(key, value))) {
redoRecord = false;
int start;
int end = -1;
@@ -126,9 +129,10 @@ public class DrillTextRecordReader implements RecordReader {
end = value.getLength();
}
if (numCols > 0 && i++ < columnIds.get(p)) {
- if (!vector.getMutator().addSafe(recordCount, value.getBytes(), start + 1, start + 1)) {
+ if (!vector.getMutator().addSafe(recordCount, value.getBytes(), start + 1, 0)) {
redoRecord = true;
vector.getMutator().setValueCount(recordCount);
+ logger.debug("text scan batch size {}", batchSize);
return recordCount;
}
continue;
@@ -137,8 +141,10 @@ public class DrillTextRecordReader implements RecordReader {
if (!vector.getMutator().addSafe(recordCount, value.getBytes(), start + 1, end - start - 1)) {
redoRecord = true;
vector.getMutator().setValueCount(recordCount);
+ logger.debug("text scan batch size {}", batchSize);
return recordCount;
}
+ batchSize += end - start;
}
recordCount++;
}
@@ -146,6 +152,7 @@ public class DrillTextRecordReader implements RecordReader {
v.getMutator().setValueCount(recordCount);
}
vector.getMutator().setValueCount(recordCount);
+ logger.debug("text scan batch size {}", batchSize);
return recordCount;
} catch (IOException e) {
cleanup();
[24/24] git commit: Fix execution of tests that are run multiple
times.
Posted by ja...@apache.org.
Fix execution of tests that are run multiple times.
Due to separation of some test in TestJdbcQuery into TestViews and TestMetadataDDL
which are derived from TestJdbcQuery, all tests in TestJdbcQuery are run thrice.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4198a17a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4198a17a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4198a17a
Branch: refs/heads/master
Commit: 4198a17a8b8ba9fd109e7f8a55c66d466a895c18
Parents: fcd5988
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Jun 11 23:41:26 2014 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Wed Jun 11 23:41:26 2014 -0700
----------------------------------------------------------------------
.../drill/jdbc/test/JdbcTestQueryBase.java | 98 ++++++++++++++++++++
.../apache/drill/jdbc/test/TestJdbcQuery.java | 61 +-----------
.../apache/drill/jdbc/test/TestMetadataDDL.java | 2 +-
.../org/apache/drill/jdbc/test/TestViews.java | 2 +-
4 files changed, 101 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4198a17a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
new file mode 100644
index 0000000..259fa8c
--- /dev/null
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.jdbc.test;
+
+import com.google.common.base.Stopwatch;
+import org.apache.commons.io.FileUtils;
+import org.apache.drill.common.util.TestTools;
+import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
+import org.apache.drill.jdbc.Driver;
+import org.apache.drill.jdbc.JdbcTest;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestRule;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.Statement;
+import java.util.concurrent.TimeUnit;
+
+public class JdbcTestQueryBase extends JdbcTest {
+ // Set a timeout unless we're debugging.
+ @Rule
+ public TestRule TIMEOUT = TestTools.getTimeoutRule(20000);
+
+ protected static final String WORKING_PATH;
+ static{
+ Driver.load();
+ WORKING_PATH = Paths.get("").toAbsolutePath().toString();
+
+ }
+
+ @BeforeClass
+ public static void generateHive() throws Exception{
+ new HiveTestDataGenerator().generateTestData();
+
+ // delete tmp workspace directory
+ File f = new File("/tmp/drilltest");
+ if(f.exists()){
+ FileUtils.cleanDirectory(f);
+ FileUtils.forceDelete(f);
+ }
+ }
+
+ protected void testQuery(String sql) throws Exception{
+ boolean success = false;
+ try (Connection c = DriverManager.getConnection("jdbc:drill:zk=local", null);) {
+ for (int x = 0; x < 1; x++) {
+ Stopwatch watch = new Stopwatch().start();
+ Statement s = c.createStatement();
+ ResultSet r = s.executeQuery(sql);
+ boolean first = true;
+ while (r.next()) {
+ ResultSetMetaData md = r.getMetaData();
+ if (first == true) {
+ for (int i = 1; i <= md.getColumnCount(); i++) {
+ System.out.print(md.getColumnName(i));
+ System.out.print('\t');
+ }
+ System.out.println();
+ first = false;
+ }
+
+ for (int i = 1; i <= md.getColumnCount(); i++) {
+ System.out.print(r.getObject(i));
+ System.out.print('\t');
+ }
+ System.out.println();
+ }
+
+ System.out.println(String.format("Query completed in %d millis.", watch.elapsed(TimeUnit.MILLISECONDS)));
+ }
+
+ System.out.println("\n\n\n");
+ success = true;
+ }finally{
+ if(!success) Thread.sleep(2000);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4198a17a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 932f207..311b3a5 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -50,32 +50,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-public class TestJdbcQuery extends JdbcTest{
+public class TestJdbcQuery extends JdbcTestQueryBase{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class);
-
- // Set a timeout unless we're debugging.
- @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(20000);
-
- protected static final String WORKING_PATH;
- static{
- Driver.load();
- WORKING_PATH = Paths.get("").toAbsolutePath().toString();
-
- }
-
- @BeforeClass
- public static void generateHive() throws Exception{
- new HiveTestDataGenerator().generateTestData();
-
- // delete tmp workspace directory
- File f = new File("/tmp/drilltest");
- if(f.exists()){
- FileUtils.cleanDirectory(f);
- FileUtils.forceDelete(f);
- }
- }
-
@Test
public void testHiveReadWithDb() throws Exception{
testQuery("select * from hive.`default`.kv");
@@ -146,42 +123,6 @@ public class TestJdbcQuery extends JdbcTest{
testQuery(String.format("SELECT unknownColumn FROM dfs.`%s/../../sample-data/region.parquet`", WORKING_PATH));
}
- protected void testQuery(String sql) throws Exception{
- boolean success = false;
- try (Connection c = DriverManager.getConnection("jdbc:drill:zk=local", null);) {
- for (int x = 0; x < 1; x++) {
- Stopwatch watch = new Stopwatch().start();
- Statement s = c.createStatement();
- ResultSet r = s.executeQuery(sql);
- boolean first = true;
- while (r.next()) {
- ResultSetMetaData md = r.getMetaData();
- if (first == true) {
- for (int i = 1; i <= md.getColumnCount(); i++) {
- System.out.print(md.getColumnName(i));
- System.out.print('\t');
- }
- System.out.println();
- first = false;
- }
-
- for (int i = 1; i <= md.getColumnCount(); i++) {
- System.out.print(r.getObject(i));
- System.out.print('\t');
- }
- System.out.println();
- }
-
- System.out.println(String.format("Query completed in %d millis.", watch.elapsed(TimeUnit.MILLISECONDS)));
- }
-
- System.out.println("\n\n\n");
- success = true;
- }finally{
- if(!success) Thread.sleep(2000);
- }
- }
-
@Test
public void testLikeNotLike() throws Exception{
JdbcAssert.withNoDefaultSchema()
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4198a17a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 123c160..bb50e40 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue;
* -- USE schema
* -- SHOW FILES
*/
-public class TestMetadataDDL extends TestJdbcQuery {
+public class TestMetadataDDL extends JdbcTestQueryBase {
@Test
public void testInfoSchema() throws Exception{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4198a17a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
index 93626d5..976c5fa 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
@@ -28,7 +28,7 @@ import java.sql.Statement;
import static org.junit.Assert.assertTrue;
/** Contains tests for creating/droping and using views in Drill. */
-public class TestViews extends TestJdbcQuery {
+public class TestViews extends JdbcTestQueryBase {
private final static String NEW_LINE = System.getProperty("line.separator");
[18/24] git commit: DRILL-863: Sort column IDs in text reader
Posted by ja...@apache.org.
DRILL-863: Sort column IDs in text reader
Without this, if the columns passed to the reader are not in order, the reader does not read them correctly.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/930bc0f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/930bc0f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/930bc0f0
Branch: refs/heads/master
Commit: 930bc0f0bf06824c43563d8f1ed18b27e1f26c96
Parents: 2712c3c
Author: Steven Phillips <sp...@maprtech.com>
Authored: Wed Jun 11 02:14:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:25:15 2014 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/store/text/DrillTextRecordReader.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/930bc0f0/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
index 20f458f..b5b5b3c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.store.text;
import java.io.IOException;
+import java.util.Collections;
import java.util.List;
import org.apache.drill.common.exceptions.DrillRuntimeException;
@@ -79,6 +80,7 @@ public class DrillTextRecordReader implements RecordReader {
columnIds.add(index);
}
}
+ Collections.sort(columnIds);
}
targetRecordCount = context.getConfig().getInt(ExecConstants.TEXT_LINE_READER_BATCH_SIZE);
numCols = columnIds.size();
[15/24] git commit: DRILL-858: Fix bugs in handling Hive views in
INFORMATION_SCHEMA generation.
Posted by ja...@apache.org.
DRILL-858: Fix bugs in handling Hive views in INFORMATION_SCHEMA generation.
Also:
- Added a new interface DrillViewInfoProvider which provides view related info to
components such as InfoSchema. Drill or Hive view table implement this interface.
- Throw UnsupportedOperationException if any queries use Hive views
as querying Hive views is not supported in current version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/60a429fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/60a429fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/60a429fb
Branch: refs/heads/master
Commit: 60a429fb7815c655234055ded237dbda1af2d769
Parents: b328d7b
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Jun 11 17:13:59 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:12:44 2014 -0700
----------------------------------------------------------------------
.../planner/logical/DrillViewInfoProvider.java | 28 ++++++++++++++
.../exec/planner/logical/DrillViewTable.java | 3 +-
.../drill/exec/store/hive/HiveReadEntry.java | 9 +++++
.../exec/store/hive/HiveStoragePlugin.java | 5 +++
.../exec/store/hive/schema/DrillHiveTable.java | 2 +-
.../store/hive/schema/DrillHiveViewTable.java | 40 ++++++++++++++++++++
.../store/hive/schema/HiveSchemaFactory.java | 7 +++-
.../exec/store/ischema/RecordGenerator.java | 4 +-
.../exec/store/hive/HiveTestDataGenerator.java | 3 ++
.../apache/drill/jdbc/test/TestMetadataDDL.java | 2 +
.../org/apache/drill/jdbc/test/TestViews.java | 8 ++++
11 files changed, 106 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java
new file mode 100644
index 0000000..50e1d8f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewInfoProvider.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+/**
+ * Interface used by Drill components such as InformationSchema generator to get view info.
+ * All view tables need to implement this interface.
+ */
+public interface DrillViewInfoProvider {
+
+ /** Get the query part of the view. */
+ String getViewSql();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
index aaf32ad..19fd7ba 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillViewTable.java
@@ -32,7 +32,7 @@ import org.eigenbase.relopt.RelOptUtil;
import org.eigenbase.reltype.RelDataType;
import org.eigenbase.reltype.RelDataTypeFactory;
-public class DrillViewTable implements TranslatableTable{
+public class DrillViewTable implements TranslatableTable, DrillViewInfoProvider {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillViewTable.class);
private final View view;
@@ -71,6 +71,7 @@ public class DrillViewTable implements TranslatableTable{
return TableType.VIEW;
}
+ @Override
public String getViewSql() {
return view.getSql();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
index f330a1e..855b05f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.store.hive;
import java.util.List;
+import net.hydromatic.optiq.Schema.TableType;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.base.Size;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -60,5 +61,13 @@ public class HiveReadEntry {
return partitionsUnwrapped;
}
+ @JsonIgnore
+ public TableType getJdbcTableType() {
+ if (table.getTable().getTableType().equals(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW.toString())) {
+ return TableType.VIEW;
+ }
+
+ return TableType.TABLE;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
index 6d5f6d2..c5a6e2c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.List;
import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.Schema.TableType;
import net.hydromatic.optiq.SchemaPlus;
import org.apache.drill.common.JSONOptions;
@@ -67,6 +68,10 @@ public class HiveStoragePlugin extends AbstractStoragePlugin {
public HiveScan getPhysicalScan(JSONOptions selection, List<SchemaPath> columns) throws IOException {
HiveReadEntry hiveReadEntry = selection.getListWith(new ObjectMapper(), new TypeReference<HiveReadEntry>(){});
try {
+ if (hiveReadEntry.getJdbcTableType() == TableType.VIEW) {
+ throw new UnsupportedOperationException("Querying Hive views from Drill is not supported in current version.");
+ }
+
return new HiveScan(hiveReadEntry, this, null);
} catch (ExecutionSetupException e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
index d1a5659..547c0bb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -41,7 +41,7 @@ import org.eigenbase.sql.type.SqlTypeName;
public class DrillHiveTable extends DrillTable{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHiveTable.class);
- private final Table hiveTable;
+ protected final Table hiveTable;
public DrillHiveTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) {
super(storageEngineName, plugin, readEntry);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java
new file mode 100644
index 0000000..b575972
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveViewTable.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive.schema;
+
+import net.hydromatic.optiq.Schema.TableType;
+import org.apache.drill.exec.planner.logical.DrillViewInfoProvider;
+import org.apache.drill.exec.store.hive.HiveReadEntry;
+import org.apache.drill.exec.store.hive.HiveStoragePlugin;
+
+public class DrillHiveViewTable extends DrillHiveTable implements DrillViewInfoProvider {
+
+ public DrillHiveViewTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) {
+ super(storageEngineName, plugin, readEntry);
+ }
+
+ @Override
+ public TableType getJdbcTableType() {
+ return TableType.VIEW;
+ }
+
+ @Override
+ public String getViewSql() {
+ return hiveTable.getViewExpandedText();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
index fb672d4..7e6b92b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
@@ -258,7 +258,12 @@ public class HiveSchemaFactory implements SchemaFactory {
DrillTable getDrillTable(String dbName, String t){
HiveReadEntry entry = getSelectionBaseOnName(dbName, t);
if(entry == null) return null;
- return new DrillHiveTable(schemaName, plugin, entry);
+
+ if (entry.getJdbcTableType() == TableType.VIEW) {
+ return new DrillHiveViewTable(schemaName, plugin, entry);
+ } else {
+ return new DrillHiveTable(schemaName, plugin, entry);
+ }
}
HiveReadEntry getSelectionBaseOnName(String dbName, String t) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java
index a8792fe..df67da1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/RecordGenerator.java
@@ -23,7 +23,7 @@ import net.hydromatic.optiq.Schema.TableType;
import net.hydromatic.optiq.SchemaPlus;
import net.hydromatic.optiq.Table;
import net.hydromatic.optiq.jdbc.JavaTypeFactoryImpl;
-import org.apache.drill.exec.planner.logical.DrillViewTable;
+import org.apache.drill.exec.planner.logical.DrillViewInfoProvider;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.pojo.PojoRecordReader;
@@ -148,7 +148,7 @@ public abstract class RecordGenerator {
@Override
public boolean visitTable(String schemaName, String tableName, Table table) {
if (table.getJdbcTableType() == TableType.VIEW) {
- records.add(new Records.View("DRILL", schemaName, tableName, ((DrillViewTable)table).getViewSql()));
+ records.add(new Records.View("DRILL", schemaName, tableName, ((DrillViewInfoProvider) table).getViewSql()));
}
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 8433931..f914661 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -118,6 +118,9 @@ public class HiveTestDataGenerator {
"uniontypeType UNIONTYPE<int, double, array<string>>)"
);
+ // create a Hive view to test how its metadata is populated in Drill's INFORMATION_SCHEMA
+ executeQuery("CREATE VIEW IF NOT EXISTS hiveview AS SELECT * FROM kv");
+
ss.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 501556a..123c160 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -51,6 +51,7 @@ public class TestMetadataDDL extends TestJdbcQuery {
.returns(
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n"
@@ -74,6 +75,7 @@ public class TestMetadataDDL extends TestJdbcQuery {
.returns(
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60a429fb/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
index 7dc9c30..93626d5 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
@@ -302,6 +302,14 @@ public class TestViews extends TestJdbcQuery {
}
@Test
+ public void testInfoSchemaWithHiveView() throws Exception {
+ JdbcAssert.withFull("hive.default")
+ .sql("SELECT * FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_NAME = 'hiveview'")
+ .returns("TABLE_CATALOG=DRILL; TABLE_SCHEMA=hive.default; TABLE_NAME=hiveview; " +
+ "VIEW_DEFINITION=SELECT `kv`.`key`, `kv`.`value` FROM `default`.`kv`");
+ }
+
+ @Test
public void testViewWithFullSchemaIdentifier() throws Exception{
JdbcAssert.withNoDefaultSchema().withConnection(new Function<Connection, Void>() {
public Void apply(Connection connection) {
[08/24] git commit: DRILL-521: Fix failures in metadata conversion of
Hive tables for INFORMATION_SCHEMA
Posted by ja...@apache.org.
DRILL-521: Fix failures in metadata conversion of Hive tables for INFORMATION_SCHEMA
- Instead of converting two forms of Hive types info (string and ObjectInspector), switch
to using only one form of type info TypeInfo.
- Add missing mappings of Hive data type to Sql data type.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/71432fd1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/71432fd1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/71432fd1
Branch: refs/heads/master
Commit: 71432fd1ed7659aec118514bdf822043da293992
Parents: 9f3b9d2
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Jun 11 12:44:26 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 16:07:07 2014 -0700
----------------------------------------------------------------------
.../exec/store/hive/schema/DrillHiveTable.java | 148 +++++++++----------
.../drill/exec/store/ischema/Records.java | 12 +-
.../exec/work/fragment/FragmentExecutor.java | 4 +-
.../exec/store/hive/HiveTestDataGenerator.java | 29 +++-
.../apache/drill/jdbc/test/TestJdbcQuery.java | 2 +-
.../apache/drill/jdbc/test/TestMetadataDDL.java | 6 +-
6 files changed, 116 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71432fd1/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
index 7d6bc72..02d19d3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -19,16 +19,20 @@ package org.apache.drill.exec.store.hive.schema;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.List;
-import org.apache.drill.common.logical.StoragePluginConfig;
+import com.google.common.collect.Lists;
import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.store.hive.HiveReadEntry;
import org.apache.drill.exec.store.hive.HiveStoragePlugin;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.eigenbase.reltype.RelDataType;
import org.eigenbase.reltype.RelDataTypeFactory;
import org.eigenbase.sql.SqlCollation;
@@ -41,82 +45,35 @@ public class DrillHiveTable extends DrillTable{
public DrillHiveTable(String storageEngineName, HiveStoragePlugin plugin, HiveReadEntry readEntry) {
super(storageEngineName, plugin, readEntry);
- this.hiveTable = new org.apache.hadoop.hive.ql.metadata.Table(readEntry.getTable());
+ this.hiveTable = readEntry.getTable();
}
@Override
public RelDataType getRowType(RelDataTypeFactory typeFactory) {
- ArrayList<RelDataType> typeList = new ArrayList<>();
- ArrayList<String> fieldNameList = new ArrayList<>();
-
- ArrayList<StructField> hiveFields = hiveTable.getFields();
- for(StructField hiveField : hiveFields) {
- fieldNameList.add(hiveField.getFieldName());
- typeList.add(getRelDataTypeFromHiveType(typeFactory, hiveField.getFieldObjectInspector()));
+ List<RelDataType> typeList = Lists.newArrayList();
+ List<String> fieldNameList = Lists.newArrayList();
+
+ List<FieldSchema> hiveFields = hiveTable.getSd().getCols();
+ for(FieldSchema hiveField : hiveFields) {
+ fieldNameList.add(hiveField.getName());
+ typeList.add(getRelDataTypeFromHiveType(
+ typeFactory, TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType())));
}
for (FieldSchema field : hiveTable.getPartitionKeys()) {
fieldNameList.add(field.getName());
- typeList.add(getRelDataTypeFromHiveTypeString(typeFactory, field.getType()));
+ typeList.add(getRelDataTypeFromHiveType(
+ typeFactory, TypeInfoUtils.getTypeInfoFromTypeString(field.getType())));
}
- final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList);
- return rowType;
+ return typeFactory.createStructType(typeList, fieldNameList);
}
- private RelDataType getRelDataTypeFromHiveTypeString(RelDataTypeFactory typeFactory, String type) {
- switch(type) {
- case "boolean":
- return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
-
- case "tinyint":
- return typeFactory.createSqlType(SqlTypeName.TINYINT);
-
- case "smallint":
- return typeFactory.createSqlType(SqlTypeName.SMALLINT);
-
- case "int":
- return typeFactory.createSqlType(SqlTypeName.INTEGER);
-
- case "bigint":
- return typeFactory.createSqlType(SqlTypeName.BIGINT);
-
- case "float":
- return typeFactory.createSqlType(SqlTypeName.FLOAT);
-
- case "double":
- return typeFactory.createSqlType(SqlTypeName.DOUBLE);
-
- case "date":
- return typeFactory.createSqlType(SqlTypeName.DATE);
-
- case "timestamp":
- return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
-
- case "binary":
- return typeFactory.createSqlType(SqlTypeName.BINARY);
-
- case "decimal":
- return typeFactory.createSqlType(SqlTypeName.DECIMAL);
-
- case "string":
- case "varchar": {
- return typeFactory.createTypeWithCharsetAndCollation(
- typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
- Charset.forName("ISO-8859-1"), /*unicode char set*/
- SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
- );
- }
-
- default:
- throw new RuntimeException("Unknown or unsupported hive type: " + type);
- }
- }
-
- private RelDataType getRelDataTypeFromHivePrimitiveType(RelDataTypeFactory typeFactory, PrimitiveObjectInspector poi) {
- switch(poi.getPrimitiveCategory()) {
+ private RelDataType getRelDataTypeFromHivePrimitiveType(RelDataTypeFactory typeFactory, PrimitiveTypeInfo pTypeInfo) {
+ switch(pTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+
case BYTE:
return typeFactory.createSqlType(SqlTypeName.TINYINT);
@@ -159,20 +116,59 @@ public class DrillHiveTable extends DrillTable{
case UNKNOWN:
case VOID:
default:
- throw new RuntimeException("Unknown or unsupported hive type");
+ throwUnsupportedHiveDataTypeError(pTypeInfo.getPrimitiveCategory().toString());
}
+
+ return null;
}
- private RelDataType getRelDataTypeFromHiveType(RelDataTypeFactory typeFactory, ObjectInspector oi) {
- switch(oi.getCategory()) {
+ private RelDataType getRelDataTypeFromHiveType(RelDataTypeFactory typeFactory, TypeInfo typeInfo) {
+ switch(typeInfo.getCategory()) {
case PRIMITIVE:
- return getRelDataTypeFromHivePrimitiveType(typeFactory, ((PrimitiveObjectInspector) oi));
- case LIST:
- case MAP:
- case STRUCT:
+ return getRelDataTypeFromHivePrimitiveType(typeFactory, ((PrimitiveTypeInfo) typeInfo));
+
+ case LIST: {
+ ListTypeInfo listTypeInfo = (ListTypeInfo)typeInfo;
+ RelDataType listElemTypeInfo = getRelDataTypeFromHiveType(typeFactory, listTypeInfo.getListElementTypeInfo());
+ return typeFactory.createArrayType(listElemTypeInfo, -1);
+ }
+
+ case MAP: {
+ MapTypeInfo mapTypeInfo = (MapTypeInfo)typeInfo;
+ RelDataType keyType = getRelDataTypeFromHiveType(typeFactory, mapTypeInfo.getMapKeyTypeInfo());
+ RelDataType valueType = getRelDataTypeFromHiveType(typeFactory, mapTypeInfo.getMapValueTypeInfo());
+ return typeFactory.createMapType(keyType, valueType);
+ }
+
+ case STRUCT: {
+ StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo;
+ ArrayList<String> fieldNames = structTypeInfo.getAllStructFieldNames();
+ ArrayList<TypeInfo> fieldHiveTypeInfoList = structTypeInfo.getAllStructFieldTypeInfos();
+ List<RelDataType> fieldRelDataTypeList = Lists.newArrayList();
+ for(TypeInfo fieldHiveType : fieldHiveTypeInfoList) {
+ fieldRelDataTypeList.add(getRelDataTypeFromHiveType(typeFactory, fieldHiveType));
+ }
+ return typeFactory.createStructType(fieldRelDataTypeList, fieldNames);
+ }
+
case UNION:
- default:
- throw new RuntimeException("Unknown or unsupported hive type");
+ logger.warn("There is no UNION data type in SQL. Converting it to Sql type OTHER to avoid " +
+ "breaking INFORMATION_SCHEMA queries");
+ return typeFactory.createSqlType(SqlTypeName.OTHER);
}
+
+ throwUnsupportedHiveDataTypeError(typeInfo.getCategory().toString());
+ return null;
+ }
+
+ private void throwUnsupportedHiveDataTypeError(String hiveType) {
+ StringBuilder errMsg = new StringBuilder();
+ errMsg.append(String.format("Unsupported Hive data type %s. ", hiveType));
+ errMsg.append(System.getProperty("line.separator"));
+ errMsg.append("Following Hive data types are supported in Drill INFORMATION_SCHEMA: ");
+ errMsg.append("BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, DATE, TIMESTAMP, BINARY, DECIMAL, STRING, " +
+ "VARCHAR, LIST, MAP, STRUCT and UNION");
+
+ throw new RuntimeException(errMsg.toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71432fd1/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java
index d999346..8d10775 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java
@@ -64,7 +64,17 @@ public class Records {
this.ORDINAL_POSITION = field.getIndex();
this.IS_NULLABLE = type.isNullable() ? "YES" : "NO";
- this.DATA_TYPE = sqlType.getName();
+
+ if (sqlType == SqlTypeName.ARRAY || sqlType == SqlTypeName.MAP || sqlType == SqlTypeName.ROW) {
+ // For complex types use the toString method to display the inside elements
+ String typeString = type.toString();
+
+ // RelDataType.toString prints "RecordType" for "STRUCT".
+ this.DATA_TYPE = type.toString().replace("RecordType", "STRUCT");
+ } else {
+ this.DATA_TYPE = sqlType.toString();
+ }
+
this.NUMERIC_PRECISION_RADIX = (sqlType == SqlTypeName.DECIMAL) ? 10 : -1; // TODO: where do we get radix?
this.CHARACTER_MAXIMUM_LENGTH = -1; // TODO: where do we get char length?
this.NUMERIC_PRECISION = (sqlType.allowsPrec())?type.getPrecision(): -1;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71432fd1/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
index 36727ec..7d4b657 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
@@ -79,9 +79,9 @@ public class FragmentExecutor implements Runnable, CancelableQuery, StatusProvid
boolean closed = false;
try {
root = ImplCreator.getExec(context, rootOperator);
- } catch (ExecutionSetupException e) {
+ } catch (AssertionError | Exception e) {
context.fail(e);
- logger.debug("Failure while running fragement", e);
+ logger.debug("Failure while initializing operator tree", e);
internalFail(e);
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71432fd1/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 6aa68b4..8433931 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -88,12 +88,35 @@ public class HiveTestDataGenerator {
// create a table with no data
executeQuery("CREATE TABLE IF NOT EXISTS default.empty_table(a INT, b STRING)");
- // create a table that has all supported types in Drill
+ // create a Hive table that has columns with data types which are supported for reading in Drill.
testDataFile = generateAllTypesDataFile();
- executeQuery("CREATE TABLE IF NOT EXISTS alltypes (c1 INT, c2 BOOLEAN, c3 DOUBLE, c4 STRING, " +
+ executeQuery("CREATE TABLE IF NOT EXISTS allReadSupportedHiveDataTypes (c1 INT, c2 BOOLEAN, c3 DOUBLE, c4 STRING, " +
"c9 TINYINT, c10 SMALLINT, c11 FLOAT, c12 BIGINT, c19 BINARY) " +
"ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE");
- executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE default.alltypes", testDataFile));
+ executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE " +
+ "default.allReadSupportedHiveDataTypes", testDataFile));
+
+ // create a table that has all Hive types. This is to test how hive tables metadata is populated in
+ // Drill's INFORMATION_SCHEMA.
+ executeQuery("CREATE TABLE IF NOT EXISTS allHiveDataTypes(" +
+ "booleanType BOOLEAN, " +
+ "tinyintType TINYINT, " +
+ "smallintType SMALLINT, " +
+ "intType INT, " +
+ "bigintType BIGINT, " +
+ "floatType FLOAT, " +
+ "doubleType DOUBLE, " +
+ "dataType DATE, " +
+ "timestampType TIMESTAMP, " +
+ "binaryType BINARY, " +
+ "decimalType DECIMAL, " +
+ "stringType STRING, " +
+ "varCharType VARCHAR(20), " +
+ "listType ARRAY<STRING>, " +
+ "mapType MAP<STRING,INT>, " +
+ "structType STRUCT<sint:INT,sboolean:BOOLEAN,sstring:STRING>, " +
+ "uniontypeType UNIONTYPE<int, double, array<string>>)"
+ );
ss.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71432fd1/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index bf4e12e..932f207 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -80,7 +80,7 @@ public class TestJdbcQuery extends JdbcTest{
public void testHiveReadWithDb() throws Exception{
testQuery("select * from hive.`default`.kv");
testQuery("select key from hive.`default`.kv group by key");
- testQuery("select * from hive.`default`.alltypes");
+ testQuery("select * from hive.`default`.allreadsupportedhivedatatypes");
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71432fd1/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 3975ead..3580711 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -50,7 +50,8 @@ public class TestMetadataDDL extends TestJdbcQuery {
.sql("SHOW TABLES")
.returns(
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
- "TABLE_SCHEMA=hive.default; TABLE_NAME=alltypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n"
);
@@ -72,7 +73,8 @@ public class TestMetadataDDL extends TestJdbcQuery {
.sql("SHOW TABLES IN hive.`default`")
.returns(
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
- "TABLE_SCHEMA=hive.default; TABLE_NAME=alltypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=allhivedatatypes\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=allreadsupportedhivedatatypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n");
}
[05/24] git commit: DRILL-738: Add test for left,
right and replace function from sql
Posted by ja...@apache.org.
DRILL-738: Add test for left, right and replace function from sql
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/fc54e8ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/fc54e8ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/fc54e8ee
Branch: refs/heads/master
Commit: fc54e8eef99410d43d584d1d270430ebbd652d62
Parents: ba4d1a6
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Jun 10 14:07:11 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 09:03:24 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/jdbc/test/TestJdbcQuery.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/fc54e8ee/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index a2438a1..501927c 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -402,4 +402,17 @@ public class TestJdbcQuery extends JdbcTest{
);
}
+ @Test
+ public void testLeftRightReplace() throws Exception {
+ JdbcAssert.withNoDefaultSchema()
+ .sql("SELECT `left`('abcdef', 2) as LEFT_STR, `right`('abcdef', 2) as RIGHT_STR, `replace`('abcdef', 'ab', 'zz') as REPLACE_STR " +
+ "from cp.`employee.json` limit 1")
+ .returns(
+ "LEFT_STR=ab; " +
+ "RIGHT_STR=ef; " +
+ "REPLACE_STR=zzcdef\n"
+ );
+ }
+
+
}
[12/24] git commit: DRILL-964: Allow enabling/disabling storage
plugin instance from UI
Posted by ja...@apache.org.
DRILL-964: Allow enabling/disabling storage plugin instance from UI
+ Creating a client per request (to avoid issues with multiple connections).
+ Correcting the links.
+ Searchable result table.
+ Minor UI tweaks.
Squashed commits:
[461182d]
[99a678a]
[2c16c3a]
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0879f830
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0879f830
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0879f830
Branch: refs/heads/master
Commit: 0879f830068e0282884b55812f4a4fba0311a63f
Parents: 2903ed3
Author: Sudheesh Katkam <sk...@maprtech.com>
Authored: Tue Jun 10 12:13:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 16:08:16 2014 -0700
----------------------------------------------------------------------
.../drill/exec/server/rest/DrillRestServer.java | 5 ---
.../drill/exec/server/rest/QueryResources.java | 10 ++++-
.../drill/exec/server/rest/StatusResources.java | 2 +-
.../exec/server/rest/StorageResources.java | 29 ++++++++++++--
.../src/main/resources/rest/generic.ftl | 6 +--
.../src/main/resources/rest/profile/list.ftl | 4 +-
.../src/main/resources/rest/profile/profile.ftl | 4 --
.../src/main/resources/rest/query/result.ftl | 33 +++++++++++++---
.../src/main/resources/rest/storage/list.ftl | 22 ++++-------
.../src/main/resources/rest/storage/update.ftl | 41 ++++++++++----------
10 files changed, 95 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
index c3c7b04..11bb776 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
@@ -64,10 +64,6 @@ public class DrillRestServer extends ResourceConfig {
register(provider);
}
- final DrillConfig config = workManager.getContext().getConfig();
- final BufferAllocator allocator = workManager.getContext().getAllocator();
- final ClusterCoordinator coordinator = workManager.getContext().getClusterCoordinator();
- final DrillClient client = new DrillClient(config, coordinator, allocator);
register(new AbstractBinder() {
@Override
protected void configure() {
@@ -75,7 +71,6 @@ public class DrillRestServer extends ResourceConfig {
bind(workManager.getContext().getConfig().getMapper()).to(ObjectMapper.class);
bind(workManager.getContext().getPersistentStoreProvider()).to(PStoreProvider.class);
bind(workManager.getContext().getStorage()).to(StoragePluginRegistry.class);
- bind(client).to(DrillClient.class);
}
});
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
index f444a02..9c1dce1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
@@ -31,8 +31,11 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
+import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
@@ -50,8 +53,6 @@ public class QueryResources {
@Inject
WorkManager work;
- @Inject
- DrillClient client;
@GET
@Produces(MediaType.TEXT_HTML)
@@ -63,6 +64,11 @@ public class QueryResources {
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@Produces(MediaType.TEXT_HTML)
public Viewable submitQuery(@FormParam("query") String query, @FormParam("queryType") String queryType) throws Exception {
+ final DrillConfig config = work.getContext().getConfig();
+ final ClusterCoordinator coordinator = work.getContext().getClusterCoordinator();
+ final BufferAllocator allocator = work.getContext().getAllocator();
+ DrillClient client = new DrillClient(config, coordinator, allocator);
+
UserBitShared.QueryType type = UserBitShared.QueryType.SQL;
switch (queryType){
case "SQL" : type = UserBitShared.QueryType.SQL; break;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
index c98c8e6..4ec4182 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
@@ -32,7 +32,7 @@ public class StatusResources {
@Produces(MediaType.TEXT_HTML)
public Viewable getStatus() {
String status = "Running!";
- return new Viewable("/rest/status/status.ftl", status);
+ return new Viewable("/rest/status.ftl", status);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
index 244d8a8..aa090cf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
@@ -19,6 +19,8 @@ package org.apache.drill.exec.server.rest;
import java.io.IOException;
import java.io.StringReader;
+import java.net.URI;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -30,7 +32,10 @@ import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriInfo;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.StoragePluginConfig;
@@ -59,12 +64,15 @@ public class StorageResources {
@Produces(MediaType.TEXT_HTML)
public Viewable getQueries() {
- List<String> names = Lists.newArrayList();
+ List<SimpleHash> list = Lists.newArrayList();
for (Map.Entry<String, StoragePluginConfig> config : storage.getStore()) {
- names.add(config.getKey());
+ SimpleHash map = new SimpleHash();
+ map.put("name", config.getKey());
+ map.put("enabled", config.getValue().isEnabled());
+ list.add(map);
}
- return new Viewable("/rest/storage/list.ftl", names);
+ return new Viewable("/rest/storage/list.ftl", list);
}
@GET
@@ -78,10 +86,25 @@ public class StorageResources {
map.put("config", conf);
map.put("name", name);
map.put("exists", config != null);
+ map.put("enabled", config.isEnabled());
return new Viewable("/rest/storage/update.ftl", map);
}
@GET
+ @Path("/{name}/enable/{val}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response setEnable(@Context UriInfo uriInfo, @PathParam("name") String name, @PathParam("val") Boolean enable) throws ExecutionSetupException {
+ StoragePluginConfig config = findConfig(name);
+ if (config != null) {
+ config.setEnabled(enable);
+ storage.createOrUpdate(name, config, true);
+ }
+
+ URI uri = uriInfo.getBaseUriBuilder().path("/storage").build();
+ return Response.seeOther(uri).build();
+ }
+
+ @GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/{name}/config")
public StoragePluginConfig getConfig(@PathParam("name") String name) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/resources/rest/generic.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/generic.ftl b/exec/java-exec/src/main/resources/rest/generic.ftl
index aad0794..1a47915 100644
--- a/exec/java-exec/src/main/resources/rest/generic.ftl
+++ b/exec/java-exec/src/main/resources/rest/generic.ftl
@@ -25,9 +25,9 @@
<title>Apache Drill</title>
<link href="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap-theme.min.css" rel="stylesheet">
- <link href="theme.css" rel="stylesheet">
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+ <script src="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -67,8 +67,6 @@
<div class="container theme-showcase" role="main">
<@page_body/>
</div>
- <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
- <script src="http://netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
</body>
</html>
</#macro>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/resources/rest/profile/list.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/profile/list.ftl b/exec/java-exec/src/main/resources/rest/profile/list.ftl
index ebd9f7a..61d3466 100644
--- a/exec/java-exec/src/main/resources/rest/profile/list.ftl
+++ b/exec/java-exec/src/main/resources/rest/profile/list.ftl
@@ -29,7 +29,7 @@
<tr>
<td>${query.getValue()}</td>
<td>
- <a href="/query/${query.getKey()}">
+ <a href="/profiles/${query.getKey()}">
<div style="height:100%;width:100%">
${query.getKey()}
</div>
@@ -52,7 +52,7 @@
<tr>
<td>${query.getValue()}</td>
<td>
- <a href="/profile/${query.getKey()}">
+ <a href="/profiles/${query.getKey()}">
<div style="height:100%;width:100%">
${query.getKey()}
</div>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/resources/rest/profile/profile.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
index d4035ae..ffd22db 100644
--- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl
+++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl
@@ -47,15 +47,11 @@
<div class="page-header">
<h2>Physical Plan</h2>
</div>
- <div class="well">
<p><pre>${model.plan}</pre></p>
- </div>
<div class="page-header">
<h2>Complete Profile</h2>
</div>
- <div class="well">
<p><pre>${model.toString()}</pre></p>
- </div>
</#macro>
<@page_html/>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/resources/rest/query/result.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/query/result.ftl b/exec/java-exec/src/main/resources/rest/query/result.ftl
index e608650..bedbf9f 100644
--- a/exec/java-exec/src/main/resources/rest/query/result.ftl
+++ b/exec/java-exec/src/main/resources/rest/query/result.ftl
@@ -11,6 +11,12 @@
<#include "*/generic.ftl">
<#macro page_head>
+ <link rel="stylesheet" type="text/css" href="//code.jquery.com/ui/1.10.3/themes/smoothness/jquery-ui.css">
+ <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/plug-ins/be7019ee387/integration/jqueryui/dataTables.jqueryui.css">
+
+ <script type="text/javascript" language="javascript" src="//code.jquery.com/jquery-1.10.2.min.js"></script>
+ <script type="text/javascript" language="javascript" src="//cdn.datatables.net/1.10.0/js/jquery.dataTables.min.js"></script>
+ <script type="text/javascript" language="javascript" src="//cdn.datatables.net/plug-ins/be7019ee387/integration/jqueryui/dataTables.jqueryui.js"></script>
</#macro>
<#macro page_body>
@@ -18,19 +24,34 @@
<div class="page-header">
</div>
<h2>Result</h2>
- <div class="table-responsive">
- <table class="table">
- <tbody>
- <#list model as rows>
+ <div style="width=100%; overflow: auto;">
+ <table id="relation" class="table table-striped table-bordered table-condensed" style="display: table; table-layout: fized; width=100%;">
+ <#assign rows = model[0]>
+ <thead style="overflow: auto;">
<tr>
<#list rows as row>
- <td style="border:none;"><pre>${row}</pre></td>
+ <th>${row}</th>
</#list>
</tr>
- </#list>
+ </thead>
+ <tbody style="overflow: auto;">
+ <#list model as rows>
+ <#if (rows_index > 0)>
+ <tr>
+ <#list rows as row>
+ <td>${row}</td>
+ </#list>
+ </tr>
+ </#if>
+ </#list>
</tbody>
</table>
</div>
+ <script charset="utf-8">
+ $(document).ready(function() {
+ $('#relation').dataTable( { "scrollX" : true } );
+ } );
+ </script>
</#macro>
<@page_html/>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/resources/rest/storage/list.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/storage/list.ftl b/exec/java-exec/src/main/resources/rest/storage/list.ftl
index 6afe4b5..5e50616 100644
--- a/exec/java-exec/src/main/resources/rest/storage/list.ftl
+++ b/exec/java-exec/src/main/resources/rest/storage/list.ftl
@@ -24,17 +24,15 @@
<#list model as plugin>
<tr>
<td style="border:none;">
- ${plugin}
+ ${plugin.name}
</td>
<td style="border:none;">
- <form action="/storage/${plugin}/config">
- <button class="btn btn-default" type="submit">View</button>
- </form>
- </td>
- <td style="border:none;">
- <form action="/storage/${plugin}/config/update">
- <button class="btn btn-primary" type="submit">Update</button>
- </form>
+ <a class="btn btn-primary" href="/storage/${plugin.name}/config/update">Update</a>
+ <#if plugin.enabled>
+ <a class="btn btn-default" href="/storage/${plugin.name}/enable/false">Disable</a>
+ <#else>
+ <a class="btn btn-primary" href="/storage/${plugin.name}/enable/true">Enable</a>
+ </#if>
</td>
</tr>
</#list>
@@ -55,13 +53,9 @@
form.submit();
}
</script>
- <button type="submit" class="btn btn-default" onclick="javascript:doSubmit();">Submit</button>
+ <button type="submit" class="btn btn-default" onclick="javascript:doSubmit();">Create</button>
</form>
</div>
- <script>
- var elem = document.getElementById("statusFontColor");
- elem.style.color = "green";
- </script>
</#macro>
<@page_html/>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0879f830/exec/java-exec/src/main/resources/rest/storage/update.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/storage/update.ftl b/exec/java-exec/src/main/resources/rest/storage/update.ftl
index a76d750..ae60852 100644
--- a/exec/java-exec/src/main/resources/rest/storage/update.ftl
+++ b/exec/java-exec/src/main/resources/rest/storage/update.ftl
@@ -19,27 +19,28 @@
</div>
<h3>Configuration</h3>
<form role="form" action="/storage/config/update" method="POST">
- <input type="hidden" name="name" value="${model.name}" />
- <div class="form-group">
- <textarea class="form-control" id="config" rows="20" cols="50" name="config">${model.config}</textarea>
- </div>
- <button class="btn btn-default" type="submit">
- <#if model.exists >Update<#else>Create</#if>
- </button>
+ <input type="hidden" name="name" value="${model.name}" />
+ <div class="form-group">
+ <textarea class="form-control" id="config" rows="20" cols="50" name="config" style="font-family: Courier;">${model.config}</textarea>
+ </div>
+ <a class="btn btn-default" href="/storage">Back</a>
+ <button class="btn btn-default" type="submit">
+ <#if model.exists >Update<#else>Create</#if>
+ </button>
+ <#if model.enabled>
+ <a class="btn btn-default" href="/storage/${model.name}/enable/false">Disable</a>
+ <#else>
+ <a class="btn btn-primary" href="/storage/${model.name}/enable/true">Enable</a>
+ </#if>
+ <#if model.exists >
+ <form role="form" action="/storage/config/delete" method="POST">
+ <input type="hidden" name="name" value="${model.name}" />
+ <button type="submit" class="btn btn-default" onclick="return confirm('Are you sure?')">
+ Delete
+ </button>
+ </form>
+ </#if>
</form>
- <br/>
- <#if model.exists >
- <form role="form" action="/storage/config/delete" method="POST">
- <input type="hidden" name="name" value="${model.name}" />
- <button type="submit" class="btn btn-default" onclick="return confirm('Are you sure?')">
- Delete
- </button>
- </form>
- </#if>
- <script>
- var elem = document.getElementById("statusFontColor");
- elem.style.color = "green";
- </script>
</#macro>
<@page_html/>
\ No newline at end of file
[07/24] git commit: DRILL-773: Add test for aggregate function with
drill custom function The actual fix is in Optiq.
Posted by ja...@apache.org.
DRILL-773: Add test for aggregate function with drill custom function
The actual fix is in Optiq.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ba4d1a6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ba4d1a6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ba4d1a6d
Branch: refs/heads/master
Commit: ba4d1a6dd3448d115709e9f8548fba2a25eef10b
Parents: b900be2
Author: Mehant Baid <me...@gmail.com>
Authored: Tue Jun 10 13:14:11 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 09:03:24 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/jdbc/test/TestJdbcQuery.java | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ba4d1a6d/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 1b83148..a2438a1 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -391,4 +391,15 @@ public class TestJdbcQuery extends JdbcTest{
"employee_id=101; EXPR$1=Test\n"
);
}
+
+ @Test
+ public void testAggWithDrillFunc() throws Exception {
+ JdbcAssert.withNoDefaultSchema()
+ .sql("SELECT extract(year from max(to_timestamp(hire_date, 'yyyy-MM-dd HH:mm:SS.SSS' ))) as MAX_YEAR " +
+ "from cp.`employee.json` ")
+ .returns(
+ "MAX_YEAR=1998\n"
+ );
+ }
+
}
[02/24] git commit: DRILL-790: In FragmentExecutor if operator tree
init fails notify failure to root fragment to avoid query hangs.
Posted by ja...@apache.org.
DRILL-790: In FragmentExecutor if operator tree init fails notify failure to root fragment to avoid query hangs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/dd650cc9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/dd650cc9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/dd650cc9
Branch: refs/heads/master
Commit: dd650cc96f655d5c84603cf8e0ad53b4fc3c37ff
Parents: 27a9c98
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Jun 10 10:23:47 2014 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Tue Jun 10 22:41:12 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/exec/work/fragment/FragmentExecutor.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/dd650cc9/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
index 4474f3f..36727ec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/fragment/FragmentExecutor.java
@@ -81,6 +81,8 @@ public class FragmentExecutor implements Runnable, CancelableQuery, StatusProvid
root = ImplCreator.getExec(context, rootOperator);
} catch (ExecutionSetupException e) {
context.fail(e);
+ logger.debug("Failure while running fragement", e);
+ internalFail(e);
return;
}
[14/24] git commit: DRILL-965: Close underlying buffers/channels when
FragmentContext is closed.
Posted by ja...@apache.org.
DRILL-965: Close underlying buffers/channels when FragmentContext is closed.
* Fixes unit test failures due race condition between closing the allocator and last message handling.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b328d7b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b328d7b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b328d7b7
Branch: refs/heads/master
Commit: b328d7b718b59dc7aa98543f02163940d75100b6
Parents: 0879f83
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue Jun 10 18:32:14 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 16:08:17 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/ops/FragmentContext.java | 4 ++
.../apache/drill/exec/rpc/RemoteConnection.java | 43 ++++++++++++--------
.../exec/work/batch/AbstractDataCollector.java | 10 +++++
.../drill/exec/work/batch/DataCollector.java | 6 +--
.../drill/exec/work/batch/IncomingBuffers.java | 22 ++++++----
5 files changed, 56 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b328d7b7/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 7a82f1d..f72d672 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -251,6 +251,10 @@ public class FragmentContext implements Closeable {
for(Thread thread: daemonThreads){
thread.interrupt();
}
+ if (buffers != null) {
+ buffers.close();
+ }
allocator.close();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b328d7b7/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
index a19f8d8..cc3ec69 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RemoteConnection.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.rpc;
+import java.util.concurrent.ExecutionException;
+
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
@@ -24,22 +26,22 @@ import io.netty.channel.ChannelInboundHandlerAdapter;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-public abstract class RemoteConnection implements ConnectionThrottle{
+public abstract class RemoteConnection implements ConnectionThrottle, AutoCloseable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemoteConnection.class);
private final Channel channel;
private final WriteManager writeManager;
-
+
public boolean inEventLoop(){
return channel.eventLoop().inEventLoop();
}
-
+
public RemoteConnection(Channel channel) {
super();
this.channel = channel;
this.writeManager = new WriteManager();
channel.pipeline().addLast(new BackPressureHandler());
}
-
+
public abstract BufferAllocator getAllocator();
public final Channel getChannel() {
@@ -57,41 +59,39 @@ public abstract class RemoteConnection implements ConnectionThrottle{
}
public void setAutoRead(boolean enableAutoRead){
- channel.config().setAutoRead(enableAutoRead);
+ channel.config().setAutoRead(enableAutoRead);
}
-
+
public boolean isActive(){
return channel.isActive();
}
-
+
/**
* The write manager is responsible for controlling whether or not a write can be sent. It controls whether or not to block a sender if we have tcp backpressure on the receive side.
*/
private static class WriteManager{
private final ResettableBarrier barrier = new ResettableBarrier();
-
+
public WriteManager(){
barrier.openBarrier();
}
-
+
public void waitForWritable() throws InterruptedException{
barrier.await();
}
-
+
public void setWritable(boolean isWritable){
// logger.debug("Set writable: {}", isWritable);
if(isWritable){
- barrier.openBarrier();
+ barrier.openBarrier();
}else{
barrier.closeBarrier();
}
-
+
}
-
+
}
-
-
private class BackPressureHandler extends ChannelInboundHandlerAdapter{
@Override
@@ -100,7 +100,16 @@ public abstract class RemoteConnection implements ConnectionThrottle{
writeManager.setWritable(ctx.channel().isWritable());
ctx.fireChannelWritabilityChanged();
}
-
-
+
+ }
+
+ @Override
+ public void close() {
+ try {
+ channel.close().get();
+ } catch (InterruptedException | ExecutionException e) {
+ logger.warn("Caught exception while closing channel.", e);
+ }
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b328d7b7/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
index 83e697d..a67f06b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/AbstractDataCollector.java
@@ -135,4 +135,14 @@ public abstract class AbstractDataCollector implements DataCollector, ReadContro
}
protected abstract RawBatchBuffer getBuffer(int minorFragmentId);
+
+ @Override
+ public void close() {
+ for (int i = 0; i < connections.length(); i++) {
+ if (connections.get(i) != null) {
+ connections.get(i).close();
+ };
+ }
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b328d7b7/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java
index 67d78ac..dc016be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/DataCollector.java
@@ -17,17 +17,15 @@
*/
package org.apache.drill.exec.work.batch;
-
import java.io.IOException;
import org.apache.drill.exec.record.RawFragmentBatch;
-import org.apache.drill.exec.rpc.RemoteConnection;
-
-interface DataCollector {
+interface DataCollector extends AutoCloseable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DataCollector.class);
public boolean batchArrived(int minorFragmentId, RawFragmentBatch batch) throws IOException ;
public int getOppositeMajorFragmentId();
public RawBatchBuffer[] getBuffers();
public int getTotalIncomingFragments();
+ public void close();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b328d7b7/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
index 9b3b870..3b97934 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/IncomingBuffers.java
@@ -35,7 +35,7 @@ import com.google.common.collect.Maps;
/**
* Determines when a particular fragment has enough data for each of its receiving exchanges to commence execution. Also monitors whether we've collected all incoming data.
*/
-public class IncomingBuffers {
+public class IncomingBuffers implements AutoCloseable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IncomingBuffers.class);
private final AtomicInteger streamsRemaining = new AtomicInteger(0);
@@ -63,7 +63,7 @@ public class IncomingBuffers {
public boolean batchArrived(RawFragmentBatch batch) throws FragmentSetupException {
// no need to do anything if we've already enabled running.
-// logger.debug("New Batch Arrived {}", batch);
+ // logger.debug("New Batch Arrived {}", batch);
if (batch.getHeader().getIsOutOfMemory()) {
for (DataCollector fSet : fragCounts.values()) {
try {
@@ -99,13 +99,13 @@ public class IncomingBuffers {
public RawBatchBuffer[] getBuffers(int senderMajorFragmentId){
return fragCounts.get(senderMajorFragmentId).getBuffers();
}
-
-
+
+
/**
* Designed to setup initial values for arriving fragment accounting.
*/
public class CountRequiredFragments extends AbstractPhysicalVisitor<Void, Map<Integer, DataCollector>, RuntimeException> {
-
+
@Override
public Void visitReceiver(Receiver receiver, Map<Integer, DataCollector> counts) throws RuntimeException {
DataCollector set;
@@ -114,13 +114,12 @@ public class IncomingBuffers {
} else {
set = new PartitionedCollector(remainingRequired, receiver, context);
}
-
+
counts.put(set.getOppositeMajorFragmentId(), set);
remainingRequired.incrementAndGet();
return null;
}
-
@Override
public Void visitOp(PhysicalOperator op, Map<Integer, DataCollector> value) throws RuntimeException {
for(PhysicalOperator o : op){
@@ -129,10 +128,17 @@ public class IncomingBuffers {
return null;
}
-
}
public boolean isDone(){
return streamsRemaining.get() < 1;
}
+
+ @Override
+ public void close() {
+ for (DataCollector fragment : fragCounts.values()) {
+ fragment.close();
+ }
+ }
+
}
[11/24] git commit: DRILL-678: Fix maxLen of varchar and precision of
decimal type in InfoSchema.
Posted by ja...@apache.org.
DRILL-678: Fix maxLen of varchar and precision of decimal type in InfoSchema.
Derive maxLen of Hive string/varchar and precision of decimal from Hive table metadata.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4773b57a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4773b57a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4773b57a
Branch: refs/heads/master
Commit: 4773b57abc04321e03f4b365a029ee593143513d
Parents: 68b9778
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Jun 11 14:14:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 16:07:08 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/hive/schema/DrillHiveTable.java | 6 ++++--
.../org/apache/drill/exec/store/ischema/Records.java | 12 ++++++++++--
.../org/apache/drill/jdbc/test/TestMetadataDDL.java | 15 +++++++++++++++
3 files changed, 29 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4773b57a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
index 1c1e4da..d1a5659 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -107,12 +107,14 @@ public class DrillHiveTable extends DrillTable{
return typeFactory.createSqlType(SqlTypeName.BINARY);
case DECIMAL:
- return typeFactory.createSqlType(SqlTypeName.DECIMAL);
+ final int precision = 38; // Hive 0.12 has standard precision
+ return typeFactory.createSqlType(SqlTypeName.DECIMAL, precision);
case STRING:
case VARCHAR: {
+ int maxLen = TypeInfoUtils.getCharacterLengthForType(pTypeInfo);
return typeFactory.createTypeWithCharsetAndCollation(
- typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
+ typeFactory.createSqlType(SqlTypeName.VARCHAR, maxLen), /*input type*/
Charset.forName("ISO-8859-1"), /*unicode char set*/
SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4773b57a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java
index 8d10775..9bb182f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/Records.java
@@ -76,8 +76,16 @@ public class Records {
}
this.NUMERIC_PRECISION_RADIX = (sqlType == SqlTypeName.DECIMAL) ? 10 : -1; // TODO: where do we get radix?
- this.CHARACTER_MAXIMUM_LENGTH = -1; // TODO: where do we get char length?
- this.NUMERIC_PRECISION = (sqlType.allowsPrec())?type.getPrecision(): -1;
+
+ if (sqlType == SqlTypeName.VARCHAR) {
+ // Max length is stored as precision in Optiq.
+ this.CHARACTER_MAXIMUM_LENGTH = (sqlType.allowsPrec()) ? type.getPrecision() : -1;
+ this.NUMERIC_PRECISION = -1;
+ } else {
+ this.CHARACTER_MAXIMUM_LENGTH = -1;
+ this.NUMERIC_PRECISION = (sqlType.allowsPrec()) ? type.getPrecision() : -1;
+ }
+
this.NUMERIC_SCALE = (sqlType.allowsScale())?type.getScale(): -1;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4773b57a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 279d637..501556a 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -232,6 +232,21 @@ public class TestMetadataDDL extends TestJdbcQuery {
}
@Test
+ public void testVarCharMaxLengthAndDecimalPrecisionInInfoSchema() throws Exception{
+ JdbcAssert.withNoDefaultSchema()
+ .sql("SELECT COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION " +
+ "FROM INFORMATION_SCHEMA.`COLUMNS` " +
+ "WHERE TABLE_SCHEMA = 'hive.default' AND TABLE_NAME = 'allhivedatatypes' AND " +
+ "(COLUMN_NAME = 'stringtype' OR COLUMN_NAME = 'varchartype' OR " +
+ "COLUMN_NAME = 'inttype' OR COLUMN_NAME = 'decimaltype')")
+ .returns(
+ "COLUMN_NAME=inttype; DATA_TYPE=INTEGER; CHARACTER_MAXIMUM_LENGTH=-1; NUMERIC_PRECISION=-1\n" +
+ "COLUMN_NAME=decimaltype; DATA_TYPE=DECIMAL; CHARACTER_MAXIMUM_LENGTH=-1; NUMERIC_PRECISION=38\n" +
+ "COLUMN_NAME=stringtype; DATA_TYPE=VARCHAR; CHARACTER_MAXIMUM_LENGTH=65535; NUMERIC_PRECISION=-1\n" +
+ "COLUMN_NAME=varchartype; DATA_TYPE=VARCHAR; CHARACTER_MAXIMUM_LENGTH=20; NUMERIC_PRECISION=-1");
+ }
+
+ @Test
public void testDefaultSchemaDfs() throws Exception{
JdbcAssert.withFull("dfs")
.sql(String.format("SELECT R_REGIONKEY FROM `%s/../../sample-data/region.parquet` LIMIT 2", WORKING_PATH))
[22/24] git commit: DRILL-956: Fix issue where LocalPStore provider
doesn't have correct Constructor signature
Posted by ja...@apache.org.
DRILL-956: Fix issue where LocalPStore provider doesn't have correct Constructor signature
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/21d9fb79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/21d9fb79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/21d9fb79
Branch: refs/heads/master
Commit: 21d9fb79da6810170dea668f1884cc73e4676410
Parents: 53e218e
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Wed Jun 11 17:09:26 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:30:30 2014 -0700
----------------------------------------------------------------------
.../exec/store/hbase/config/HBasePStoreProvider.java | 8 ++++++++
.../org/apache/drill/hbase/TestHBaseTableProvider.java | 10 ++--------
distribution/src/resources/drill-override-example.conf | 3 ++-
.../org/apache/drill/exec/store/sys/PStoreRegistry.java | 8 ++++----
.../drill/exec/store/sys/local/LocalPStoreProvider.java | 10 +++++++---
5 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/21d9fb79/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java
index c16d3df..48e6c42 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.util.Bytes;
+import com.google.common.annotations.VisibleForTesting;
+
public class HBasePStoreProvider implements PStoreProvider {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBasePStoreProvider.class);
@@ -65,6 +67,12 @@ public class HBasePStoreProvider implements PStoreProvider {
this.storeTableName = registry.getConfig().getString(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_TABLE);
}
+ @VisibleForTesting
+ public HBasePStoreProvider(Configuration conf, String storeTableName) {
+ this.hbaseConf = conf;
+ this.storeTableName = storeTableName;
+ }
+
@Override
public <V> PStore<V> getPStore(PStoreConfig<V> store) throws IOException {
return new HBasePStore<V>(store, this.table);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/21d9fb79/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
index 8fb7b39..c94e61f 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
import org.apache.drill.exec.store.hbase.config.HBasePStoreProvider;
import org.apache.drill.exec.store.sys.PStore;
@@ -43,14 +44,7 @@ public class TestHBaseTableProvider extends BaseHBaseTest {
@BeforeClass // mask HBase cluster start function
public static void setUpBeforeTestHBaseTableProvider() throws Exception {
- Map<String, String> hbaseProps = Maps.newHashMap();
- hbaseProps.put(HConstants.ZOOKEEPER_QUORUM, storagePluginConfig.getZookeeperQuorum());
- hbaseProps.put(DrillHBaseConstants.HBASE_ZOOKEEPER_PORT, storagePluginConfig.getZookeeperport());
- Config newConfig = bit.getContext().getConfig()
- .withValue(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_CONFIG, ConfigValueFactory.fromMap(hbaseProps))
- .withValue(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_TABLE, ConfigValueFactory.fromAnyRef("drill_store"));
- PStoreRegistry registry = new PStoreRegistry(bit.getCoordinator(), newConfig);
- provider = new HBasePStoreProvider(registry);
+ provider = new HBasePStoreProvider(storagePluginConfig.getHBaseConf(), "drill_store");
provider.start();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/21d9fb79/distribution/src/resources/drill-override-example.conf
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-override-example.conf b/distribution/src/resources/drill-override-example.conf
index 9d87f76..64a7bc5 100644
--- a/distribution/src/resources/drill-override-example.conf
+++ b/distribution/src/resources/drill-override-example.conf
@@ -95,7 +95,8 @@ drill.exec: {
executor.threads: 4
},
sys.store.provider: {
- class: "org.apache.drill.exec.store.sys.local.LocalPStoreProvider",
+ class: "org.apache.drill.exec.store.sys.zk.ZkPStoreProvider",
+ # The following section is only required by LocalPStoreProvider
local: {
path: "/tmp/drill",
write: true
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/21d9fb79/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java
index 7ba7ada..e69c12d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java
@@ -20,20 +20,20 @@ package org.apache.drill.exec.store.sys;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.coord.ClusterCoordinator;
-import com.typesafe.config.Config;
import com.typesafe.config.ConfigException;
public class PStoreRegistry {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PStoreRegistry.class);
- private Config config;
+ private DrillConfig config;
private ClusterCoordinator coord;
- public PStoreRegistry(ClusterCoordinator coord, Config config) {
+ public PStoreRegistry(ClusterCoordinator coord, DrillConfig config) {
this.coord = coord;
this.config = config;
}
@@ -42,7 +42,7 @@ public class PStoreRegistry {
return this.coord;
}
- public Config getConfig() {
+ public DrillConfig getConfig() {
return this.config;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/21d9fb79/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
index 32a981a..16fd875 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
@@ -20,12 +20,12 @@ package org.apache.drill.exec.store.sys.local;
import java.io.File;
import java.io.IOException;
+import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.store.sys.PStore;
import org.apache.drill.exec.store.sys.PStoreConfig;
import org.apache.drill.exec.store.sys.PStoreProvider;
-
-import com.typesafe.config.Config;
+import org.apache.drill.exec.store.sys.PStoreRegistry;
/**
* A really simple provider that stores data in the local file system, one value per file.
@@ -36,11 +36,15 @@ public class LocalPStoreProvider implements PStoreProvider{
private File path;
private final boolean enableWrite;
- public LocalPStoreProvider(Config config) {
+ public LocalPStoreProvider(DrillConfig config) {
path = new File(config.getString(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH));
enableWrite = config.getBoolean(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE);
}
+ public LocalPStoreProvider(PStoreRegistry registry) {
+ this(registry.getConfig());
+ }
+
@Override
public void close() {
}
[09/24] git commit: DRILL-537: Add a unittest that reads from a Hive
table which has all supported data types in Hive->Drill.
Posted by ja...@apache.org.
DRILL-537: Add a unittest that reads from a Hive table which has all supported data types in Hive->Drill.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/9f3b9d29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/9f3b9d29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/9f3b9d29
Branch: refs/heads/master
Commit: 9f3b9d298d68bc6cfc1aae512a0d1addc27810b6
Parents: 72084e3
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Jun 10 16:09:08 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 16:07:07 2014 -0700
----------------------------------------------------------------------
.../exec/store/hive/HiveTestDataGenerator.java | 39 ++++++++++++++------
.../apache/drill/jdbc/test/TestJdbcQuery.java | 1 +
.../apache/drill/jdbc/test/TestMetadataDDL.java | 2 +
3 files changed, 30 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9f3b9d29/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 5a511c0..6aa68b4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -88,6 +88,13 @@ public class HiveTestDataGenerator {
// create a table with no data
executeQuery("CREATE TABLE IF NOT EXISTS default.empty_table(a INT, b STRING)");
+ // create a table that has all supported types in Drill
+ testDataFile = generateAllTypesDataFile();
+ executeQuery("CREATE TABLE IF NOT EXISTS alltypes (c1 INT, c2 BOOLEAN, c3 DOUBLE, c4 STRING, " +
+ "c9 TINYINT, c10 SMALLINT, c11 FLOAT, c12 BIGINT, c19 BINARY) " +
+ "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE");
+ executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE default.alltypes", testDataFile));
+
ss.close();
}
@@ -98,7 +105,7 @@ public class HiveTestDataGenerator {
executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE %s.%s", dataFile, dbName, tblName));
}
- private String generateTestDataFile() throws Exception {
+ private File getTempFile() throws Exception {
File file = null;
while (true) {
file = File.createTempFile("drill-hive-test", ".txt");
@@ -111,6 +118,12 @@ public class HiveTestDataGenerator {
logger.debug("retry creating tmp file");
}
+ return file;
+ }
+
+ private String generateTestDataFile() throws Exception {
+ File file = getTempFile();
+
PrintWriter printWriter = new PrintWriter(file);
for (int i=1; i<=5; i++)
printWriter.println (String.format("%d, key_%d", i, i));
@@ -120,17 +133,7 @@ public class HiveTestDataGenerator {
}
private String generateTestDataFileWithDate() throws Exception {
- File file = null;
- while (true) {
- file = File.createTempFile("drill-hive-test-date", ".txt");
- if (file.exists()) {
- boolean success = file.delete();
- if (success) {
- break;
- }
- }
- logger.debug("retry creating tmp file");
- }
+ File file = getTempFile();
PrintWriter printWriter = new PrintWriter(file);
for (int i=1; i<=5; i++) {
@@ -143,6 +146,18 @@ public class HiveTestDataGenerator {
return file.getPath();
}
+ private String generateAllTypesDataFile() throws Exception {
+ File file = getTempFile();
+
+ PrintWriter printWriter = new PrintWriter(file);
+ printWriter.println("\\N,\\N,\\N,\\N,\\N,\\N,\\N,\\N,\\N");
+ printWriter.println("-1,false,-1.1,,-1,-1,-1.0,-1,\\N");
+ printWriter.println("1,true,1.1,1,1,1,1.0,1,YWJjZA==");
+ printWriter.close();
+
+ return file.getPath();
+ }
+
private void executeQuery(String query) {
CommandProcessorResponse response = null;
boolean failed = false;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9f3b9d29/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 2a69469..bf4e12e 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -80,6 +80,7 @@ public class TestJdbcQuery extends JdbcTest{
public void testHiveReadWithDb() throws Exception{
testQuery("select * from hive.`default`.kv");
testQuery("select key from hive.`default`.kv group by key");
+ testQuery("select * from hive.`default`.alltypes");
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9f3b9d29/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 5299bb5..3975ead 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -50,6 +50,7 @@ public class TestMetadataDDL extends TestJdbcQuery {
.sql("SHOW TABLES")
.returns(
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=alltypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n"
);
@@ -71,6 +72,7 @@ public class TestMetadataDDL extends TestJdbcQuery {
.sql("SHOW TABLES IN hive.`default`")
.returns(
"TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" +
+ "TABLE_SCHEMA=hive.default; TABLE_NAME=alltypes\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" +
"TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n");
}
[10/24] git commit: DRILL-677: Convert Hive data types to nullable
SQL types in metadata conversion.
Posted by ja...@apache.org.
DRILL-677: Convert Hive data types to nullable SQL types in metadata conversion.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/68b9778d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/68b9778d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/68b9778d
Branch: refs/heads/master
Commit: 68b9778d1bcf45fda30bf3017371f9e0705d0de5
Parents: 71432fd
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Jun 11 13:14:56 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 16:07:08 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/hive/schema/DrillHiveTable.java | 9 +++++++--
.../java/org/apache/drill/jdbc/test/TestMetadataDDL.java | 11 +++++++++++
2 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68b9778d/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
index 02d19d3..1c1e4da 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -56,19 +56,24 @@ public class DrillHiveTable extends DrillTable{
List<FieldSchema> hiveFields = hiveTable.getSd().getCols();
for(FieldSchema hiveField : hiveFields) {
fieldNameList.add(hiveField.getName());
- typeList.add(getRelDataTypeFromHiveType(
+ typeList.add(getNullableRelDataTypeFromHiveType(
typeFactory, TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType())));
}
for (FieldSchema field : hiveTable.getPartitionKeys()) {
fieldNameList.add(field.getName());
- typeList.add(getRelDataTypeFromHiveType(
+ typeList.add(getNullableRelDataTypeFromHiveType(
typeFactory, TypeInfoUtils.getTypeInfoFromTypeString(field.getType())));
}
return typeFactory.createStructType(typeList, fieldNameList);
}
+ private RelDataType getNullableRelDataTypeFromHiveType(RelDataTypeFactory typeFactory, TypeInfo typeInfo) {
+ RelDataType relDataType = getRelDataTypeFromHiveType(typeFactory, typeInfo);
+ return typeFactory.createTypeWithNullability(relDataType, true);
+ }
+
private RelDataType getRelDataTypeFromHivePrimitiveType(RelDataTypeFactory typeFactory, PrimitiveTypeInfo pTypeInfo) {
switch(pTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/68b9778d/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 3580711..279d637 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -135,6 +135,17 @@ public class TestMetadataDDL extends TestJdbcQuery {
"COLUMN_NAME=CATALOG_CONNECT; DATA_TYPE=VARCHAR; IS_NULLABLE=NO\n");
}
+
+ @Test
+ public void testDescribeTableNullableColumns() throws Exception{
+ JdbcAssert.withNoDefaultSchema()
+ .sql("DESCRIBE hive.`default`.kv")
+ .returns(
+ "COLUMN_NAME=key; DATA_TYPE=INTEGER; IS_NULLABLE=YES\n" +
+ "COLUMN_NAME=value; DATA_TYPE=VARCHAR; IS_NULLABLE=YES\n"
+ );
+ }
+
@Test
public void testDescribeTableWithSchema() throws Exception{
JdbcAssert.withNoDefaultSchema()
[20/24] git commit: DRILL-600: Support planning for Union-All. Added
infrastructure for planning Union-Distinct (not enabled yet).
Posted by ja...@apache.org.
DRILL-600: Support planning for Union-All. Added infrastructure for planning Union-Distinct (not enabled yet).
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/53e218ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/53e218ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/53e218ef
Branch: refs/heads/master
Commit: 53e218ef6f81eed4c72dcc253ea961348310f199
Parents: d9a2f1c
Author: Aman Sinha <as...@maprtech.com>
Authored: Tue Jun 10 14:54:09 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Jun 11 21:25:16 2014 -0700
----------------------------------------------------------------------
.../physical/base/AbstractPhysicalVisitor.java | 2 +-
.../exec/physical/base/PhysicalVisitor.java | 2 +-
.../drill/exec/physical/config/Union.java | 65 --------
.../drill/exec/physical/config/UnionAll.java | 65 ++++++++
.../impl/union/UnionAllBatchCreator.java | 40 +++++
.../impl/union/UnionAllRecordBatch.java | 157 +++++++++++++++++++
.../physical/impl/union/UnionBatchCreator.java | 40 -----
.../physical/impl/union/UnionRecordBatch.java | 148 -----------------
.../exec/planner/common/DrillUnionRelBase.java | 7 +-
.../exec/planner/logical/DrillRuleSets.java | 2 +
.../exec/planner/logical/DrillUnionRel.java | 8 +-
.../exec/planner/logical/DrillUnionRule.java | 10 +-
.../drill/exec/planner/physical/PrelUtil.java | 4 +
.../exec/planner/physical/UnionAllPrel.java | 94 +++++++++++
.../exec/planner/physical/UnionAllPrule.java | 72 +++++++++
.../planner/physical/UnionDistinctPrel.java | 95 +++++++++++
.../planner/physical/UnionDistinctPrule.java | 72 +++++++++
.../drill/exec/planner/physical/UnionPrel.java | 48 ++++++
.../org/apache/drill/TestExampleQueries.java | 35 ++++-
.../src/test/resources/union/test1.json | 4 +-
20 files changed, 707 insertions(+), 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
index 0107101..c7c82aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
@@ -28,7 +28,7 @@ public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> impleme
}
@Override
- public T visitUnion(Union union, X value) throws E {
+ public T visitUnion(UnionAll union, X value) throws E {
return visitOp(union, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
index 7a1440a..1798289 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
@@ -35,7 +35,7 @@ public interface PhysicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
public RETURN visitStore(Store store, EXTRA value) throws EXCEP;
public RETURN visitFilter(Filter filter, EXTRA value) throws EXCEP;
- public RETURN visitUnion(Union union, EXTRA value) throws EXCEP;
+ public RETURN visitUnion(UnionAll union, EXTRA value) throws EXCEP;
public RETURN visitProject(Project project, EXTRA value) throws EXCEP;
public RETURN visitTrace(Trace trace, EXTRA value) throws EXCEP;
public RETURN visitSort(Sort sort, EXTRA value) throws EXCEP;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Union.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Union.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Union.java
deleted file mode 100644
index 522100f..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/Union.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.config;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.*;
-import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
-
-import java.util.List;
-
-@JsonTypeName("union")
-
-public class Union extends AbstractMultiple {
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Filter.class);
-
- @JsonCreator
- public Union(@JsonProperty("children") PhysicalOperator[] children) {
- super(children);
- }
-
- @Override
- public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
- return physicalVisitor.visitUnion(this, value);
- }
-
- @Override
- public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
- return new Union(children.toArray(new PhysicalOperator[children.size()]));
- }
-
- @Override
- public OperatorCost getCost() {
- OperatorCost cost = new OperatorCost(0,0,0,0);
- for (int i = 0; i < children.length; i++) {
- PhysicalOperator child = children[i];
- cost.add(child.getCost());
- }
- return cost;
- }
-
- @Override
- public int getOperatorType() {
- return CoreOperatorType.UNION_VALUE;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
new file mode 100644
index 0000000..f0ca6ad
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/UnionAll.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.config;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.base.*;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
+
+import java.util.List;
+
+@JsonTypeName("union-all")
+
+public class UnionAll extends AbstractMultiple {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Filter.class);
+
+ @JsonCreator
+ public UnionAll(@JsonProperty("children") PhysicalOperator[] children) {
+ super(children);
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
+ return physicalVisitor.visitUnion(this, value);
+ }
+
+ @Override
+ public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
+ return new UnionAll(children.toArray(new PhysicalOperator[children.size()]));
+ }
+
+ @Override
+ public OperatorCost getCost() {
+ OperatorCost cost = new OperatorCost(0,0,0,0);
+ for (int i = 0; i < children.length; i++) {
+ PhysicalOperator child = children[i];
+ cost.add(child.getCost());
+ }
+ return cost;
+ }
+
+ @Override
+ public int getOperatorType() {
+ return CoreOperatorType.UNION_VALUE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllBatchCreator.java
new file mode 100644
index 0000000..395cab4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllBatchCreator.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.union;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.UnionAll;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.record.RecordBatch;
+
+import com.google.common.base.Preconditions;
+
+public class UnionAllBatchCreator implements BatchCreator<UnionAll>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnionAllBatchCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, UnionAll config, List<RecordBatch> children) throws ExecutionSetupException {
+ Preconditions.checkArgument(children.size() >= 1);
+ return new UnionAllRecordBatch(config, children, context);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
new file mode 100644
index 0000000..e17c06c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.union;
+
+import com.google.common.collect.Lists;
+
+import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.UnionAll;
+import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.record.selection.SelectionVector2;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnionAllRecordBatch.class);
+
+ private final List<RecordBatch> incoming;
+ private SelectionVector2 sv;
+ private Iterator<RecordBatch> incomingIterator = null;
+ private RecordBatch current = null;
+ private ArrayList<TransferPair> transfers;
+ private int outRecordCount;
+
+ public UnionAllRecordBatch(UnionAll config, List<RecordBatch> children, FragmentContext context) throws OutOfMemoryException {
+ super(config, context);
+ this.incoming = children;
+ this.incomingIterator = incoming.iterator();
+ current = incomingIterator.next();
+ sv = null;
+ }
+
+ @Override
+ public int getRecordCount() {
+ return outRecordCount;
+ }
+
+ @Override
+ public void kill() {
+ if(current != null){
+ current.kill();
+ current = null;
+ }
+ for(;incomingIterator.hasNext();){
+ incomingIterator.next().kill();
+ }
+ }
+
+ @Override
+ protected void killIncoming() {
+ for (int i = 0; i < incoming.size(); i++) {
+ RecordBatch in = incoming.get(i);
+ in.kill();
+ }
+ }
+
+
+ @Override
+ public SelectionVector2 getSelectionVector2() {
+ return sv;
+ }
+
+ @Override
+ public IterOutcome innerNext() {
+ if (current == null) { // end of iteration
+ return IterOutcome.NONE;
+ }
+ IterOutcome upstream = current.next();
+ logger.debug("Upstream... {}", upstream);
+ while (upstream == IterOutcome.NONE) {
+ if (!incomingIterator.hasNext()) {
+ current = null;
+ return IterOutcome.NONE;
+ }
+ current = incomingIterator.next();
+ upstream = current.next();
+ }
+ switch (upstream) {
+ case NONE:
+ throw new IllegalArgumentException("not possible!");
+ case NOT_YET:
+ case STOP:
+ return upstream;
+ case OK_NEW_SCHEMA:
+ setupSchema();
+ // fall through.
+ case OK:
+ doTransfer();
+ return upstream; // change if upstream changed, otherwise normal.
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private void doTransfer() {
+ outRecordCount = current.getRecordCount();
+ if (container.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.TWO_BYTE) {
+ this.sv = current.getSelectionVector2();
+ }
+ for (TransferPair transfer : transfers) {
+ transfer.transfer();
+ }
+
+// for (VectorWrapper<?> vw : this.container) {
+// ValueVector.Mutator m = vw.getValueVector().getMutator();
+// m.setValueCount(outRecordCount);
+// }
+
+ }
+
+ private void setupSchema() {
+ if (container != null) {
+ container.clear();
+ }
+ transfers = Lists.newArrayList();
+
+ for (VectorWrapper<?> vw : current) {
+ TransferPair pair = vw.getValueVector().getTransferPair();
+ container.add(pair.getTo());
+ transfers.add(pair);
+ }
+ container.buildSchema(current.getSchema().getSelectionVectorMode());
+ }
+
+ @Override
+ public WritableBatch getWritableBatch() {
+ return WritableBatch.get(this);
+ }
+
+ @Override
+ public void cleanup() {
+ super.cleanup();
+ for (int i = 0; i < incoming.size(); i++) {
+ RecordBatch in = incoming.get(i);
+ in.cleanup();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionBatchCreator.java
deleted file mode 100644
index d9f8813..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionBatchCreator.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.union;
-
-import java.util.List;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.config.Union;
-import org.apache.drill.exec.physical.impl.BatchCreator;
-import org.apache.drill.exec.record.RecordBatch;
-
-import com.google.common.base.Preconditions;
-
-public class UnionBatchCreator implements BatchCreator<Union>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnionBatchCreator.class);
-
- @Override
- public RecordBatch getBatch(FragmentContext context, Union config, List<RecordBatch> children) throws ExecutionSetupException {
- Preconditions.checkArgument(children.size() >= 1);
- return new UnionRecordBatch(config, children, context);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionRecordBatch.java
deleted file mode 100644
index d515323..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionRecordBatch.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.union;
-
-import com.google.common.collect.Lists;
-import org.apache.drill.exec.memory.OutOfMemoryException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.ops.OperatorContext;
-import org.apache.drill.exec.physical.config.Union;
-import org.apache.drill.exec.record.*;
-import org.apache.drill.exec.record.selection.SelectionVector2;
-import org.apache.drill.exec.vector.ValueVector;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-public class UnionRecordBatch extends AbstractRecordBatch<Union> {
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UnionRecordBatch.class);
-
- private final List<RecordBatch> incoming;
- private SelectionVector2 sv;
- private Iterator<RecordBatch> incomingIterator = null;
- private RecordBatch current = null;
- private ArrayList<TransferPair> transfers;
- private int outRecordCount;
-
- public UnionRecordBatch(Union config, List<RecordBatch> children, FragmentContext context) throws OutOfMemoryException {
- super(config, context);
- this.incoming = children;
- this.incomingIterator = incoming.iterator();
- current = incomingIterator.next();
- sv = null;
- }
-
- @Override
- public int getRecordCount() {
- return outRecordCount;
- }
-
- @Override
- public void kill() {
- if(current != null){
- current.kill();
- current = null;
- }
- for(;incomingIterator.hasNext();){
- incomingIterator.next().kill();
- }
- }
-
- @Override
- protected void killIncoming() {
- for (int i = 0; i < incoming.size(); i++) {
- RecordBatch in = incoming.get(i);
- in.kill();
- }
- }
-
-
- @Override
- public SelectionVector2 getSelectionVector2() {
- return sv;
- }
-
- @Override
- public IterOutcome innerNext() {
- if (current == null) { // end of iteration
- return IterOutcome.NONE;
- }
- IterOutcome upstream = current.next();
- logger.debug("Upstream... {}", upstream);
- while (upstream == IterOutcome.NONE) {
- if (!incomingIterator.hasNext()) {
- current = null;
- return IterOutcome.NONE;
- }
- current = incomingIterator.next();
- upstream = current.next();
- }
- switch (upstream) {
- case NONE:
- throw new IllegalArgumentException("not possible!");
- case NOT_YET:
- case STOP:
- return upstream;
- case OK_NEW_SCHEMA:
- setupSchema();
- // fall through.
- case OK:
- doTransfer();
- return upstream; // change if upstream changed, otherwise normal.
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- private void doTransfer() {
- outRecordCount = current.getRecordCount();
- if (container.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.TWO_BYTE) {
- this.sv = current.getSelectionVector2();
- }
- for (TransferPair transfer : transfers) {
- transfer.transfer();
- }
-
-// for (VectorWrapper<?> vw : this.container) {
-// ValueVector.Mutator m = vw.getValueVector().getMutator();
-// m.setValueCount(outRecordCount);
-// }
-
- }
-
- private void setupSchema() {
- if (container != null) {
- container.clear();
- }
- transfers = Lists.newArrayList();
-
- for (VectorWrapper<?> vw : current) {
- TransferPair pair = vw.getValueVector().getTransferPair();
- container.add(pair.getTo());
- transfers.add(pair);
- }
- container.buildSchema(current.getSchema().getSelectionVectorMode());
- }
-
- @Override
- public WritableBatch getWritableBatch() {
- return WritableBatch.get(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
index a16b8ee..6a828e2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
@@ -19,10 +19,10 @@ package org.apache.drill.exec.planner.common;
import java.util.List;
+import org.eigenbase.rel.InvalidRelException;
import org.eigenbase.rel.RelNode;
import org.eigenbase.rel.UnionRelBase;
import org.eigenbase.relopt.RelOptCluster;
-
import org.eigenbase.relopt.RelTraitSet;
/**
@@ -31,7 +31,10 @@ import org.eigenbase.relopt.RelTraitSet;
public abstract class DrillUnionRelBase extends UnionRelBase implements DrillRelNode {
public DrillUnionRelBase(RelOptCluster cluster, RelTraitSet traits,
- List<RelNode> inputs, boolean all) {
+ List<RelNode> inputs, boolean all) throws InvalidRelException {
super(cluster, traits, inputs, all);
+ if (! this.isHomogeneous(false /* don't compare names */)) {
+ throw new InvalidRelException("Input row types of the Union are not compatible.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
index c07fee3..9a256bb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
@@ -165,6 +165,8 @@ public class DrillRuleSets {
ruleList.add(LimitPrule.INSTANCE);
ruleList.add(WriterPrule.INSTANCE);
ruleList.add(PushLimitToTopN.INSTANCE);
+ ruleList.add(UnionAllPrule.INSTANCE);
+ // ruleList.add(UnionDistinctPrule.INSTANCE);
PlannerSettings ps = qcontext.getPlannerSettings();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
index 93c4ca7..fdf43dd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
@@ -39,14 +39,18 @@ import org.eigenbase.relopt.RelTraitSet;
public class DrillUnionRel extends DrillUnionRelBase implements DrillRel {
/** Creates a DrillUnionRel. */
public DrillUnionRel(RelOptCluster cluster, RelTraitSet traits,
- List<RelNode> inputs, boolean all) {
+ List<RelNode> inputs, boolean all) throws InvalidRelException {
super(cluster, traits, inputs, all);
}
@Override
public DrillUnionRel copy(RelTraitSet traitSet, List<RelNode> inputs,
boolean all) {
- return new DrillUnionRel(getCluster(), traitSet, inputs, all);
+ try {
+ return new DrillUnionRel(getCluster(), traitSet, inputs, all);
+ } catch (InvalidRelException e) {
+ throw new AssertionError(e) ;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
index b607605..379d8e1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
@@ -18,18 +18,22 @@
package org.apache.drill.exec.planner.logical;
import org.apache.drill.exec.planner.common.DrillUnionRelBase;
+import org.eigenbase.rel.InvalidRelException;
import org.eigenbase.rel.UnionRel;
import org.eigenbase.rel.RelNode;
import org.eigenbase.relopt.*;
+import org.eigenbase.trace.EigenbaseTrace;
import java.util.ArrayList;
import java.util.List;
+import java.util.logging.Logger;
/**
* Rule that converts a {@link UnionRel} to a {@link DrillUnionRelBase}, implemented by a "union" operation.
*/
public class DrillUnionRule extends RelOptRule {
public static final RelOptRule INSTANCE = new DrillUnionRule();
+ protected static final Logger tracer = EigenbaseTrace.getPlannerTracer();
private DrillUnionRule() {
super(RelOptHelper.any(UnionRel.class, Convention.NONE), "DrillUnionRule");
@@ -44,6 +48,10 @@ public class DrillUnionRule extends RelOptRule {
final RelNode convertedInput = convert(input, input.getTraitSet().plus(DrillRel.DRILL_LOGICAL));
convertedInputs.add(convertedInput);
}
- call.transformTo(new DrillUnionRel(union.getCluster(), traits, convertedInputs, union.all));
+ try {
+ call.transformTo(new DrillUnionRel(union.getCluster(), traits, convertedInputs, union.all));
+ } catch (InvalidRelException e) {
+ tracer.warning(e.toString()) ;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
index d982647..c66ff5d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
@@ -95,6 +95,10 @@ public class PrelUtil {
public static Iterator<Prel> iter(RelNode... nodes){
return (Iterator<Prel>) (Object) Arrays.asList(nodes).iterator();
}
+
+ public static Iterator<Prel> iter(List<RelNode> nodes) {
+ return (Iterator<Prel>) (Object) nodes.iterator();
+ }
public static PlannerSettings getSettings(RelOptCluster cluster){
return cluster.getPlanner().getFrameworkContext().unwrap(PlannerSettings.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
new file mode 100644
index 0000000..20722e9
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.UnionAll;
+import org.apache.drill.exec.planner.cost.DrillCostBase;
+import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.UnionRelBase;
+import org.eigenbase.rel.metadata.RelMetadataQuery;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptCost;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.relopt.RelTraitSet;
+
+import com.google.common.collect.Lists;
+
+public class UnionAllPrel extends UnionPrel {
+
+ public UnionAllPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs) throws InvalidRelException {
+ super(cluster, traits, inputs, true /* all */);
+ }
+
+
+ @Override
+ public UnionRelBase copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
+ try {
+ return new UnionAllPrel(this.getCluster(), traitSet, inputs);
+ }catch (InvalidRelException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
+ return super.computeSelfCost(planner).multiplyBy(.1);
+ }
+ double totalInputRowCount = 0;
+ for (int i = 0; i < this.getInputs().size(); i++) {
+ totalInputRowCount += RelMetadataQuery.getRowCount(this.getInputs().get(i));
+ }
+
+ double cpuCost = totalInputRowCount * DrillCostBase.BASE_CPU_COST;
+ DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
+ return costFactory.makeCost(totalInputRowCount, cpuCost, 0, 0);
+ }
+
+ @Override
+ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
+ List<PhysicalOperator> inputPops = Lists.newArrayList();
+
+ for (int i = 0; i < this.getInputs().size(); i++) {
+ inputPops.add( ((Prel)this.getInputs().get(i)).getPhysicalOperator(creator));
+ }
+
+ UnionAll unionall = new UnionAll(inputPops.toArray(new PhysicalOperator[inputPops.size()]));
+ unionall.setOperatorId(creator.getOperatorId(this));
+
+ return unionall;
+ }
+
+ @Override
+ public SelectionVectorMode[] getSupportedEncodings() {
+ return SelectionVectorMode.DEFAULT;
+ }
+
+ @Override
+ public SelectionVectorMode getEncoding() {
+ return SelectionVectorMode.NONE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
new file mode 100644
index 0000000..1ce73a4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical;
+
+import java.util.List;
+import java.util.logging.Logger;
+
+import org.apache.drill.exec.planner.logical.DrillUnionRel;
+import org.apache.drill.exec.planner.logical.RelOptHelper;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.trace.EigenbaseTrace;
+
+import com.google.common.collect.Lists;
+
+public class UnionAllPrule extends Prule {
+ public static final RelOptRule INSTANCE = new UnionAllPrule();
+ protected static final Logger tracer = EigenbaseTrace.getPlannerTracer();
+
+ private UnionAllPrule() {
+ super(
+ RelOptHelper.any(DrillUnionRel.class), "Prel.UnionAllPrule");
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ DrillUnionRel union = (DrillUnionRel) call.rel(0);
+ return ((! union.isDistinct()) && union.isHomogeneous(false /* don't compare names */));
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final DrillUnionRel union = (DrillUnionRel) call.rel(0);
+ final List<RelNode> inputs = union.getInputs();
+ List<RelNode> convertedInputList = Lists.newArrayList();
+ RelTraitSet traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
+
+ try {
+ for (int i = 0; i < inputs.size(); i++) {
+ RelNode convertedInput = convert(inputs.get(i), PrelUtil.fixTraits(call, traits));
+ convertedInputList.add(convertedInput);
+ }
+
+ traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
+ UnionAllPrel unionAll = new UnionAllPrel(union.getCluster(), traits, convertedInputList);
+
+ call.transformTo(unionAll);
+
+ } catch (InvalidRelException e) {
+ tracer.warning(e.toString());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
new file mode 100644
index 0000000..ad63906
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.UnionAll;
+import org.apache.drill.exec.planner.cost.DrillCostBase;
+import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.UnionRelBase;
+import org.eigenbase.rel.metadata.RelMetadataQuery;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptCost;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.relopt.RelTraitSet;
+
+import com.google.common.collect.Lists;
+
+public class UnionDistinctPrel extends UnionPrel {
+
+ public UnionDistinctPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs) throws InvalidRelException {
+ super(cluster, traits, inputs, false /* all = false */);
+ }
+
+
+ @Override
+ public UnionRelBase copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
+ try {
+ return new UnionDistinctPrel(this.getCluster(), traitSet, inputs);
+ }catch (InvalidRelException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
+ return super.computeSelfCost(planner).multiplyBy(.1);
+ }
+ double totalInputRowCount = 0;
+ for (int i = 0; i < this.getInputs().size(); i++) {
+ totalInputRowCount += RelMetadataQuery.getRowCount(this.getInputs().get(i));
+ }
+
+ double cpuCost = totalInputRowCount * DrillCostBase.BASE_CPU_COST;
+ DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
+ return costFactory.makeCost(totalInputRowCount, cpuCost, 0, 0);
+ }
+
+ @Override
+ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
+ List<PhysicalOperator> inputPops = Lists.newArrayList();
+
+ for (int i = 0; i < this.getInputs().size(); i++) {
+ inputPops.add( ((Prel)this.getInputs().get(i)).getPhysicalOperator(creator));
+ }
+
+ ///TODO: change this to UnionDistinct once implemented end-to-end..
+ UnionAll unionAll = new UnionAll(inputPops.toArray(new PhysicalOperator[inputPops.size()]));
+ unionAll.setOperatorId(creator.getOperatorId(this));
+
+ return unionAll;
+ }
+
+ @Override
+ public SelectionVectorMode[] getSupportedEncodings() {
+ return SelectionVectorMode.DEFAULT;
+ }
+
+ @Override
+ public SelectionVectorMode getEncoding() {
+ return SelectionVectorMode.NONE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
new file mode 100644
index 0000000..7cd5733
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.physical;
+
+import java.util.List;
+import java.util.logging.Logger;
+
+import org.apache.drill.exec.planner.logical.DrillUnionRel;
+import org.apache.drill.exec.planner.logical.RelOptHelper;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptRule;
+import org.eigenbase.relopt.RelOptRuleCall;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.trace.EigenbaseTrace;
+
+import com.google.common.collect.Lists;
+
+public class UnionDistinctPrule extends Prule {
+ public static final RelOptRule INSTANCE = new UnionDistinctPrule();
+ protected static final Logger tracer = EigenbaseTrace.getPlannerTracer();
+
+ private UnionDistinctPrule() {
+ super(
+ RelOptHelper.any(DrillUnionRel.class), "Prel.UnionDistinctPrule");
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ DrillUnionRel union = (DrillUnionRel) call.rel(0);
+ return (union.isDistinct() && union.isHomogeneous(false /* don't compare names */));
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final DrillUnionRel union = (DrillUnionRel) call.rel(0);
+ final List<RelNode> inputs = union.getInputs();
+ List<RelNode> convertedInputList = Lists.newArrayList();
+ RelTraitSet traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
+
+ try {
+ for (int i = 0; i < inputs.size(); i++) {
+ RelNode convertedInput = convert(inputs.get(i), PrelUtil.fixTraits(call, traits));
+ convertedInputList.add(convertedInput);
+ }
+
+ traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
+ UnionDistinctPrel unionDistinct = new UnionDistinctPrel(union.getCluster(), traits, convertedInputList);
+
+ call.transformTo(unionDistinct);
+
+ } catch (InvalidRelException e) {
+ tracer.warning(e.toString());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
new file mode 100644
index 0000000..cade2df
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.planner.physical;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.exec.planner.common.DrillUnionRelBase;
+import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
+import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.reltype.RelDataType;
+
+public abstract class UnionPrel extends DrillUnionRelBase implements Prel{
+
+ public UnionPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs, boolean all) throws InvalidRelException{
+ super(cluster, traits, inputs, all);
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E {
+ return logicalVisitor.visitPrel(this, value);
+ }
+
+ @Override
+ public Iterator<Prel> iterator() {
+ return PrelUtil.iter(this.getInputs());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 1757290..2fcc4fb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -18,6 +18,7 @@
package org.apache.drill;
import org.apache.drill.common.util.FileUtils;
+import org.junit.Ignore;
import org.junit.Test;
public class TestExampleQueries extends BaseTestQuery{
@@ -37,7 +38,7 @@ public class TestExampleQueries extends BaseTestQuery{
test("select count(*) from cp.`customer.json` limit 1");
test("select count(*) from cp.`customer.json` limit 1");
}
-
+
@Test
public void testCaseReturnValueVarChar() throws Exception{
test("select case when employee_id < 1000 then 'ABC' else 'DEF' end from cp.`employee.json` limit 5");
@@ -156,4 +157,36 @@ public class TestExampleQueries extends BaseTestQuery{
test("select count(*) as mycnt, count(*) + 2 * count(*) as mycnt2 from cp.`tpch/nation.parquet` where 1 < 2");
}
+
+ @Test // Simple Union-All over two scans
+ public void testUnionAll1() throws Exception {
+ test("select n_regionkey from cp.`tpch/nation.parquet` union all select r_regionkey from cp.`tpch/region.parquet`");
+ }
+
+ @Test // Union-All over inner joins
+ public void testUnionAll2() throws Exception {
+ test("select n1.n_nationkey from cp.`tpch/nation.parquet` n1 inner join cp.`tpch/region.parquet` r1 on n1.n_regionkey = r1.r_regionkey where n1.n_nationkey in (1, 2) union all select n2.n_nationkey from cp.`tpch/nation.parquet` n2 inner join cp.`tpch/region.parquet` r2 on n2.n_regionkey = r2.r_regionkey where n2.n_nationkey in (3, 4)");
+ }
+
+ @Test // Union-All over grouped aggregates
+ public void testUnionAll3() throws Exception {
+ test("select n1.n_nationkey from cp.`tpch/nation.parquet` n1 where n1.n_nationkey in (1, 2) group by n1.n_nationkey union all select r1.r_regionkey from cp.`tpch/region.parquet` r1 group by r1.r_regionkey");
+ }
+
+ @Test // Chain of Union-Alls
+ public void testUnionAll4() throws Exception {
+ test("select n_regionkey from cp.`tpch/nation.parquet` union all select r_regionkey from cp.`tpch/region.parquet` union all select n_nationkey from cp.`tpch/nation.parquet` union all select c_custkey from cp.`tpch/customer.parquet` where c_custkey < 5");
+ }
+
+ @Test // Union-All of all columns in the table
+ public void testUnionAll5() throws Exception {
+ test("select * from cp.`tpch/region.parquet` r1 union all select * from cp.`tpch/region.parquet` r2");
+ }
+
+ @Test
+ @Ignore // Produces wrong result
+ public void testUnionAll6() throws Exception {
+ test("select n_nationkey, n_regionkey from cp.`tpch/nation.parquet` where n_regionkey = 1 union all select r_regionkey, r_regionkey from cp.`tpch/region.parquet` where r_regionkey = 2");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/53e218ef/exec/java-exec/src/test/resources/union/test1.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/union/test1.json b/exec/java-exec/src/test/resources/union/test1.json
index 5b33232..a4dcc08 100644
--- a/exec/java-exec/src/test/resources/union/test1.json
+++ b/exec/java-exec/src/test/resources/union/test1.json
@@ -40,7 +40,7 @@
{
@id:4,
children: [1,3],
- pop:"union"
+ pop:"union-all"
},
{
@id:5,
@@ -54,4 +54,4 @@
pop: "screen"
}
]
-}
\ No newline at end of file
+}