You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/12 01:08:51 UTC
[4/8] git commit: Fix hash join functions for variable types to use
the correct buffer.
Fix hash join functions for variable types to use the correct buffer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/3eaf6d63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/3eaf6d63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/3eaf6d63
Branch: refs/heads/master
Commit: 3eaf6d6348d05e5c8a2f1d60fd3c19df5b4889b6
Parents: a694c0a
Author: vkorukanti <ve...@gmail.com>
Authored: Sat May 10 16:02:03 2014 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Sat May 10 16:02:03 2014 -0700
----------------------------------------------------------------------
.../drill/exec/expr/fn/impl/HashFunctions.java | 37 +++++++++++++--
.../exec/physical/impl/writer/TestWriter.java | 49 ++++++++++++++------
2 files changed, 68 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3eaf6d63/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
index 36300ee..766a32a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
@@ -102,7 +102,7 @@ public class HashFunctions {
if (in.isSet == 0)
out.value = 0;
else
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(), 0);
+ out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
}
}
@@ -119,7 +119,24 @@ public class HashFunctions {
if (in.isSet == 0)
out.value = 0;
else
- out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(), 0);
+ out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableVar16CharHash implements DrillSimpleFunc {
+
+ @Param NullableVar16CharHolder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ if (in.isSet == 0)
+ out.value = 0;
+ else
+ out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
}
}
@@ -185,7 +202,21 @@ public class HashFunctions {
out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
}
}
-
+
+ @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Var16CharHash implements DrillSimpleFunc {
+
+ @Param Var16CharHolder in;
+ @Output IntHolder out;
+
+ public void setup(RecordBatch incoming) {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(in.start, in.end - in.start), 0);
+ }
+ }
+
@FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
public static class HashBigInt implements DrillSimpleFunc {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/3eaf6d63/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
index 54e6b7f..6ddb73d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
@@ -90,46 +90,65 @@ public class TestWriter extends BaseTestQuery {
@Test
public void simpleCTAS() throws Exception {
- String testQuery = "USE dfs.tmp;" +
- "CREATE TABLE simplectas AS SELECT * FROM cp.`employee.json`;";
+ testSqlWithResults("Use dfs.tmp");
- ctasHelper("/tmp/drilltest/simplectas", testQuery, 1);
+ String testQuery = "CREATE TABLE simplectas AS SELECT * FROM cp.`employee.json`";
+
+ ctasHelper("/tmp/drilltest/simplectas", testQuery, 1155);
}
@Test
public void complex1CTAS() throws Exception {
- String testQuery = "USE dfs.tmp;" +
- "CREATE TABLE complex1ctas AS SELECT first_name, last_name, position_id FROM cp.`employee.json`;";
+ testSqlWithResults("Use dfs.tmp");
+ String testQuery = "CREATE TABLE complex1ctas AS SELECT first_name, last_name, position_id FROM cp.`employee.json`";
- ctasHelper("/tmp/drilltest/complex1ctas", testQuery, 1);
+ ctasHelper("/tmp/drilltest/complex1ctas", testQuery, 1155);
}
@Test
public void complex2CTAS() throws Exception {
- String testQuery = "USE dfs.tmp;" +
- "CREATE TABLE complex2ctas AS SELECT CAST(`birth_date` as Timestamp) FROM cp.`employee.json` GROUP BY birth_date;";
+ testSqlWithResults("Use dfs.tmp");
+ String testQuery = "CREATE TABLE complex2ctas AS SELECT CAST(`birth_date` as Timestamp) FROM cp.`employee.json` GROUP BY birth_date";
- ctasHelper("/tmp/drilltest/complex2ctas", testQuery, 3);
+ ctasHelper("/tmp/drilltest/complex2ctas", testQuery, 52);
}
@Test
public void simpleCTASWithSchemaInTableName() throws Exception {
- String testQuery = "CREATE TABLE dfs.tmp.`/test/simplectas2` AS SELECT * FROM cp.`employee.json`;";
+ String testQuery = "CREATE TABLE dfs.tmp.`/test/simplectas2` AS SELECT * FROM cp.`employee.json`";
- ctasHelper("/tmp/drilltest/test/simplectas2", testQuery, 1);
+ ctasHelper("/tmp/drilltest/test/simplectas2", testQuery, 1155);
}
- private void ctasHelper(String tableDir, String testQuery, int numExpectedFiles) throws Exception {
+ private void ctasHelper(String tableDir, String testQuery, int expectedOutputCount) throws Exception {
Path tableLocation = new Path(tableDir);
if (fs.exists(tableLocation)){
fs.delete(tableLocation, true);
}
- test(testQuery);
+ List<QueryResultBatch> results = testSqlWithResults(testQuery);
+
+ RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
+
+ int recordsWritten = 0;
+ for(QueryResultBatch batch : results) {
+ batchLoader.load(batch.getHeader().getDef(), batch.getData());
+
+ if (batchLoader.getRecordCount() <= 0)
+ continue;
+
+ BigIntVector recordWrittenV = (BigIntVector) batchLoader.getValueAccessorById(1, BigIntVector.class).getValueVector();
+
+ for (int i = 0; i < batchLoader.getRecordCount(); i++) {
+ recordsWritten += recordWrittenV.getAccessor().get(i);
+ }
+
+ batch.release();
+ }
+ batchLoader.clear();
assertTrue(fs.exists(tableLocation));
- FileStatus[] fileStatuses = fs.globStatus(new Path(tableLocation.toString(), "*.csv"));
- assertEquals(numExpectedFiles, fileStatuses.length);
+ assertEquals(expectedOutputCount, recordsWritten);
}
}