You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by js...@apache.org on 2016/03/09 19:48:24 UTC

[2/2] drill git commit: DRILL-4441: Fix varchar data read out of Avro filtering incorrectly due to metadata bug

DRILL-4441: Fix varchar data read out of Avro filtering incorrectly due to metadata bug

The precision of the Varchar datatype was not being set causing inconsistent
truncation of values to the default length of 1. Fixed the same issue with varbinary.

The test framework was previously taking a string as the baseline for a binary value,
which cannot express all possible values. Fixed the test to intstead use a byte array.
Thie required updating the hive tests that were using the old method of specifying
baselines with a String.

Fix cast to varbinary when reading from a data source with schema needed for writing
a test.

Updated patch to remove varchar lengths from table creation.
This issue was fixed more generally by DRILL-4465, which provides a default
type length for varchar and varbinary during the setup of calcite. This update now
just provides tests to verify the fix in this case.

Closes #393


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2bf48113
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2bf48113
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2bf48113

Branch: refs/heads/master
Commit: 2bf48113a2b666075ec27184a5c1f70b8979b1e6
Parents: c047f04
Author: Jason Altekruse <al...@gmail.com>
Authored: Fri Feb 26 09:55:05 2016 -0800
Committer: Jason Altekruse <al...@gmail.com>
Committed: Wed Mar 9 10:46:51 2016 -0800

----------------------------------------------------------------------
 .../drill/exec/fn/hive/TestInbuiltHiveUDFs.java |  2 +-
 .../apache/drill/exec/hive/TestHiveStorage.java |  4 +-
 .../drill/exec/planner/logical/DrillOptiq.java  |  1 +
 .../java/org/apache/drill/DrillTestWrapper.java |  3 -
 .../drill/exec/store/avro/AvroFormatTest.java   | 66 ++++++++++++++++++++
 .../drill/exec/store/avro/AvroTestUtil.java     |  5 +-
 6 files changed, 73 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/2bf48113/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
index 3ee832c..aba7573 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java
@@ -39,7 +39,7 @@ public class TestInbuiltHiveUDFs extends HiveTestBase {
         .sqlQuery("SELECT encode(varchar_field, 'UTF-8') as rst from hive.readtest")
         .unOrdered()
         .baselineColumns("rst")
-        .baselineValues("varcharfield")
+        .baselineValues("varcharfield".getBytes())
         .baselineValues(new Object[] { null })
         .go();
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/2bf48113/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index 078c375..3e14846 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -134,7 +134,7 @@ public class TestHiveStorage extends HiveTestBase {
             "timestamp_part",
             "date_part")
         .baselineValues(
-            "binaryfield",
+            "binaryfield".getBytes(),
             false,
             34,
             new BigDecimal("66"),
@@ -244,7 +244,7 @@ public class TestHiveStorage extends HiveTestBase {
               "timestamp_part",
               "date_part")
           .baselineValues(
-              "binaryfield",
+              "binaryfield".getBytes(),
               false,
               34,
               new BigDecimal("66"),

http://git-wip-us.apache.org/repos/asf/drill/blob/2bf48113/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 43d8b6b..bd029bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -295,6 +295,7 @@ public class DrillOptiq {
         case "INTERVAL_YEAR_MONTH": castType = Types.required(MinorType.INTERVALYEAR); break;
         case "INTERVAL_DAY_TIME": castType = Types.required(MinorType.INTERVALDAY); break;
         case "BOOLEAN": castType = Types.required(MinorType.BIT); break;
+        case "BINARY": castType = Types.required(MinorType.VARBINARY).toBuilder().setWidth(call.getType().getPrecision()).build(); break;
         case "ANY": return arg; // Type will be same as argument.
         default: castType = Types.required(MinorType.valueOf(call.getType().getSqlTypeName().getName()));
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/2bf48113/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
index 41d3ff6..67017ce 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
@@ -516,9 +516,6 @@ public class DrillTestWrapper {
             if (obj instanceof Text) {
               obj = obj.toString();
             }
-            else if (obj instanceof byte[]) {
-              obj = new String((byte[]) obj, "UTF-8");
-            }
             record.put(SchemaPath.getSimplePath(w.getField().getPath()).toExpr(), obj);
           }
           record.put(SchemaPath.getSimplePath(w.getField().getPath()).toExpr(), obj);

http://git-wip-us.apache.org/repos/asf/drill/blob/2bf48113/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
index ecf8369..b68f19d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
@@ -18,6 +18,7 @@
 package org.apache.drill.exec.store.avro;
 
 import org.apache.drill.BaseTestQuery;
+import org.apache.drill.TestBuilder;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.exceptions.UserRemoteException;
 import org.junit.Assert;
@@ -49,6 +50,71 @@ public class AvroFormatTest extends BaseTestQuery {
         .go();
   }
 
+  /**
+   * Previously a bug in the Avro table metadata would cause wrong results
+   * for some queries on varchar types, as a length was not provided during metadata
+   * population. In some cases casts were being added with the default length
+   * of 1 and truncating values.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testFiltersOnVarchar() throws Exception {
+
+    final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues(5000);
+    final String file = testSetup.getFilePath();
+    final String sql =
+        "select a_string " +
+            "from dfs_test.`" + file + "` where a_string = 'a_1'";
+    testBuilder()
+        .sqlQuery(sql)
+        .unOrdered()
+        .baselineColumns("a_string")
+        .baselineValues("a_1")
+        .go();
+
+    final String sql2 =
+        "select a_string " +
+            "from dfs_test.`" + file + "` where a_string IN ('a_1')";
+    testBuilder()
+        .sqlQuery(sql2)
+        .unOrdered()
+        .baselineColumns("a_string")
+        .baselineValues("a_1")
+        .go();
+  }
+
+  @Test
+  public void testFiltersOnVarBinary() throws Exception {
+    final AvroTestUtil.AvroTestRecordWriter testSetup = AvroTestUtil.generateSimplePrimitiveSchema_NoNullValues(5000);
+    final String file = testSetup.getFilePath();
+    final String sql =
+        "select f_bytes " +
+            "from dfs_test.`" + file + "` where f_bytes = BINARY_STRING('\\x61\\x31')";
+    TestBuilder testBuilder = testBuilder()
+        .sqlQuery(sql)
+        .unOrdered()
+        .baselineColumns("f_bytes");
+
+    for (int i = 0; i < 500; i++) {
+      testBuilder.baselineValues(new byte[] {'a', '1'});
+    }
+    testBuilder.go();
+
+    final String sql2 =
+        "select f_bytes " +
+            "from dfs_test.`" + file + "` where f_bytes IN (BINARY_STRING('\\x61\\x31'))";
+    testBuilder = testBuilder()
+        .sqlQuery(sql2)
+        .unOrdered()
+        .baselineColumns("f_bytes");
+
+    for (int i = 0; i < 500; i++) {
+      testBuilder.baselineValues(new byte[] {'a', '1'});
+    }
+    testBuilder.go();
+  }
+
   @Test
   public void testSimplePrimitiveSchema_NoNullValues() throws Exception {
 

http://git-wip-us.apache.org/repos/asf/drill/blob/2bf48113/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java
index b6736d6..ce3f90a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroTestUtil.java
@@ -94,7 +94,7 @@ public class AvroTestUtil {
         byte[] drillVal = new byte[((ByteBuffer)value).remaining()];
         bb.get(drillVal);
         bb.position(0);
-        value = new String(drillVal, Charsets.UTF_8);
+        value = drillVal;
       }
       currentExpectedRecord.put("`" + key + "`", value);
     }
@@ -143,10 +143,11 @@ public class AvroTestUtil {
 
     final AvroTestRecordWriter record = new AvroTestRecordWriter(schema, file);
     try {
-      ByteBuffer bb = ByteBuffer.allocate(1);
+      ByteBuffer bb = ByteBuffer.allocate(2);
       bb.put(0, (byte) 'a');
 
       for (int i = 0; i < numRecords; i++) {
+        bb.put(1, (byte) ('0' + (i % 10)));
         bb.position(0);
         record.startRecord();
         record.put("a_string", "a_" + i);