You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by so...@apache.org on 2018/09/14 22:45:08 UTC

[drill] branch master updated (f88a73c -> b28355c)

This is an automated email from the ASF dual-hosted git repository.

sorabh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.


    from f88a73c  DRILL-6705: Fix various failures in Crypto / Network / Phonetic functions when invalid input is given
     new 2bb0294  DRILL-6733: Unit tests from KafkaFilterPushdownTest are failing in some environments.
     new c6df1d0  DRILL-6732: Queries are runnable on disable plugins (#1460)
     new 12af476  DRILL-6544: Allow timestamp / date / time formatting when displaying on Web UI
     new b28355c  DRILL-1248: Allow positional / named aliases in group by / having clauses

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../drill/exec/store/kafka/KafkaTestBase.java      |   4 +-
 .../org/apache/calcite/jdbc/DynamicRootSchema.java |   2 +-
 .../java/org/apache/drill/exec/ExecConstants.java  |  22 ++++
 .../drill/exec/planner/sql/DrillConformance.java   |  25 +++-
 .../exec/server/options/SystemOptionManager.java   |   3 +
 .../drill/exec/server/options/TypeValidators.java  |  26 ++++
 .../drill/exec/server/rest/WebUserConnection.java  |   6 +-
 .../exec/store/StoragePluginRegistryImpl.java      |   1 +
 .../exec/util/ValueVectorElementFormatter.java     | 121 +++++++++++++++++
 .../java-exec/src/main/resources/drill-module.conf |   3 +
 .../drill/exec/fn/impl/TestAggregateFunctions.java |   4 +-
 .../org/apache/drill/exec/sql/TestConformance.java |  68 ++++++++--
 .../drill/exec/store/store/TestDisabledPlugin.java |  80 ++++++++++++
 .../exec/util/TestValueVectorElementFormatter.java | 145 +++++++++++++++++++++
 .../java/org/apache/drill/test/ClusterFixture.java |   1 +
 .../drill/test/ClusterMockStorageFixture.java      |   1 +
 pom.xml                                            |   2 +-
 17 files changed, 496 insertions(+), 18 deletions(-)
 create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/util/ValueVectorElementFormatter.java
 create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestDisabledPlugin.java
 create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/util/TestValueVectorElementFormatter.java


[drill] 03/04: DRILL-6544: Allow timestamp / date / time formatting when displaying on Web UI

Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git

commit 12af476247789a1decba8d4c715bd6c41d6690e8
Author: Anton Gozhiy <an...@gmail.com>
AuthorDate: Fri Jul 20 16:59:24 2018 +0300

    DRILL-6544: Allow timestamp / date / time formatting when displaying on Web UI
    
    Added the following options that are setting the format pattens:
    web.timestamp.display_format, web.date.display_format, web.time.display_format.
    See https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for the details about acceptable values.
    Default formatting is used, if the corresponding option has empty string.
---
 .../java/org/apache/drill/exec/ExecConstants.java  |  22 ++++
 .../exec/server/options/SystemOptionManager.java   |   3 +
 .../drill/exec/server/options/TypeValidators.java  |  26 ++++
 .../drill/exec/server/rest/WebUserConnection.java  |   6 +-
 .../exec/util/ValueVectorElementFormatter.java     | 121 +++++++++++++++++
 .../java-exec/src/main/resources/drill-module.conf |   3 +
 .../exec/util/TestValueVectorElementFormatter.java | 145 +++++++++++++++++++++
 7 files changed, 325 insertions(+), 1 deletion(-)

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 2ecb3c8..03707b2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -22,6 +22,7 @@ import org.apache.drill.exec.physical.impl.common.HashTable;
 import org.apache.drill.exec.rpc.user.InboundImpersonationManager;
 import org.apache.drill.exec.server.options.OptionValidator;
 import org.apache.drill.exec.server.options.OptionValidator.OptionDescription;
+import org.apache.drill.exec.server.options.TypeValidators.DateTimeFormatValidator;
 import org.apache.drill.exec.server.options.TypeValidators.IntegerValidator;
 import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator;
 import org.apache.drill.exec.server.options.TypeValidators.DoubleValidator;
@@ -684,6 +685,27 @@ public final class ExecConstants {
   public static final String WEB_LOGS_MAX_LINES = "web.logs.max_lines";
   public static final OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE, null);
 
+  public static final String WEB_DISPLAY_FORMAT_TIMESTAMP = "web.display_format.timestamp";
+  public static final OptionValidator WEB_DISPLAY_FORMAT_TIMESTAMP_VALIDATOR = new DateTimeFormatValidator(WEB_DISPLAY_FORMAT_TIMESTAMP,
+      new OptionDescription("Display format template for timestamp. "
+                          + "It will be passed to java.time.format.DateTimeFormatter. "
+                          + "See https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for the details about acceptable patterns. "
+                          + "If empty then the default formatting will be used. (Drill 1.15 and later)"));
+
+  public static final String WEB_DISPLAY_FORMAT_DATE = "web.display_format.date";
+  public static final OptionValidator WEB_DISPLAY_FORMAT_DATE_VALIDATOR = new DateTimeFormatValidator(WEB_DISPLAY_FORMAT_DATE,
+      new OptionDescription("Display format template for date. "
+                          + "It will be passed to java.time.format.DateTimeFormatter. "
+                          + "See https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for the details about acceptable patterns. "
+                          + "If empty then the default formatting will be used. (Drill 1.15 and later)"));
+
+  public static final String WEB_DISPLAY_FORMAT_TIME = "web.display_format.time";
+  public static final OptionValidator WEB_DISPLAY_FORMAT_TIME_VALIDATOR = new DateTimeFormatValidator(WEB_DISPLAY_FORMAT_TIME,
+      new OptionDescription("Display format template for time. "
+                          + "It will be passed to java.time.format.DateTimeFormatter. "
+                          + "See https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for the details about acceptable patterns. "
+                          + "If empty then the default formatting will be used. (Drill 1.15 and later)"));
+
   public static final String CODE_GEN_EXP_IN_METHOD_SIZE = "exec.java.compiler.exp_in_method_size";
   public static final LongValidator CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR = new LongValidator(CODE_GEN_EXP_IN_METHOD_SIZE,
       new OptionDescription("Introduced in Drill 1.8. For queries with complex or multiple expressions in the query logic, this option limits the number of expressions allowed in each method to prevent Drill from generating code that exceeds the Java limit of 64K bytes. If a method approaches the 64K limit, the Java compiler returns a message stating that the code is too large to compile. If queries return such a message, reduce the value of this option at the session level. The default  [...]
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 80b8ceb..1a168c3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -220,6 +220,9 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
       new OptionDefinition(ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST),
       new OptionDefinition(ExecConstants.BULK_LOAD_TABLE_LIST_BULK_SIZE),
       new OptionDefinition(ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR),
+      new OptionDefinition(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP_VALIDATOR),
+      new OptionDefinition(ExecConstants.WEB_DISPLAY_FORMAT_DATE_VALIDATOR),
+      new OptionDefinition(ExecConstants.WEB_DISPLAY_FORMAT_TIME_VALIDATOR),
       new OptionDefinition(ExecConstants.IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR),
       new OptionDefinition(ExecConstants.IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR),
       new OptionDefinition(ExecConstants.IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR),
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
index 106d436..3f8f8a4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
@@ -17,6 +17,7 @@
  */
 package org.apache.drill.exec.server.options;
 
+import java.time.format.DateTimeFormatter;
 import java.util.Set;
 
 import org.apache.drill.shaded.guava.com.google.common.base.Joiner;
@@ -289,6 +290,31 @@ public class TypeValidators {
     }
   }
 
+  /**
+   * Validator that checks if the given DateTime format template is valid.
+   * See {@link DateTimeFormatter} for the acceptable values.
+   */
+  public static class DateTimeFormatValidator extends StringValidator {
+
+    public DateTimeFormatValidator(String name, OptionDescription description) {
+      super(name, description);
+    }
+
+    @Override
+    public void validate(OptionValue v, OptionMetaData metaData, OptionSet manager) {
+      super.validate(v, metaData, manager);
+      if (!v.string_val.isEmpty()) {
+        try {
+          DateTimeFormatter.ofPattern(v.string_val);
+        } catch (IllegalArgumentException e) {
+          throw UserException.validationError()
+              .message("'%s' is not a valid DateTime format pattern: %s", v.string_val, e.getMessage())
+              .build(logger);
+        }
+      }
+    }
+  }
+
   public static abstract class TypeValidator extends OptionValidator {
     private final Kind kind;
 
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
index 80a0a7e..df4c8a5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
@@ -17,6 +17,7 @@
  */
 package org.apache.drill.exec.server.rest;
 
+import org.apache.drill.exec.util.ValueVectorElementFormatter;
 import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
 import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
 import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
@@ -24,6 +25,7 @@ import io.netty.buffer.ByteBuf;
 import io.netty.buffer.DrillBuf;
 import io.netty.channel.ChannelFuture;
 import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.physical.impl.materialize.QueryWritableBatch;
 import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
@@ -106,13 +108,15 @@ public class WebUserConnection extends AbstractDisposableUserClientConnection im
         for (int i = 0; i < loader.getSchema().getFieldCount(); ++i) {
           columns.add(loader.getSchema().getColumn(i).getName());
         }
+        ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(webSessionResources.getSession().getOptions());
         for (int i = 0; i < rows; ++i) {
           final Map<String, String> record = Maps.newHashMap();
           for (VectorWrapper<?> vw : loader) {
             final String field = vw.getValueVector().getMetadata().getNamePart().getName();
+            final TypeProtos.MinorType fieldMinorType = vw.getValueVector().getMetadata().getMajorType().getMinorType();
             final Accessor accessor = vw.getValueVector().getAccessor();
             final Object value = i < accessor.getValueCount() ? accessor.getObject(i) : null;
-            final String display = value == null ? null : value.toString();
+            final String display = value == null ? null : formatter.format(value, fieldMinorType);
             record.put(field, display);
           }
           results.add(record);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ValueVectorElementFormatter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ValueVectorElementFormatter.java
new file mode 100644
index 0000000..bbb13a7
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ValueVectorElementFormatter.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.server.options.OptionManager;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.util.function.BiFunction;
+
+/**
+ * This class is responsible for formatting ValueVector elements.
+ * Specific format templates are taken from the options.
+ * It creates and reuses the concrete formatter instances for performance purposes.
+ */
+public class ValueVectorElementFormatter {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueVectorElementFormatter.class);
+  private final OptionManager options;
+
+  private DateTimeFormatter timestampFormatter;
+  private DateTimeFormatter dateFormatter;
+  private DateTimeFormatter timeFormatter;
+
+  public ValueVectorElementFormatter(OptionManager options) {
+    this.options = options;
+  }
+
+  /**
+   * Formats ValueVector elements in accordance with it's minor type.
+   *
+   * @param value ValueVector element to format
+   * @param minorType the minor type of the element
+   * @return the formatted value, null if failed
+   */
+  public String format(Object value, TypeProtos.MinorType minorType) {
+    switch (minorType) {
+      case TIMESTAMP:
+        if (value instanceof LocalDateTime) {
+          return format((LocalDateTime) value,
+                        options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP),
+                        (v, p) -> v.format(getTimestampFormatter(p)));
+        }
+      case DATE:
+        if (value instanceof LocalDate) {
+          return format((LocalDate) value,
+                        options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE),
+                        (v, p) -> v.format(getDateFormatter(p)));
+        }
+      case TIME:
+        if (value instanceof LocalTime) {
+          return format((LocalTime) value,
+                        options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME),
+                        (v, p) -> v.format(getTimeFormatter(p)));
+        }
+      default:
+        return value.toString();
+    }
+  }
+
+  /**
+   * Formats ValueVector elements using given function.
+   *
+   * @param value ValueVector element to format, casted to it's actual type.
+   * @param formatPattern pattern used for the formatting. If empty then value.toString() will be returned.
+   * @param formatFunction function that takes ValueVector element and format pattern as arguments and returns the formatted string.
+   * @param <T> actual type of the ValueVector element.
+   * @return the formatted value, null if failed
+   */
+  private <T> String format(T value, String formatPattern, BiFunction<T, String, String> formatFunction) {
+    if (formatPattern.isEmpty()) {
+      return value.toString();
+    }
+    String formattedValue = null;
+    try {
+      formattedValue = formatFunction.apply(value, formatPattern);
+    } catch (Exception e) {
+      logger.debug(String.format("Could not format the value '%s' with the pattern '%s': %s", value, formatPattern, e.getMessage()));
+    }
+    return formattedValue;
+  }
+
+  private DateTimeFormatter getTimestampFormatter(String formatPattern) {
+    if (timestampFormatter == null) {
+      timestampFormatter = DateTimeFormatter.ofPattern(formatPattern);
+    }
+    return timestampFormatter;
+  }
+
+  private DateTimeFormatter getDateFormatter(String formatPattern) {
+    if (dateFormatter == null) {
+      dateFormatter = DateTimeFormatter.ofPattern(formatPattern);
+    }
+    return dateFormatter;
+  }
+
+  private DateTimeFormatter getTimeFormatter(String formatPattern) {
+    if (timeFormatter == null) {
+      timeFormatter = DateTimeFormatter.ofPattern(formatPattern);
+    }
+    return timeFormatter;
+  }
+}
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index e9e1940..66ad9e8 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -615,6 +615,9 @@ drill.exec.options: {
     store.kafka.record.reader: "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader",
     store.kafka.poll.timeout: 200,
     web.logs.max_lines: 10000,
+    web.display_format.timestamp: "",
+    web.display_format.date: "",
+    web.display_format.time: "",
     window.enable: true,
     storage.list_files_recursively: false
 }
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestValueVectorElementFormatter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestValueVectorElementFormatter.java
new file mode 100644
index 0000000..5c6666d
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestValueVectorElementFormatter.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.when;
+
+public class TestValueVectorElementFormatter {
+
+  @Mock
+  private OptionManager options;
+
+  @Before
+  public void setup() {
+    MockitoAnnotations.initMocks(this);
+  }
+
+  @Test
+  public void testFormatValueVectorElementTimestampEmptyPattern() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP)).thenReturn("");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedValue = formatter.format(
+        LocalDateTime.of(2012, 11, 5, 13, 0, 30, 120000000),
+        TypeProtos.MinorType.TIMESTAMP);
+    assertEquals("2012-11-05T13:00:30.120", formattedValue);
+  }
+
+  @Test
+  public void testFormatValueVectorElementTimestampValidPattern() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP)).thenReturn("yyyy-MM-dd HH:mm:ss.SS");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedValue = formatter.format(
+        LocalDateTime.of(2012, 11, 5, 13, 0, 30, 120000000),
+        TypeProtos.MinorType.TIMESTAMP);
+    assertEquals("2012-11-05 13:00:30.12", formattedValue);
+  }
+
+  @Test
+  public void testFormatValueVectorElementDateEmptyPattern() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE)).thenReturn("");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedValue = formatter.format(
+        LocalDate.of(2012, 11, 5),
+        TypeProtos.MinorType.DATE);
+    assertEquals("2012-11-05", formattedValue);
+  }
+
+  @Test
+  public void testFormatValueVectorElementDateValidPattern() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE)).thenReturn("EEE, MMM d, yyyy");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedValue = formatter.format(
+        LocalDate.of(2012, 11, 5),
+        TypeProtos.MinorType.DATE);
+    assertEquals("Mon, Nov 5, 2012", formattedValue);
+  }
+
+  @Test
+  public void testFormatValueVectorElementDateUnsupportedPattern() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE)).thenReturn("yyyy-MM-dd HH:mm:ss.SS");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedValue = formatter.format(
+        LocalDate.of(2012, 11, 5),
+        TypeProtos.MinorType.DATE);
+    assertNull(formattedValue);
+  }
+
+  @Test
+  public void testFormatValueVectorElementTimeEmptyPattern() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME)).thenReturn("");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedValue = formatter.format(
+        LocalTime.of(13, 0, 30, 120000000),
+        TypeProtos.MinorType.TIME);
+    assertEquals("13:00:30.120", formattedValue);
+  }
+
+  @Test
+  public void testFormatValueVectorElementTimeValidPattern() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME)).thenReturn("h:mm:ss a");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedValue = formatter.format(
+        LocalTime.of(13, 0, 30),
+        TypeProtos.MinorType.TIME);
+    assertEquals("1:00:30 PM", formattedValue);
+  }
+
+  @Test
+  public void testFormatValueVectorElementTimeUnsupportedPattern() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME)).thenReturn("yyyy-MM-dd HH:mm:ss.SS");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedValue = formatter.format(
+        LocalTime.of(13, 0, 30),
+        TypeProtos.MinorType.TIME);
+    assertNull(formattedValue);
+  }
+
+  @Test
+  public void testFormatValueVectorElementAllDateTimeFormats() {
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP)).thenReturn("yyyy-MM-dd HH:mm:ss.SS");
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE)).thenReturn("EEE, MMM d, yyyy");
+    when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME)).thenReturn("h:mm:ss a");
+    ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options);
+    String formattedTimestamp = formatter.format(
+        LocalDateTime.of(2012, 11, 5, 13, 0, 30, 120000000),
+        TypeProtos.MinorType.TIMESTAMP);
+    String formattedDate = formatter.format(
+        LocalDate.of(2012, 11, 5),
+        TypeProtos.MinorType.DATE);
+    String formattedTime = formatter.format(
+        LocalTime.of(13, 0, 30),
+        TypeProtos.MinorType.TIME);
+    assertEquals("2012-11-05 13:00:30.12", formattedTimestamp);
+    assertEquals("Mon, Nov 5, 2012", formattedDate);
+    assertEquals("1:00:30 PM", formattedTime);
+  }
+}


[drill] 02/04: DRILL-6732: Queries are runnable on disable plugins (#1460)

Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git

commit c6df1d0e8654369fa8f4f98a371a08290510c95e
Author: shuifeng lu <lu...@gmail.com>
AuthorDate: Fri Sep 7 15:03:47 2018 +0800

    DRILL-6732: Queries are runnable on disable plugins (#1460)
---
 .../org/apache/calcite/jdbc/DynamicRootSchema.java |  2 +-
 .../exec/store/StoragePluginRegistryImpl.java      |  1 +
 .../drill/exec/store/store/TestDisabledPlugin.java | 80 ++++++++++++++++++++++
 .../java/org/apache/drill/test/ClusterFixture.java |  1 +
 .../drill/test/ClusterMockStorageFixture.java      |  1 +
 5 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/exec/java-exec/src/main/java/org/apache/calcite/jdbc/DynamicRootSchema.java b/exec/java-exec/src/main/java/org/apache/calcite/jdbc/DynamicRootSchema.java
index 5fecfdd..6bef3d5 100644
--- a/exec/java-exec/src/main/java/org/apache/calcite/jdbc/DynamicRootSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/calcite/jdbc/DynamicRootSchema.java
@@ -78,7 +78,7 @@ public class DynamicRootSchema extends DynamicSchema {
     try {
       SchemaPlus schemaPlus = this.plus();
       StoragePlugin plugin = getSchemaFactories().getPlugin(schemaName);
-      if (plugin != null) {
+      if (plugin != null && plugin.getConfig().isEnabled()) {
         plugin.registerSchemas(schemaConfig, schemaPlus);
         return;
       }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
index cf8ea50..c5554f8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
@@ -373,6 +373,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
             logger.debug("Storage plugin name {} is not defined. Skipping plugin initialization.", annotatedClass.getClassName());
             continue;
           }
+          storagePlugin.getConfig().setEnabled(true);
           plugins.put(name, storagePlugin);
           isPluginInitialized = true;
 
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestDisabledPlugin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestDisabledPlugin.java
new file mode 100644
index 0000000..0f31e57
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/store/TestDisabledPlugin.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.store;
+
+import org.apache.drill.categories.SqlTest;
+import org.apache.drill.common.exceptions.UserRemoteException;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.dfs.FileSystemConfig;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterTest;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.apache.drill.exec.util.StoragePluginTestUtils.CP_PLUGIN_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category(SqlTest.class)
+public class TestDisabledPlugin extends ClusterTest {
+  private static StoragePluginRegistry pluginRegistry;
+  private static FileSystemConfig pluginConfig;
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    startCluster(ClusterFixture.builder(dirTestWatcher));
+    pluginRegistry = cluster.drillbit().getContext().getStorage();
+    pluginConfig = (FileSystemConfig) pluginRegistry.getPlugin(CP_PLUGIN_NAME).getConfig();
+    pluginConfig.setEnabled(false);
+    pluginRegistry.createOrUpdate(CP_PLUGIN_NAME, pluginConfig, true);
+  }
+
+  @AfterClass
+  public static void restore() throws Exception {
+    pluginConfig.setEnabled(true);
+    pluginRegistry.createOrUpdate(CP_PLUGIN_NAME, pluginConfig, true);
+  }
+
+  @Test
+  public void testDisabledPluginQuery() throws Exception {
+    try {
+      run("SELECT * FROM cp.`employee.json` LIMIT 10");
+      fail("Query should have failed!");
+    } catch (UserRemoteException e) {
+      assertEquals(UserBitShared.DrillPBError.ErrorType.VALIDATION, e.getErrorType());
+      assertTrue("Incorrect error message",
+        e.getMessage().contains("VALIDATION ERROR: Schema"));
+    }
+  }
+
+  @Test
+  public void testUseStatement() throws Exception {
+    try {
+      run("use cp");
+      fail("Query should have failed!");
+    } catch (UserRemoteException e) {
+      assertEquals(UserBitShared.DrillPBError.ErrorType.VALIDATION, e.getErrorType());
+      assertTrue("Incorrect error message",
+        e.getMessage().contains("VALIDATION ERROR: Schema"));
+    }
+  }
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
index 5f06b8f..8e045a2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java
@@ -276,6 +276,7 @@ public class ClusterFixture extends BaseFixture implements AutoCloseable {
     MockStorageEngine plugin = new MockStorageEngine(
         MockStorageEngineConfig.INSTANCE, bit.getContext(),
         MockStorageEngineConfig.NAME);
+    config.setEnabled(true);
     ((StoragePluginRegistryImpl) pluginRegistry).addPluginToPersistentStoreIfAbsent(MockStorageEngineConfig.NAME, config, plugin);
   }
 
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterMockStorageFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterMockStorageFixture.java
index 03b0828..4a548b3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterMockStorageFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterMockStorageFixture.java
@@ -42,6 +42,7 @@ public class ClusterMockStorageFixture extends ClusterFixture {
       @SuppressWarnings("resource")
       MockBreakageStorage plugin = new MockBreakageStorage(
           MockStorageEngineConfig.INSTANCE, bit.getContext(), name);
+      config.setEnabled(true);
       ((StoragePluginRegistryImpl) pluginRegistry).addPluginToPersistentStoreIfAbsent(name, config, plugin);
 
       plugin.setBreakRegister(breakRegisterSchema);


[drill] 01/04: DRILL-6733: Unit tests from KafkaFilterPushdownTest are failing in some environments.

Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git

commit 2bb0294026960d0cec3bb78516b168ff231d2096
Author: Anton Gozhiy <an...@gmail.com>
AuthorDate: Tue Sep 11 12:31:41 2018 +0300

    DRILL-6733: Unit tests from KafkaFilterPushdownTest are failing in some environments.
    
    Added a check that prevents the cluster tear down if TestKafkaSuit is not running.
---
 .../test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java
index 24e6f6d..9f06606 100644
--- a/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java
+++ b/contrib/storage-kafka/src/test/java/org/apache/drill/exec/store/kafka/KafkaTestBase.java
@@ -86,7 +86,9 @@ public class KafkaTestBase extends PlanTestBase {
 
   @AfterClass
   public static void tearDownKafkaTestBase() throws Exception {
-    TestKafkaSuit.tearDownCluster();
+    if (TestKafkaSuit.isRunningSuite()) {
+      TestKafkaSuit.tearDownCluster();
+    }
   }
 
 }


[drill] 04/04: DRILL-1248: Allow positional / named aliases in group by / having clauses

Posted by so...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git

commit b28355c0581529168a5962d3f41242166033a01c
Author: Arina Ielchiieva <ar...@gmail.com>
AuthorDate: Thu Aug 30 13:46:59 2018 +0300

    DRILL-1248: Allow positional / named aliases in group by / having clauses
---
 .../drill/exec/planner/sql/DrillConformance.java   | 25 ++++++--
 .../drill/exec/fn/impl/TestAggregateFunctions.java |  4 +-
 .../org/apache/drill/exec/sql/TestConformance.java | 68 +++++++++++++++++++---
 pom.xml                                            |  2 +-
 4 files changed, 84 insertions(+), 15 deletions(-)

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java
index e6efeb9..4a6aefc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillConformance.java
@@ -21,10 +21,7 @@ import org.apache.calcite.sql.validate.SqlConformanceEnum;
 import org.apache.calcite.sql.validate.SqlDelegatingConformance;
 
 /**
- * Drill's SQL conformance is SqlConformanceEnum.DEFAULT except for method isApplyAllowed().
- * Since Drill is going to allow OUTER APPLY and CROSS APPLY to allow each row from left child of Join
- * to join with output of right side (sub-query or table function that will be invoked for each row).
- * Refer to DRILL-5999 for more information.
+ * Drill's SQL conformance is SqlConformanceEnum.DEFAULT with a couple of deviations.
  */
 public class DrillConformance extends SqlDelegatingConformance {
 
@@ -36,8 +33,28 @@ public class DrillConformance extends SqlDelegatingConformance {
     super(flavor);
   }
 
+  /**
+   * Drill allows OUTER APPLY and CROSS APPLY to allow each row from left child of Join
+   * to join with output of right side (sub-query or table function that will be invoked for each row).
+   * Refer to DRILL-5999 for more information.
+   */
   @Override
   public boolean isApplyAllowed() {
     return true;
   }
+
+  @Override
+  public boolean isGroupByOrdinal() {
+    return true;
+  }
+
+  @Override
+  public boolean isGroupByAlias() {
+    return true;
+  }
+
+  @Override
+  public boolean isHavingAlias() {
+    return true;
+  }
 }
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
index d9564b7..f1f74a6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java
@@ -447,7 +447,7 @@ public class TestAggregateFunctions extends BaseTestQuery {
       alterSession(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY, true);
       testBuilder()
           .sqlQuery("select sum(cast(a as decimal(9,0))) as s,\n" +
-              "avg(cast(a as decimal(9,0))) as a,\n" +
+              "avg(cast(a as decimal(9,0))) as av,\n" +
               "var_samp(cast(a as decimal(9,0))) as varSamp,\n" +
               "var_pop(cast(a as decimal(9,0))) as varPop,\n" +
               "stddev_pop(cast(a as decimal(9,0))) as stddevPop,\n" +
@@ -455,7 +455,7 @@ public class TestAggregateFunctions extends BaseTestQuery {
               "max(cast(a as decimal(9,0))) as mx," +
             "min(cast(a as decimal(9,0))) as mn from dfs.`%s` t group by a", fileName)
           .unOrdered()
-          .baselineColumns("s", "a", "varSamp", "varPop", "stddevPop", "stddevSamp", "mx", "mn")
+          .baselineColumns("s", "av", "varSamp", "varPop", "stddevPop", "stddevSamp", "mx", "mn")
           .baselineValues(BigDecimal.valueOf(1), new BigDecimal("1.000000"), new BigDecimal("0.000000"),
               new BigDecimal("0.000000"), new BigDecimal("0.000000"), new BigDecimal("0.000000"),
               BigDecimal.valueOf(1), BigDecimal.valueOf(1))
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestConformance.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestConformance.java
index 4af1a84..f058bd7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestConformance.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestConformance.java
@@ -17,30 +17,82 @@
  */
 package org.apache.drill.exec.sql;
 
-import org.apache.drill.PlanTestBase;
 import org.apache.drill.categories.SqlTest;
-import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.ClusterTest;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import static org.junit.Assert.assertTrue;
+
 @Category(SqlTest.class)
-public class TestConformance extends BaseTestQuery {
+public class TestConformance extends ClusterTest {
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
+    startCluster(builder);
+  }
 
   @Test
   public void testApply() throws Exception{
-
     //cross join is not support yet in Drill: DRILL-1921, so we are testing OUTER APPLY only
     String query = "SELECT c.c_nationkey, o.orderdate from " +
       "cp.`tpch/customer.parquet` c outer apply " +
       "cp.`tpch/orders.parquet` o " +
       "where c.c_custkey = o.o_custkey";
 
-    PlanTestBase.testPlanMatchingPatterns(query,
-        new String[] {"Join"}, new String[] {}
-    );
+    String plan = queryBuilder().sql(query).explainText();
+    assertTrue(plan.contains("Join(condition="));
+  }
 
-    return;
+  @Test
+  public void testGroupByWithPositionalAlias() throws Exception {
+    testBuilder()
+        .sqlQuery("select length(n_name), n_regionkey from cp.`tpch/nation.parquet` group by 1, 2")
+        .unOrdered()
+        .sqlBaselineQuery("select length(n_name), n_regionkey from cp.`tpch/nation.parquet` group by length(n_name), n_regionkey")
+        .go();
+  }
+
+  @Test
+  public void testGroupByWithNamedAlias() throws Exception {
+    testBuilder()
+        .sqlQuery("select length(n_name) as len, n_regionkey as key from cp.`tpch/nation.parquet` group by len, key")
+        .unOrdered()
+        .sqlBaselineQuery("select length(n_name) as len, n_regionkey as key from cp.`tpch/nation.parquet` group by length(n_name), n_regionkey")
+        .go();
   }
 
+  @Test
+  public void testHavingWithNamedAlias() throws Exception {
+    testBuilder()
+        .sqlQuery("select length(n_name) as len, count(*) as cnt from cp.`tpch/nation.parquet` " +
+            "group by length(n_name) having cnt > 1")
+        .unOrdered()
+        .sqlBaselineQuery("select length(n_name) as len, count(*) as cnt from cp.`tpch/nation.parquet` " +
+            "group by length(n_name) having count(*) > 1")
+        .go();
+  }
+
+  @Test
+  public void testOrderWithPositionalAlias() throws Exception {
+    testBuilder()
+        .sqlQuery("select n_regionkey, n_name from cp.`tpch/nation.parquet` order by 1, 2")
+        .unOrdered()
+        .sqlBaselineQuery("select n_regionkey, n_name from cp.`tpch/nation.parquet` order by n_regionkey, n_name")
+        .go();
+  }
+
+  @Test
+  public void testOrderWithNamedAlias() throws Exception {
+    testBuilder()
+        .sqlQuery("select n_regionkey as r, n_name as n from cp.`tpch/nation.parquet` order by r, n")
+        .unOrdered()
+        .sqlBaselineQuery("select n_regionkey as r, n_name as n from cp.`tpch/nation.parquet` order by n_regionkey, n_name")
+        .go();
+  }
 
 }
diff --git a/pom.xml b/pom.xml
index 5bb908f..a6e1f41 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,7 +46,7 @@
     <guava.version>19.0</guava.version>
     <forkCount>2</forkCount>
     <parquet.version>1.10.0</parquet.version>
-    <calcite.version>1.17.0-drill-r0</calcite.version>
+    <calcite.version>1.17.0-drill-r1</calcite.version>
     <avatica.version>1.12.0</avatica.version>
     <janino.version>2.7.6</janino.version>
     <sqlline.version>1.1.9-drill-r7</sqlline.version>