You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by gp...@apache.org on 2019/04/05 02:07:20 UTC
[drill] branch master updated: DRILL-7076: Fix NPE in
StatsMaterializationVisitor
This is an automated email from the ASF dual-hosted git repository.
gparai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new e477480 DRILL-7076: Fix NPE in StatsMaterializationVisitor
e477480 is described below
commit e477480e7b9626abc8efd70914d3bfd4321b7258
Author: Gautam Parai <gp...@maprtech.com>
AuthorDate: Mon Mar 25 17:12:06 2019 -0700
DRILL-7076: Fix NPE in StatsMaterializationVisitor
closes #1722
---
.../impl/statistics/StatisticsAggBatch.java | 10 +++++----
.../drill/exec/planner/common/DrillStatsTable.java | 24 ++++++++++++----------
.../org/apache/drill/exec/sql/TestAnalyze.java | 2 +-
3 files changed, 20 insertions(+), 16 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/StatisticsAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/StatisticsAggBatch.java
index df64a5f..40fc445 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/StatisticsAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/StatisticsAggBatch.java
@@ -44,6 +44,7 @@ import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.store.ColumnExplorer;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.FieldIdUtil;
@@ -90,8 +91,9 @@ public class StatisticsAggBatch extends StreamingAggBatch {
/*
* Returns whether the given column is an implicit column
*/
- private boolean isImplicitFileColumn(MaterializedField mf) {
- return implicitFileColumnsMap.get(SchemaPath.getSimplePath(mf.getName()).toString()) != null;
+ private boolean isImplicitFileOrPartitionColumn(MaterializedField mf, OptionManager optionManager) {
+ return implicitFileColumnsMap.get(SchemaPath.getSimplePath(mf.getName()).toString()) != null ||
+ ColumnExplorer.isPartitionColumn(optionManager, SchemaPath.getSimplePath(mf.getName()));
}
/*
@@ -191,7 +193,7 @@ public class StatisticsAggBatch extends StreamingAggBatch {
expr = ValueExpressions.getChar(DrillStatsTable.getMapper().writeValueAsString(mf.getType()), 0);
}
// Ignore implicit columns
- if (!isImplicitFileColumn(mf)) {
+ if (!isImplicitFileOrPartitionColumn(mf, incoming.getContext().getOptions())) {
createNestedKeyColumn(
parent,
SchemaPath.getSimplePath(mf.getName()).toString(),
@@ -213,7 +215,7 @@ public class StatisticsAggBatch extends StreamingAggBatch {
for (MaterializedField mf : incoming.getSchema()) {
// Check stats collection is only being done for supported data-types. Complex types
// such as MAP, LIST are not supported!
- if (isColMinorTypeValid(mf) && !isImplicitFileColumn(mf)) {
+ if (isColMinorTypeValid(mf) && !isImplicitFileOrPartitionColumn(mf, incoming.getContext().getOptions())) {
List<LogicalExpression> args = Lists.newArrayList();
args.add(SchemaPath.getSimplePath(mf.getName()));
LogicalExpression call = FunctionCallFactory.createExpression(func, args);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java
index 24cb888..65b0b64 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStatsTable.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -33,7 +34,6 @@ import java.util.Map;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.core.TableScan;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.types.TypeProtos;
@@ -196,7 +196,7 @@ public class DrillStatsTable {
* @param context - Query context
* @throws Exception
*/
- public void materialize(final DrillTable table, final QueryContext context) throws IOException {
+ public void materialize(final DrillTable table, final QueryContext context) {
if (materialized) {
return;
}
@@ -223,8 +223,11 @@ public class DrillStatsTable {
materialized = true;
}
} catch (IOException ex) {
- logger.warn("Failed to read the stats file.", ex);
- throw ex;
+ if (ex instanceof FileNotFoundException) {
+ logger.debug(String.format("Did not find statistics file %s", tablePath.toString()), ex);
+ } else {
+ logger.debug(String.format("Error trying to read statistics table %s", tablePath.toString()), ex);
+ }
}
}
@@ -247,16 +250,15 @@ public class DrillStatsTable {
if (node instanceof TableScan) {
try {
final DrillTable drillTable = Utilities.getDrillTable(node.getTable());
- final DrillStatsTable statsTable = drillTable.getStatsTable();
- if (statsTable != null) {
+ final DrillStatsTable statsTable = drillTable != null ? drillTable.getStatsTable() : null;
+ if (drillTable != null && statsTable != null) {
statsTable.materialize(drillTable, context);
- } else {
- throw new DrillRuntimeException(
- String.format("Failed to find the stats for table [%s] in schema [%s]",
- node.getTable().getQualifiedName(), node.getTable().getRelOptSchema()));
+ } else if (drillTable != null) {
+ logger.debug(String.format("Unable to find statistics table info for table [%s] in schema [%s]",
+ node.getTable().getQualifiedName(), node.getTable().getRelOptSchema()));
}
} catch (Exception e) {
- // Log a warning and proceed. We don't want to fail a query.
+ // Something unexpected happened! Log a warning and proceed. We don't want to fail the query.
logger.warn("Failed to materialize the stats. Continuing without stats.", e);
}
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestAnalyze.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestAnalyze.java
index c44dfdd..30d23b3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestAnalyze.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestAnalyze.java
@@ -170,6 +170,7 @@ public class TestAnalyze extends BaseTestQuery {
}
}
+ @Ignore("For 1.16.0, we do not plan to support statistics on dir columns")
@Test
public void testAnalyzePartitionedTables() throws Exception {
//Computing statistics on columns, dir0, dir1
@@ -330,7 +331,6 @@ public class TestAnalyze extends BaseTestQuery {
PlanTestBase.testPlanWithAttributesMatchingPatterns(query, expectedPlan11, new String[]{});
}
- @Ignore("Fails intermittently. Enable after fixing the issue.")
@Test
public void testWithMetadataCaching() throws Exception {
test("ALTER SESSION SET `planner.slice_target` = 1");