You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2015/03/19 00:13:18 UTC
drill git commit: DRILL-2358: Ensure DrillScanRel differentiates
skip-all, scan-all & scan-some in a backward compatible fashion
Repository: drill
Updated Branches:
refs/heads/master 12a7d0179 -> 54df129ca
DRILL-2358: Ensure DrillScanRel differentiates skip-all, scan-all & scan-some in a backward compatible fashion
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/54df129c
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/54df129c
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/54df129c
Branch: refs/heads/master
Commit: 54df129cab544c3df8e75a7dae3f85a91a9ded5a
Parents: 12a7d01
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Thu Mar 5 13:35:48 2015 -0800
Committer: Hanifi Gunes <hg...@maprtech.com>
Committed: Wed Mar 18 14:21:23 2015 -0700
----------------------------------------------------------------------
.../drill/exec/planner/logical/ColumnList.java | 200 +++++++++++++++++++
.../exec/planner/logical/DrillScanRel.java | 11 +-
.../drill/exec/store/AbstractRecordReader.java | 14 ++
3 files changed, 222 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/54df129c/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ColumnList.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ColumnList.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ColumnList.java
new file mode 100644
index 0000000..e448bc5
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ColumnList.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.physical.base.GroupScan;
+
+/**
+ * A list decorator that wraps a list of columns along with a {@link org.apache.drill.exec.planner.logical.ColumnList.Mode mode}
+ * that informs {@link org.apache.drill.exec.store.AbstractRecordReader readers} to scan or skip the underlying list of columns.
+ */
+public class ColumnList implements List<SchemaPath> {
+
+ public static enum Mode {
+ SKIP_ALL,
+ SCAN_ALL,
+ SCAN_SOME
+ }
+
+ private final List<SchemaPath> backend;
+ private final Mode mode;
+
+ protected ColumnList(List<SchemaPath> backend, Mode mode) {
+ this.backend = Preconditions.checkNotNull(backend);
+ this.mode = Preconditions.checkNotNull(mode);
+ }
+
+ public Mode getMode() {
+ return mode;
+ }
+
+ public static ColumnList none() {
+ return new ColumnList(GroupScan.ALL_COLUMNS, Mode.SKIP_ALL);
+ }
+
+ public static ColumnList all() {
+ return new ColumnList(GroupScan.ALL_COLUMNS, Mode.SCAN_ALL);
+ }
+
+ public static ColumnList some(List<SchemaPath> columns) {
+ return new ColumnList(columns, Mode.SCAN_SOME);
+ }
+
+
+ @Override
+ public int size() {
+ return backend.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return backend.isEmpty();
+ }
+
+ @Override
+ public boolean contains(Object o) {
+ return backend.contains(o);
+ }
+
+ @Override
+ public Iterator<SchemaPath> iterator() {
+ return backend.iterator();
+ }
+
+ @Override
+ public Object[] toArray() {
+ return backend.toArray();
+ }
+
+ @Override
+ public <T> T[] toArray(T[] a) {
+ return backend.toArray(a);
+ }
+
+ @Override
+ public boolean add(SchemaPath path) {
+ return backend.add(path);
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ return backend.remove(o);
+ }
+
+ @Override
+ public boolean containsAll(Collection<?> c) {
+ return backend.containsAll(c);
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends SchemaPath> c) {
+ return backend.addAll(c);
+ }
+
+ @Override
+ public boolean addAll(int index, Collection<? extends SchemaPath> c) {
+ return backend.addAll(index, c);
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ return backend.removeAll(c);
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ return backend.retainAll(c);
+ }
+
+ @Override
+ public void clear() {
+ backend.clear();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ColumnList) {
+ final ColumnList other = ColumnList.class.cast(o);
+ return backend.equals(other) && Objects.equal(mode, other.mode);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(backend, mode);
+ }
+
+ @Override
+ public SchemaPath get(int index) {
+ return backend.get(index);
+ }
+
+ @Override
+ public SchemaPath set(int index, SchemaPath element) {
+ return backend.set(index, element);
+ }
+
+ @Override
+ public void add(int index, SchemaPath element) {
+ backend.add(index, element);
+ }
+
+ @Override
+ public SchemaPath remove(int index) {
+ return backend.remove(index);
+ }
+
+ @Override
+ public int indexOf(Object o) {
+ return backend.indexOf(o);
+ }
+
+ @Override
+ public int lastIndexOf(Object o) {
+ return backend.lastIndexOf(o);
+ }
+
+ @Override
+ public ListIterator<SchemaPath> listIterator() {
+ return backend.listIterator();
+ }
+
+ @Override
+ public ListIterator<SchemaPath> listIterator(int index) {
+ return backend.listIterator(index);
+ }
+
+ @Override
+ public List<SchemaPath> subList(int fromIndex, int toIndex) {
+ return backend.subList(fromIndex, toIndex);
+ }
+
+ @Override
+ public String toString() {
+ return backend.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/54df129c/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
index 45e1058..ab3d61b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
@@ -25,7 +25,6 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.LogicalOperator;
import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.ScanStats;
import org.apache.drill.exec.planner.common.DrillScanRelBase;
@@ -59,7 +58,7 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel {
RelOptTable table) {
// By default, scan does not support project pushdown.
// Decision whether push projects into scan will be made solely in DrillPushProjIntoScanRule.
- this(cluster, traits, table, table.getRowType(), AbstractGroupScan.ALL_COLUMNS);
+ this(cluster, traits, table, table.getRowType(), GroupScan.ALL_COLUMNS);
}
/** Creates a DrillScan. */
@@ -67,7 +66,13 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel {
RelOptTable table, RelDataType rowType, List<SchemaPath> columns) {
super(DRILL_LOGICAL, cluster, traits, table);
this.rowType = rowType;
- this.columns = columns == null || columns.size() == 0 ? GroupScan.ALL_COLUMNS : columns;
+ if (columns == null) { // planner asks to scan all of the columns
+ this.columns = ColumnList.all();
+ } else if (columns.size() == 0) { // planner asks to skip all of the columns
+ this.columns = ColumnList.none();
+ } else { // planner asks to scan some columns
+ this.columns = ColumnList.some(columns);
+ }
try {
this.groupScan = drillTable.getGroupScan().clone(this.columns);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/drill/blob/54df129c/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
index 554b3e7..6e27628 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.planner.logical.ColumnList;
import org.apache.drill.exec.record.MaterializedField.Key;
import org.apache.drill.exec.vector.ValueVector;
@@ -36,9 +37,14 @@ public abstract class AbstractRecordReader implements RecordReader {
private Collection<SchemaPath> columns = null;
private boolean isStarQuery = false;
+ private boolean isSkipQuery = false;
protected final void setColumns(Collection<SchemaPath> projected) {
assert Preconditions.checkNotNull(projected, COL_NULL_ERROR).size() > 0 : COL_EMPTY_ERROR;
+ if (projected instanceof ColumnList) {
+ final ColumnList columns = ColumnList.class.cast(projected);
+ isSkipQuery = columns.getMode() == ColumnList.Mode.SKIP_ALL;
+ }
isStarQuery = isStarQuery(projected);
columns = transformColumns(projected);
}
@@ -55,6 +61,14 @@ public abstract class AbstractRecordReader implements RecordReader {
return isStarQuery;
}
+ /**
+ * Returns true if reader should skip all of the columns, reporting number of records only. Handling of a skip query
+ * is storage plugin-specific.
+ */
+ protected boolean isSkipQuery() {
+ return isSkipQuery;
+ }
+
public static boolean isStarQuery(Collection<SchemaPath> projected) {
return Iterables.tryFind(Preconditions.checkNotNull(projected, COL_NULL_ERROR), new Predicate<SchemaPath>() {
@Override