You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/06/29 08:46:52 UTC
[5/5] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Conflicts:
CHANGES
tajo-common/src/main/java/org/apache/tajo/SessionVars.java
tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/806469a2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/806469a2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/806469a2
Branch: refs/heads/index_support
Commit: 806469a26f7228b0ce7b6f0134dd72ee109accc3
Parents: 17dfe86 773c3eb
Author: Jihoon Son <ji...@apache.org>
Authored: Mon Jun 29 15:29:01 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Mon Jun 29 15:46:39 2015 +0900
----------------------------------------------------------------------
CHANGES | 14 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 4 +-
.../java/org/apache/tajo/catalog/TableMeta.java | 92 +-------
.../main/java/org/apache/tajo/SessionVars.java | 5 +
.../java/org/apache/tajo/conf/TajoConf.java | 7 +-
.../tajo/engine/function/builtin/Corr.java | 224 +++++++++++++++++++
.../planner/physical/BSTIndexScanExec.java | 10 -
.../planner/physical/EmptyScanIterator.java | 40 ++++
.../engine/planner/physical/SeqScanExec.java | 111 +++++----
.../tajo/engine/utils/TupleCacheScanner.java | 114 ----------
.../ws/rs/resources/QueryResultResource.java | 35 +--
tajo-core/src/main/proto/InternalTypes.proto | 9 +
.../engine/function/TestBuiltinFunctions.java | 33 +++
.../tajo/engine/query/TestOuterJoinQuery.java | 2 +-
.../tajo/engine/query/TestTablePartitions.java | 106 ++++-----
.../rs/resources/TestQueryResultResource.java | 73 +-----
.../TestTajoCli/testHelpSessionVars.result | 1 +
.../tajo/plan/LogicalPlanPreprocessor.java | 5 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 10 +-
.../org/apache/tajo/plan/TablePropertyUtil.java | 92 ++++++++
.../org/apache/tajo/plan/util/PlannerUtil.java | 28 ---
.../apache/tajo/storage/AbstractScanner.java | 27 ++-
.../org/apache/tajo/storage/MergeScanner.java | 5 +-
.../org/apache/tajo/storage/NullScanner.java | 8 +-
.../java/org/apache/tajo/storage/Scanner.java | 9 +-
.../apache/tajo/storage/SeekableScanner.java | 4 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 6 +-
.../java/org/apache/tajo/storage/CSVFile.java | 4 +-
.../org/apache/tajo/storage/FileScanner.java | 6 -
.../org/apache/tajo/storage/FileTablespace.java | 7 +-
.../java/org/apache/tajo/storage/RawFile.java | 7 +
.../java/org/apache/tajo/storage/RowFile.java | 7 +
.../apache/tajo/storage/avro/AvroScanner.java | 7 +
.../tajo/storage/parquet/ParquetScanner.java | 7 +
.../org/apache/tajo/storage/rcfile/RCFile.java | 7 +
.../sequencefile/SequenceFileScanner.java | 7 +
.../tajo/storage/text/DelimitedTextFile.java | 7 +-
.../apache/tajo/storage/TestMergeScanner.java | 2 +-
.../org/apache/tajo/storage/TestStorages.java | 10 +-
39 files changed, 667 insertions(+), 485 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index ffed05c,90ef54a..746ed92
--- a/CHANGES
+++ b/CHANGES
@@@ -161,10 -166,9 +166,13 @@@ Release 0.11.0 - unrelease
BUG FIXES
+ TAJO-1608: Fix test failure in index_support branch. (jihoon)
+
+ TAJO-1594: Catalog schema is invalid for some databases. (jihoon)
+
+ TAJO-1644: When inserting empty data into a partitioned table,
+ existing data would be removed. (jaehwa)
+
TAJO-1642: CatalogServer need to check meta table first. (jaehwa)
TAJO-1650: TestQueryResource.testGetAllQueries() occasionally fails.
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/SessionVars.java
index ba6a0cd,28fdb0b..9fa14c2
--- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
+++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
@@@ -126,10 -126,10 +126,15 @@@ public enum SessionVars implements Conf
NULL_CHAR(ConfVars.$TEXT_NULL, "null char of text file output", DEFAULT),
CODEGEN(ConfVars.$CODEGEN, "Runtime code generation enabled (experiment)", DEFAULT),
+ // for index
+ INDEX_ENABLED(ConfVars.$INDEX_ENABLED, "index scan enabled", DEFAULT),
+ INDEX_SELECTIVITY_THRESHOLD(ConfVars.$INDEX_SELECTIVITY_THRESHOLD, "the selectivity threshold for index scan", DEFAULT),
+
++ // for partition overwrite
+ PARTITION_NO_RESULT_OVERWRITE_ENABLED(ConfVars.$PARTITION_NO_RESULT_OVERWRITE_ENABLED,
+ "If True, a partitioned table is overwritten even if a sub query leads to no result. "
+ + "Otherwise, the table data will be kept if there is no result", DEFAULT),
+
// Behavior Control ---------------------------------------------------------
ARITHABORT(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT,
"If true, a running query will be terminated when an overflow or divide-by-zero occurs.", DEFAULT),
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
index 521b6b9,54abca8..19da87b
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
@@@ -96,98 -67,12 +96,88 @@@ public class BSTIndexScanExec extends P
this.reader.open();
}
+ private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, Target[] targets, EvalNode qual) {
+ Schema mergedSchema = new Schema();
+ Set<Column> qualAndTargets = TUtil.newHashSet();
+ qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(qual));
+ for (Target target : targets) {
+ qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree()));
+ }
+ for (Column column : originalSchema.getRootColumns()) {
+ if (subSchema.contains(column)
+ || qualAndTargets.contains(column)
+ || qualAndTargets.contains(column)) {
+ mergedSchema.addColumn(column);
+ }
+ }
+ return mergedSchema;
+ }
+
@Override
public void init() throws IOException {
+ Schema projected;
+
+ // in the case where projected column or expression are given
+ // the target can be an empty list.
+ if (plan.hasTargets()) {
+ projected = new Schema();
+ Set<Column> columnSet = new HashSet<Column>();
+
+ if (plan.hasQual()) {
+ columnSet.addAll(EvalTreeUtil.findUniqueColumns(qual));
+ }
+
+ for (Target t : plan.getTargets()) {
+ columnSet.addAll(EvalTreeUtil.findUniqueColumns(t.getEvalTree()));
+ }
+
+ for (Column column : inSchema.getAllColumns()) {
+ if (columnSet.contains(column)) {
+ projected.addColumn(column);
+ }
+ }
+
+ } else {
+ // no any projected columns, meaning that all columns should be projected.
+ // TODO - this implicit rule makes code readability bad. So, we should remove it later
+ projected = outSchema;
+ }
+
+ initScanner(projected);
super.init();
progress = 0.0f;
- if (qual != null) {
- qual.bind(context.getEvalContext(), inSchema);
+
+ if (plan.hasQual()) {
+ if (fileScanner.isProjectable()) {
+ qual.bind(context.getEvalContext(), projected);
+ } else {
+ qual.bind(context.getEvalContext(), inSchema);
+ }
+ }
+ }
+
+ private void initScanner(Schema projected) throws IOException {
+
- TableMeta meta;
- try {
- meta = (TableMeta) plan.getTableDesc().getMeta().clone();
- } catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
-
- // set system default properties
- PlannerUtil.applySystemDefaultToTableProperties(context.getQueryContext(), meta);
-
+ // Why we should check nullity? See https://issues.apache.org/jira/browse/TAJO-1422
+ if (fragment != null) {
+
+ Schema fileScanOutSchema = mergeSubSchemas(projected, keySchema, plan.getTargets(), qual);
+
+ this.fileScanner = OldStorageManager.getStorageManager(context.getConf(),
+ plan.getTableDesc().getMeta().getStoreType())
+ .getSeekableScanner(plan.getTableDesc().getMeta(), plan.getPhysicalSchema(), fragment, fileScanOutSchema);
+ this.fileScanner.init();
+
+ // See Scanner.isProjectable() method Depending on the result of isProjectable(),
+ // the width of retrieved tuple is changed.
+ //
+ // If TRUE, the retrieved tuple will contain only projected fields.
+ // If FALSE, the retrieved tuple will contain projected fields and NullDatum for non-projected fields.
+ if (fileScanner.isProjectable()) {
+ this.projector = new Projector(context, projected, outSchema, plan.getTargets());
+ } else {
+ this.projector = new Projector(context, inSchema, outSchema, plan.getTargets());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
----------------------------------------------------------------------
diff --cc tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
index 46c879c,5c2ffe3..1f878f1
--- a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
@@@ -35,8 -35,7 +35,9 @@@ Available Session Variables
\set MAX_OUTPUT_FILE_SIZE [int value] - Maximum per-output file size (mb). 0 means infinite.
\set NULL_CHAR [text value] - null char of text file output
\set CODEGEN [true or false] - Runtime code generation enabled (experiment)
+\set INDEX_ENABLED [true or false] - index scan enabled
+\set INDEX_SELECTIVITY_THRESHOLD [real value] - the selectivity threshold for index scan
+ \set PARTITION_NO_RESULT_OVERWRITE_ENABLED [true or false] - If True, a partitioned table is overwritten even if a sub query leads to no result. Otherwise, the table data will be kept if there is no result
\set ARITHABORT [true or false] - If true, a running query will be terminated when an overflow or divide-by-zero occurs.
\set FETCH_ROWNUM [int value] - Sets the number of rows at a time from Master
\set BLOCK_ON_RESULT [true or false] - Whether to block result set on query execution
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------