You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2016/01/07 07:01:30 UTC
tajo git commit: TAJO-1991: Tablespace::getVolume should take filter
predication.
Repository: tajo
Updated Branches:
refs/heads/master 5853b47b4 -> 9c14dc3ad
TAJO-1991: Tablespace::getVolume should take filter predication.
Closes #901
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/9c14dc3a
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/9c14dc3a
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/9c14dc3a
Branch: refs/heads/master
Commit: 9c14dc3ad794ace2d2ab46c0b0185f265cc703bf
Parents: 5853b47
Author: Hyunsik Choi <hy...@apache.org>
Authored: Wed Jan 6 21:59:31 2016 -0800
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Wed Jan 6 21:59:31 2016 -0800
----------------------------------------------------------------------
CHANGES | 2 +
.../org/apache/tajo/storage/StorageService.java | 42 -------------------
.../org/apache/tajo/plan/LogicalOptimizer.java | 3 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 1 -
.../org/apache/tajo/plan/StorageService.java | 44 ++++++++++++++++++++
.../rewrite/BaseLogicalPlanRewriteEngine.java | 2 +-
.../rewrite/LogicalPlanRewriteRuleContext.java | 2 +-
.../plan/rewrite/TableStatUpdateRewriter.java | 12 +++---
.../org/apache/tajo/storage/Tablespace.java | 3 +-
.../apache/tajo/storage/TablespaceManager.java | 7 +++-
.../tajo/storage/hbase/HBaseTablespace.java | 2 +-
.../org/apache/tajo/storage/FileTablespace.java | 2 +-
tajo-storage/tajo-storage-jdbc/pom.xml | 1 -
.../tajo/storage/jdbc/JdbcTablespace.java | 3 +-
14 files changed, 67 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 2029fe9..7ca2377 100644
--- a/CHANGES
+++ b/CHANGES
@@ -8,6 +8,8 @@ Release 0.12.0 - unreleased
IMPROVEMENT
+ TAJO-1991: Tablespace::getVolume should take filter predication. (hyunsik)
+
TAJO-1944: Support text resultset for REST (DaeMyung)
TAJO-1950: Query master uses too much memory during range shuffle. (jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
deleted file mode 100644
index 0c3c031..0000000
--- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage;
-
-import org.apache.tajo.exception.UnsupportedException;
-
-import javax.annotation.Nullable;
-import java.net.URI;
-
-/**
- * TablespaceManager interface for loosely coupled usages
- */
-public interface StorageService {
-
- /**
- * Get Table URI
- *
- * @param spaceName Tablespace name. If it is null, the default space will be used
- * @param databaseName Database name
- * @param tableName Table name
- * @return Table URI
- */
- URI getTableURI(@Nullable String spaceName, String databaseName, String tableName);
-
- long getTableVolumn(URI uri) throws UnsupportedException;
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
index 23f8584..65d1759 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
@@ -30,8 +30,8 @@ import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
-import org.apache.tajo.plan.expr.*;
import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.joinorder.*;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteEngine;
@@ -39,7 +39,6 @@ import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleProvider;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
-import org.apache.tajo.storage.StorageService;
import org.apache.tajo.util.ReflectionUtil;
import org.apache.tajo.util.graph.DirectedGraphCursor;
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 50ab2e0..190e12b 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -48,7 +48,6 @@ import org.apache.tajo.plan.nameresolver.NameResolvingMode;
import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule;
import org.apache.tajo.plan.util.ExprFinder;
import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.storage.StorageService;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.StringUtils;
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
new file mode 100644
index 0000000..10d11f0
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan;
+
+import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.expr.EvalNode;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+import java.util.Optional;
+
+/**
+ * TablespaceManager interface for loosely coupled usages
+ */
+public interface StorageService {
+
+ /**
+ * Get Table URI
+ *
+ * @param spaceName Tablespace name. If it is null, the default space will be used
+ * @param databaseName Database name
+ * @param tableName Table name
+ * @return Table URI
+ */
+ URI getTableURI(@Nullable String spaceName, String databaseName, String tableName);
+
+ long getTableVolumn(URI uri, Optional<EvalNode> filter) throws UnsupportedException;
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java
index f38b202..21a0905 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java
@@ -22,7 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.storage.StorageService;
+import org.apache.tajo.plan.StorageService;
import java.util.LinkedHashMap;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
index a899961..aa7308a 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
@@ -21,7 +21,7 @@ package org.apache.tajo.plan.rewrite;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.storage.StorageService;
+import org.apache.tajo.plan.StorageService;
public class LogicalPlanRewriteRuleContext {
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
index d4c4363..3683f60 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
@@ -27,11 +27,13 @@ import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.StorageService;
+import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
-import org.apache.tajo.storage.StorageService;
+import java.util.Optional;
import java.util.Stack;
public class TableStatUpdateRewriter implements LogicalPlanRewriteRule {
@@ -75,15 +77,15 @@ public class TableStatUpdateRewriter implements LogicalPlanRewriteRule {
final TableDesc table = scanNode.getTableDesc();
if (!isVirtual(table)) {
-
final TableStats stats = getTableStat(table);
final long tableSize = stats.getNumBytes();
+ final Optional<EvalNode> filter = scanNode.hasQual() ? Optional.of(scanNode.getQual()) : Optional.empty();
// If USE_TABLE_VOLUME is set, we will update the table volume through a storage handler.
// In addition, if the table size is zero, we will update too.
// It is a good workaround to avoid suboptimal join orders without cheap cost.
if (conf.getBool(SessionVars.USE_TABLE_VOLUME) || tableSize == 0) {
- table.getStats().setNumBytes(getTableVolume(table));
+ table.getStats().setNumBytes(getTableVolume(table, filter));
}
}
@@ -105,10 +107,10 @@ public class TableStatUpdateRewriter implements LogicalPlanRewriteRule {
return table.getMeta().getDataFormat().equals("SYSTEM");
}
- private long getTableVolume(TableDesc table) {
+ private long getTableVolume(TableDesc table, Optional<EvalNode> filter) {
try {
if (table.getStats() != null) {
- return storage.getTableVolumn(table.getUri());
+ return storage.getTableVolumn(table.getUri(), filter);
}
} catch (UnsupportedException t) {
LOG.warn(table.getName() + " does not support Tablespace::getTableVolume()");
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
index 81fbbc0..6c97754 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
@@ -40,6 +40,7 @@ import java.io.IOException;
import java.net.URI;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
/**
@@ -98,7 +99,7 @@ public abstract class Tablespace {
return name + "=" + uri.toString();
}
- public abstract long getTableVolume(URI uri) throws UnsupportedException;
+ public abstract long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException;
/**
* if {@link StorageProperty#isArbitraryPathAllowed} is true,
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
index b1e3275..12e283f 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
@@ -36,6 +36,8 @@ import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.exception.UndefinedTablespaceException;
import org.apache.tajo.exception.UndefinedTablespaceHandlerException;
import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.StorageService;
+import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.Pair;
@@ -433,8 +435,9 @@ public class TablespaceManager implements StorageService {
}
@Override
- public long getTableVolumn(URI tableUri) throws UnsupportedException {
- return get(tableUri).getTableVolume(tableUri);
+ public long getTableVolumn(URI tableUri, Optional<EvalNode> filter)
+ throws UnsupportedException {
+ return get(tableUri).getTableVolume(tableUri, filter);
}
public static Iterable<Tablespace> getAllTablespaces() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
index c79de33..3541c50 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
@@ -100,7 +100,7 @@ public class HBaseTablespace extends Tablespace {
}
@Override
- public long getTableVolume(URI uri) throws UnsupportedException {
+ public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException {
throw new UnsupportedException();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
index d95852c..ee4512b 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
@@ -125,7 +125,7 @@ public class FileTablespace extends Tablespace {
}
@Override
- public long getTableVolume(URI uri) throws UnsupportedException {
+ public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException {
Path path = new Path(uri);
ContentSummary summary;
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml
index 61481c7..804cad4 100644
--- a/tajo-storage/tajo-storage-jdbc/pom.xml
+++ b/tajo-storage/tajo-storage-jdbc/pom.xml
@@ -193,7 +193,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
-
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
index 7489307..1f7f299 100644
--- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
@@ -47,6 +47,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
/**
@@ -109,7 +110,7 @@ public abstract class JdbcTablespace extends Tablespace {
}
@Override
- public long getTableVolume(URI uri) throws UnsupportedException {
+ public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException {
throw new UnsupportedException();
}