You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hj...@apache.org on 2014/12/05 09:21:30 UTC
[26/29] tajo git commit: TAJO-1231: Implicit table properties in
session is not stored in table property.
TAJO-1231: Implicit table properties in session is not stored in table property.
Closes #287
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ab2efce8
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ab2efce8
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ab2efce8
Branch: refs/heads/hbase_storage
Commit: ab2efce8fb4015523a9a818f0e03fa57c6366585
Parents: 2a69bcc
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri Dec 5 11:43:15 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri Dec 5 16:36:12 2014 +0900
----------------------------------------------------------------------
CHANGES | 7 ++--
.../org/apache/tajo/client/TestTajoClient.java | 37 +++++++++++++++++---
.../org/apache/tajo/plan/LogicalPlanner.java | 20 ++++++++---
.../org/apache/tajo/plan/util/PlannerUtil.java | 17 +++++++++
4 files changed, 70 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 0c785ce..8ecd795 100644
--- a/CHANGES
+++ b/CHANGES
@@ -83,8 +83,11 @@ Release 0.9.1 - unreleased
BUG FIXES
- TAJO-1194: 'INSERT OVERWRITE .. SELECT' does not remove existing data when result is empty.
- (jaehwa)
+ TAJO-1231: Implicit table properties in session are not stored in
+ table property. (hyunsik)
+
+ TAJO-1194: 'INSERT OVERWRITE .. SELECT' does not remove existing data
+ when result is empty. (jaehwa)
TAJO-1191: Change DateDatum timezone to UTC. (Jaewoong Jung via hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 616799f..21dd7d5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -680,7 +680,7 @@ public class TestTajoClient {
}
@Test
- public void testSetCvsNull() throws Exception {
+ public void testNullCharSession() throws Exception {
String sql =
"select\n" +
" c_custkey,\n" +
@@ -692,17 +692,44 @@ public class TestTajoClient {
" c_custkey,\n" +
" orders.o_orderkey;\n";
- TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
-
Map<String, String> variables = new HashMap<String, String>();
variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T");
client.updateSessionVariables(variables);
+ TajoResultSet resultDesc = (TajoResultSet)client.executeQueryAndGetResult(sql);
+ resultDesc.close();
+ assertNullCharSessionVar(resultDesc.getTableDesc());
+ }
+
+ @Test
+ public void testNullCharSessionInCTAS() throws Exception {
+ String sql =
+ "create table nullcharsession as select\n" +
+ " c_custkey,\n" +
+ " orders.o_orderkey,\n" +
+ " orders.o_orderstatus \n" +
+ "from\n" +
+ " orders full outer join customer on c_custkey = o_orderkey\n" +
+ "order by\n" +
+ " c_custkey,\n" +
+ " orders.o_orderkey;\n";
+ Map<String, String> variables = new HashMap<String, String>();
+ variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T");
+ client.updateSessionVariables(variables);
TajoResultSet res = (TajoResultSet)client.executeQueryAndGetResult(sql);
+ res.close();
+
+ TableDesc resultDesc = client.getTableDesc("nullcharsession");
+ assertNullCharSessionVar(resultDesc);
+ }
+
+
+ public void assertNullCharSessionVar(TableDesc resultDesc) throws Exception {
+ TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- assertEquals(res.getTableDesc().getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T");
+ assertEquals(resultDesc.getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T");
- Path path = new Path(res.getTableDesc().getPath());
+ Path path = new Path(resultDesc.getPath());
FileSystem fs = path.getFileSystem(tajoConf);
FileStatus[] files = fs.listStatus(path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 3b1b4e3..69c0e4b 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -1706,13 +1706,25 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
}
- // Set default storage properties to be created.
- KeyValueSet keyValueSet = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType());
+
+
+ // Set default storage properties to table
+ KeyValueSet properties = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType());
+
+ // Priority to apply table properties
+ // 1. Explicit table properties specified in WITH clause
+ // 2. Session variables
+
+ // Set session variables to properties
+ PlannerUtil.applySessionToTableProperties(context.queryContext, createTableNode.getStorageType(), properties);
+ // Set table properties specified in WITH clause
if (expr.hasParams()) {
- keyValueSet.putAll(expr.getParams());
+ properties.putAll(expr.getParams());
}
- createTableNode.setOptions(keyValueSet);
+ createTableNode.setOptions(properties);
+
+
if (expr.hasPartition()) {
if (expr.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 6868b6c..c55c203 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -21,6 +21,8 @@ package org.apache.tajo.plan.util;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.*;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.Column;
@@ -35,10 +37,15 @@ import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor;
import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
+import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
import java.util.*;
+import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.CSV;
+import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.TEXTFILE;
+
public class PlannerUtil {
public static boolean checkIfDDLPlan(LogicalNode node) {
@@ -776,4 +783,14 @@ public class PlannerUtil {
return explains.toString();
}
+
+ public static void applySessionToTableProperties(OverridableConf sessionVars,
+ CatalogProtos.StoreType storeType,
+ KeyValueSet tableProperties) {
+ if (storeType == CSV || storeType == TEXTFILE) {
+ if (sessionVars.containsKey(SessionVars.NULL_CHAR)) {
+ tableProperties.set(StorageConstants.TEXT_NULL, sessionVars.get(SessionVars.NULL_CHAR));
+ }
+ }
+ }
}