You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/08/11 05:53:16 UTC
[5/5] git commit: TAJO-928: Session variables should override query
configs in TajoConf.
TAJO-928: Session variables should override query configs in TajoConf.
Closes #98
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ddfc3f33
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ddfc3f33
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ddfc3f33
Branch: refs/heads/master
Commit: ddfc3f33039ee15ab0a2d3fe5890b9acb40aec3d
Parents: 0603b49
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 11 12:52:52 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 11 12:52:52 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../java/org/apache/tajo/catalog/TableMeta.java | 14 +-
.../apache/tajo/catalog/TestKeyValueSet.java | 83 ++++++-
.../tajo/catalog/store/HCatalogStore.java | 28 +--
.../tajo/catalog/store/TestHCatalogStore.java | 12 +-
.../org/apache/tajo/catalog/TestCatalog.java | 10 +-
.../tajo/cli/DefaultTajoCliOutputFormatter.java | 16 +-
.../java/org/apache/tajo/cli/HelpCommand.java | 100 ++++----
.../java/org/apache/tajo/cli/SetCommand.java | 78 +++++-
.../main/java/org/apache/tajo/cli/TajoCli.java | 149 ++++++++----
.../apache/tajo/cli/TajoCliOutputFormatter.java | 5 +-
.../org/apache/tajo/cli/TajoShellCommand.java | 10 +-
.../main/java/org/apache/tajo/ConfigKey.java | 52 ++++
.../java/org/apache/tajo/OverridableConf.java | 235 +++++++++++++++++++
.../main/java/org/apache/tajo/SessionVars.java | 211 +++++++++++++++++
.../java/org/apache/tajo/conf/TajoConf.java | 199 ++++++++--------
.../main/java/org/apache/tajo/datum/Datum.java | 2 +-
.../java/org/apache/tajo/datum/NullDatum.java | 1 +
.../java/org/apache/tajo/util/KeyValueSet.java | 149 ++++++++++--
.../tajo/datum/TestArithmeticOperator.java | 4 +-
.../tajo/engine/planner/LogicalOptimizer.java | 9 +-
.../apache/tajo/engine/planner/LogicalPlan.java | 4 +-
.../engine/planner/LogicalPlanPreprocessor.java | 18 +-
.../engine/planner/LogicalPlanVerifier.java | 12 +-
.../tajo/engine/planner/LogicalPlanner.java | 35 ++-
.../engine/planner/PhysicalPlannerImpl.java | 44 +++-
.../engine/planner/PreLogicalPlanVerifier.java | 21 +-
.../engine/planner/global/GlobalPlanner.java | 18 +-
.../planner/physical/ExternalSortExec.java | 4 +-
.../engine/planner/physical/StoreTableExec.java | 6 +-
.../apache/tajo/engine/query/QueryContext.java | 144 ++++++------
.../tajo/engine/query/QueryUnitRequestImpl.java | 3 +-
.../org/apache/tajo/master/GlobalEngine.java | 94 ++++----
.../tajo/master/TajoMasterClientService.java | 31 ++-
.../apache/tajo/master/querymaster/Query.java | 4 +-
.../tajo/master/querymaster/QueryMaster.java | 11 +-
.../querymaster/QueryMasterManagerService.java | 4 +-
.../master/querymaster/QueryMasterTask.java | 4 +-
.../tajo/master/querymaster/Repartitioner.java | 16 +-
.../tajo/master/querymaster/SubQuery.java | 15 +-
.../org/apache/tajo/master/session/Session.java | 50 ++--
.../session/SessionLivelinessMonitor.java | 2 +-
.../main/java/org/apache/tajo/worker/Task.java | 4 +-
.../apache/tajo/worker/TaskAttemptContext.java | 12 +-
.../resources/webapps/worker/querydetail.jsp | 10 +
.../apache/tajo/LocalTajoTestingUtility.java | 5 +
.../java/org/apache/tajo/QueryTestCaseBase.java | 2 +-
.../test/java/org/apache/tajo/TpchTestBase.java | 2 +-
.../tajo/cli/TestDefaultCliOutputFormatter.java | 47 +++-
.../java/org/apache/tajo/cli/TestTajoCli.java | 172 ++++++++------
.../org/apache/tajo/client/TestTajoClient.java | 8 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 9 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 16 +-
.../engine/planner/TestLogicalOptimizer.java | 20 +-
.../tajo/engine/planner/TestLogicalPlan.java | 3 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 135 ++++++++---
.../tajo/engine/planner/TestPlannerUtil.java | 5 +-
.../planner/global/TestBroadcastJoinPlan.java | 62 ++---
.../planner/physical/TestBNLJoinExec.java | 9 +-
.../planner/physical/TestBSTIndexExec.java | 4 +-
.../planner/physical/TestExternalSortExec.java | 4 +-
.../physical/TestFullOuterHashJoinExec.java | 24 +-
.../physical/TestFullOuterMergeJoinExec.java | 29 +--
.../planner/physical/TestHashAntiJoinExec.java | 4 +-
.../planner/physical/TestHashJoinExec.java | 19 +-
.../planner/physical/TestHashSemiJoinExec.java | 4 +-
.../physical/TestLeftOuterHashJoinExec.java | 28 +--
.../physical/TestLeftOuterNLJoinExec.java | 25 +-
.../planner/physical/TestMergeJoinExec.java | 4 +-
.../engine/planner/physical/TestNLJoinExec.java | 17 +-
.../planner/physical/TestPhysicalPlanner.java | 100 ++++----
.../physical/TestProgressExternalSortExec.java | 4 +-
.../physical/TestRightOuterHashJoinExec.java | 16 +-
.../physical/TestRightOuterMergeJoinExec.java | 32 +--
.../engine/planner/physical/TestSortExec.java | 4 +-
.../tajo/engine/query/TestGroupByQuery.java | 22 +-
.../tajo/engine/query/TestJoinBroadcast.java | 8 +-
.../apache/tajo/engine/query/TestJoinQuery.java | 90 +++----
.../tajo/engine/query/TestNullValues.java | 14 +-
.../tajo/engine/query/TestSelectQuery.java | 10 +-
.../apache/tajo/engine/query/TestSortQuery.java | 8 +-
.../org/apache/tajo/jdbc/TestResultSet.java | 2 +-
.../tajo/master/TestExecutionBlockCursor.java | 6 +-
.../apache/tajo/master/TestGlobalPlanner.java | 4 +-
.../querymaster/TestQueryUnitStatusUpdate.java | 2 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 8 +-
.../TestTajoCli/testHelpSessionVars.result | 36 +++
...estSelectResultWithNullTrueDeprecated.result | 9 +
.../testStopWhenErrorDeprecated.result | 6 +
.../java/org/apache/tajo/storage/Appender.java | 2 +-
.../java/org/apache/tajo/storage/CSVFile.java | 11 +-
.../org/apache/tajo/storage/StorageUtil.java | 18 +-
.../org/apache/tajo/storage/avro/AvroUtil.java | 17 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileAppender.java | 10 +-
.../sequencefile/SequenceFileScanner.java | 5 +-
.../apache/tajo/storage/v2/RCFileScanner.java | 2 +-
97 files changed, 2057 insertions(+), 992 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index deb41a0..182d07d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -29,6 +29,9 @@ Release 0.9.0 - unreleased
IMPROVEMENT
+ TAJO-928: Session variables should override query configs in TajoConf.
+ (hyunsik)
+
TAJO-989: Cleanup of child blocks after parent execution block is complete
(jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
index 6015fc9..2d95e6b 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
@@ -78,7 +78,19 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj
public void putOption(String key, String val) {
maybeInitBuilder();
- options.put(key, val);
+ options.set(key, val);
+ }
+
+ public boolean containsOption(String key) {
+ TableProtoOrBuilder p = viaProto ? proto : builder;
+ if (options != null) {
+ return this.options.containsKey(key);
+ }
+ if (!p.hasParams()) {
+ return false;
+ }
+ this.options = new KeyValueSet(p.getParams());
+ return options.containsKey(key);
}
public String getOption(String key) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java
index 39d5fbe..b317ba4 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java
@@ -24,23 +24,70 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
public class TestKeyValueSet {
+
@Test
- public final void testPutAndGet() {
+ public final void testPutAndGetString() {
KeyValueSet opts = new KeyValueSet();
- opts.put("name", "abc");
- opts.put("delimiter", ",");
+ opts.set("k1", "v1");
+ opts.set("k2", "v2");
- assertEquals(",", opts.get("delimiter"));
- assertEquals("abc", opts.get("name"));
+ assertEquals("v1", opts.get("k1"));
+ assertEquals("v2", opts.get("k2"));
+ assertEquals("default", opts.get("k3", "default"));
}
+ @Test
+ public final void testPutAndGetBool() {
+ KeyValueSet opts = new KeyValueSet();
+ opts.setBool("k1", true);
+ opts.setBool("k2", false);
+
+ assertEquals(true, opts.getBool("k1"));
+ assertEquals(false, opts.getBool("k2"));
+ assertEquals(true, opts.getBool("k3", true));
+ }
+
+ @Test
+ public final void testPutAndGetInt() {
+ KeyValueSet opts = new KeyValueSet();
+ opts.setInt("k1", 1980);
+ opts.setInt("k2", 401);
+
+ assertEquals(1980, opts.getInt("k1"));
+ assertEquals(401, opts.getInt("k2"));
+ assertEquals(2020, opts.getInt("k3", 2020));
+ }
+
+ @Test
+ public final void testPutAndGetLong() {
+ KeyValueSet opts = new KeyValueSet();
+ opts.setLong("k1", 1980);
+ opts.setLong("k2", 401);
+
+ assertEquals(1980, opts.getLong("k1"));
+ assertEquals(401, opts.getLong("k2"));
+ assertEquals(2020, opts.getLong("k3", 2020l));
+ }
+
+ @Test
+ public final void testPutAndGetFloat() {
+ KeyValueSet opts = new KeyValueSet();
+ opts.setFloat("k1", 1980.4f);
+ opts.setFloat("k2", 401.150f);
+
+ assertTrue(1980.4f == opts.getFloat("k1"));
+ assertTrue(401.150f == opts.getFloat("k2"));
+ assertTrue(3.14f == opts.getFloat("k3", 3.14f));
+ }
+
@Test
public final void testGetProto() {
KeyValueSet opts = new KeyValueSet();
- opts.put("name", "abc");
- opts.put("delimiter", ",");
+ opts.set("name", "abc");
+ opts.set("delimiter", ",");
PrimitiveProtos.KeyValueSetProto proto = opts.getProto();
KeyValueSet opts2 = new KeyValueSet(proto);
@@ -49,16 +96,26 @@ public class TestKeyValueSet {
}
@Test
- public final void testDelete() {
+ public final void testRemove() {
KeyValueSet opts = new KeyValueSet();
- opts.put("name", "abc");
- opts.put("delimiter", ",");
+ opts.set("name", "abc");
+ opts.set("delimiter", ",");
assertEquals("abc", opts.get("name"));
- assertEquals("abc", opts.delete("name"));
- assertNull(opts.get("name"));
+ assertEquals("abc", opts.remove("name"));
+ try {
+ opts.get("name");
+ assertTrue(false);
+ } catch (IllegalArgumentException iae) {
+ assertTrue(true);
+ }
KeyValueSet opts2 = new KeyValueSet(opts.getProto());
- assertNull(opts2.get("name"));
+ try {
+ opts2.get("name");
+ assertTrue(false);
+ } catch (IllegalArgumentException iae) {
+ assertTrue(true);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 6f48348..61bdccd 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -166,7 +166,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
stats = new TableStats();
options = new KeyValueSet();
options.putAll(table.getParameters());
- options.delete("EXTERNAL");
+ options.remove("EXTERNAL");
Properties properties = table.getMetadata();
if (properties != null) {
@@ -186,31 +186,31 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
} else {
nullFormat = "\\N";
}
- options.delete(serdeConstants.SERIALIZATION_NULL_FORMAT);
+ options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT);
// set file output format
String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT);
storeType = CatalogUtil.getStoreType(HCatalogUtil.getStoreType(fileOutputformat));
if (storeType.equals(CatalogProtos.StoreType.CSV)) {
- options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.set(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.set(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
} else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) {
- options.put(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) {
- options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+ options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (ColumnarSerDe.class.getName().equals(serde)) {
- options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
} else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) {
- options.put(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.put(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinarySerDe.class.getName().equals(serde)) {
- options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+ options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (LazySimpleSerDe.class.getName().equals(serde)) {
- options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
}
@@ -481,7 +481,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName());
}
- if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) {
+ if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) {
table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
@@ -504,7 +504,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
StringEscapeUtils.unescapeJava(fieldDelimiter));
table.getParameters().remove(StorageConstants.CSVFILE_DELIMITER);
- if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) {
+ if (tableDesc.getMeta().containsOption(StorageConstants.CSVFILE_NULL)) {
table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL)));
table.getParameters().remove(StorageConstants.CSVFILE_NULL);
@@ -534,7 +534,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
}
- if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) {
+ if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) {
table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index a507b08..f225424 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -117,7 +117,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingRCFileWithBinarySerde() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+ options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -146,7 +146,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingRCFileWithTextSerde() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -174,8 +174,8 @@ public class TestHCatalogStore {
@Test
public void testTableWithNullValue() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
- options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003"));
+ options.set(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
+ options.set(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003"));
TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -315,7 +315,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingSequenceFileWithBinarySerde() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+ options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -344,7 +344,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingSequenceFileWithTextSerde() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 4039ff2..654736d 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -565,7 +565,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
@@ -605,7 +605,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
Schema partSchema = new Schema();
@@ -643,7 +643,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
Schema partSchema = new Schema();
@@ -680,7 +680,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
Schema partSchema = new Schema();
@@ -717,7 +717,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
Schema partSchema = new Schema();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
index 62736b5..543ac8c 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
@@ -20,11 +20,10 @@ package org.apache.tajo.cli;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.tajo.QueryId;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.QueryStatus;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.util.FileUtil;
import java.io.InputStream;
@@ -33,20 +32,17 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
- private TajoConf tajoConf;
private int printPauseRecords;
private boolean printPause;
private boolean printErrorTrace;
private String nullChar;
@Override
- public void init(TajoConf tajoConf) {
- this.tajoConf = tajoConf;
-
- this.printPause = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_PAUSE);
- this.printPauseRecords = tajoConf.getIntVar(TajoConf.ConfVars.CLI_PRINT_PAUSE_NUM_RECORDS);
- this.printErrorTrace = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_ERROR_TRACE);
- this.nullChar = tajoConf.getVar(ConfVars.CLI_NULL_CHAR);
+ public void init(TajoCli.TajoCliContext context) {
+ this.printPause = context.getBool(SessionVars.CLI_PAGING_ENABLED);
+ this.printPauseRecords = context.getInt(SessionVars.CLI_PAGE_ROWS);
+ this.printErrorTrace = context.getBool(SessionVars.CLI_DISPLAY_ERROR_TRACE);
+ this.nullChar = context.get(SessionVars.CLI_NULL_CHAR);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java
index 8d908ec..52f13b0 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java
@@ -40,53 +40,69 @@ public class HelpCommand extends TajoShellCommand {
}
@Override
+ public String [] getAliases() {
+ return new String [] {"\\help"};
+ }
+
+ @Override
public void invoke(String[] cmd) throws Exception {
if(targetDocVersion.equalsIgnoreCase("")) {
targetDocVersion = getDocumentationVersion();
}
- PrintWriter sout = context.getOutput();
- sout.println();
-
- sout.println("General");
- sout.println(" \\copyright show Apache License 2.0");
- sout.println(" \\version show Tajo version");
- sout.println(" \\? show help");
- sout.println(" \\q quit tsql");
- sout.println();
- sout.println();
-
- sout.println("Informational");
- sout.println(" \\l list databases");
- sout.println(" \\c show current database");
- sout.println(" \\c [DBNAME] connect to new database");
- sout.println(" \\d list tables");
- sout.println(" \\d [TBNAME] describe table");
- sout.println(" \\df list functions");
- sout.println(" \\df NAME describe function");
- sout.println();
- sout.println();
-
- sout.println("Tool");
- sout.println(" \\! execute a linux shell command");
- sout.println(" \\dfs execute a dfs command");
- sout.println(" \\admin execute tajo admin command");
- sout.println();
- sout.println();
-
- sout.println("Variables");
- sout.println(" \\set [[NAME] [VALUE] set session variable or list session variables");
- sout.println(" \\unset NAME unset session variable");
- sout.println();
- sout.println();
-
- sout.println("Documentations");
- sout.println(" tsql guide http://tajo.apache.org/docs/"+ targetDocVersion +"/cli.html");
- sout.println(" Query language http://tajo.apache.org/docs/"+ targetDocVersion +"/sql_language.html");
- sout.println(" Functions http://tajo.apache.org/docs/"+ targetDocVersion +"/functions.html");
- sout.println(" Backup & restore http://tajo.apache.org/docs/"+ targetDocVersion +"/backup_and_restore.html");
- sout.println(" Configuration http://tajo.apache.org/docs/"+ targetDocVersion +"/configuration.html");
- sout.println();
+ if (cmd.length == 1) {
+ PrintWriter sout = context.getOutput();
+ sout.println();
+
+ sout.println("General");
+ sout.println(" \\copyright show Apache License 2.0");
+ sout.println(" \\version show Tajo version");
+ sout.println(" \\? show help");
+ sout.println(" \\? [COMMAND] show help of a given command");
+ sout.println(" \\help alias of \\?");
+ sout.println(" \\q quit tsql");
+ sout.println();
+ sout.println();
+
+ sout.println("Informational");
+ sout.println(" \\l list databases");
+ sout.println(" \\c show current database");
+ sout.println(" \\c [DBNAME] connect to new database");
+ sout.println(" \\d list tables");
+ sout.println(" \\d [TBNAME] describe table");
+ sout.println(" \\df list functions");
+ sout.println(" \\df NAME describe function");
+ sout.println();
+ sout.println();
+
+ sout.println("Tool");
+ sout.println(" \\! execute a linux shell command");
+ sout.println(" \\dfs execute a dfs command");
+ sout.println(" \\admin execute tajo admin command");
+ sout.println();
+ sout.println();
+
+ sout.println("Variables");
+ sout.println(" \\set [[NAME] [VALUE] set session variable or list session variables");
+ sout.println(" \\unset NAME unset session variable");
+ sout.println();
+ sout.println();
+
+ sout.println("Documentations");
+ sout.println(" tsql guide http://tajo.apache.org/docs/" + targetDocVersion + "/cli.html");
+ sout.println(" Query language http://tajo.apache.org/docs/" + targetDocVersion + "/sql_language.html");
+ sout.println(" Functions http://tajo.apache.org/docs/" + targetDocVersion + "/functions.html");
+ sout.println(" Backup & restore http://tajo.apache.org/docs/" + targetDocVersion + "/backup_and_restore.html");
+ sout.println(" Configuration http://tajo.apache.org/docs/" + targetDocVersion + "/configuration.html");
+ sout.println();
+ } else if (cmd.length == 2) {
+ String slashCommand = "\\" + cmd[1];
+ if (context.getCommands().containsKey(slashCommand)) {
+ context.getCommands().get(slashCommand).printHelp();
+ } else {
+ context.getOutput().println("Command not found: " + cmd[1]);
+ }
+ }
}
private String getDocumentationVersion() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java
index c694507..c957a19 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java
@@ -18,11 +18,15 @@
package org.apache.tajo.cli;
+import com.google.protobuf.ServiceException;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.util.StringUtils;
import java.util.HashMap;
import java.util.Map;
+import static org.apache.tajo.SessionVars.VariableMode;
+
public class SetCommand extends TajoShellCommand {
public SetCommand(TajoCli.TajoCliContext context) {
@@ -34,17 +38,46 @@ public class SetCommand extends TajoShellCommand {
return "\\set";
}
+ private void showAllSessionVars() throws ServiceException {
+ for (Map.Entry<String, String> entry: client.getAllSessionVariables().entrySet()) {
+ context.getOutput().println(StringUtils.quote(entry.getKey()) + "=" + StringUtils.quote(entry.getValue()));
+ }
+ }
+
+ private void updateSessionVariable(String key, String val) throws ServiceException {
+ Map<String, String> variables = new HashMap<String, String>();
+ variables.put(key, val);
+ client.updateSessionVariables(variables);
+ }
+
+ void set(String key, String val) throws ServiceException {
+ SessionVars sessionVar = null;
+
+ if (SessionVars.exists(key)) { // if the variable is one of the session variables
+ sessionVar = SessionVars.get(key);
+
+ // is it cli-side variable?
+ if (sessionVar.getMode() == VariableMode.CLI_SIDE_VAR) {
+ context.setCliSideVar(key, val);
+ } else {
+ updateSessionVariable(key, val);
+ }
+
+ if (SessionVars.isDeprecated(key)) {
+ context.getOutput().println("Warning: deprecated to directly use config key in TajoConf.ConfVars. " +
+ "Please execute '\\help set'.");
+ }
+ } else {
+ updateSessionVariable(key, val);
+ }
+ }
+
@Override
public void invoke(String[] cmd) throws Exception {
if (cmd.length == 1) {
- for (Map.Entry<String, String> entry: client.getAllSessionVariables().entrySet()) {
- context.getOutput().println(StringUtils.quote(entry.getKey()) + "=" + StringUtils.quote(entry.getValue()));
- }
+ showAllSessionVars();
} else if (cmd.length == 3) {
- Map<String, String> variables = new HashMap<String, String>();
- variables.put(cmd[1], cmd[2]);
- client.updateSessionVariables(variables);
- context.setVariable(cmd[1], cmd[2]);
+ set(cmd[1], cmd[2]);
} else {
context.getOutput().println("usage: \\set [[NAME] VALUE]");
}
@@ -59,4 +92,35 @@ public class SetCommand extends TajoShellCommand {
public String getDescription() {
return "set session variable or shows all session variables";
}
+
+ @Override
+ public void printHelp() {
+ context.getOutput().println("\nAvailable Session Variables:\n");
+ for (SessionVars var : SessionVars.values()) {
+
+ if (var.getMode() == VariableMode.DEFAULT ||
+ var.getMode() == VariableMode.CLI_SIDE_VAR ||
+ var.getMode() == VariableMode.FROM_SHELL_ENV) {
+
+ context.getOutput().println("\\set " + var.keyname() + " " + getDisplayType(var.getVarType()) + " - " + var
+ .getDescription());
+ }
+ }
+ }
+
+ public static String getDisplayType(Class<?> clazz) {
+ if (clazz == String.class) {
+ return "[text value]";
+ } else if (clazz == Integer.class) {
+ return "[int value]";
+ } else if (clazz == Long.class) {
+ return "[long value]";
+ } else if (clazz == Float.class) {
+ return "[real value]";
+ } else if (clazz == Boolean.class) {
+ return "[true or false]";
+ } else {
+ return clazz.getSimpleName();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
index c20e44b..81427ab 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
@@ -18,11 +18,12 @@
package org.apache.tajo.cli;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.protobuf.ServiceException;
import jline.console.ConsoleReader;
import org.apache.commons.cli.*;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.QueryIdFactory;
+import org.apache.tajo.*;
import org.apache.tajo.TajoProtos.QueryState;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.client.QueryStatus;
@@ -62,7 +63,7 @@ public class TajoCli {
// Current States
private String currentDatabase;
- private TajoCliOutputFormatter outputFormatter;
+ private TajoCliOutputFormatter displayFormatter;
private boolean wasError = false;
@@ -99,7 +100,10 @@ public class TajoCli {
options.addOption("help", "help", false, "help");
}
- public class TajoCliContext {
+ public class TajoCliContext extends OverridableConf {
+ public TajoCliContext(TajoConf conf) {
+ super(conf, ConfigKey.ConfigType.SESSION);
+ }
public TajoClient getTajoClient() {
return client;
@@ -121,18 +125,50 @@ public class TajoCli {
return conf;
}
- public void setVariable(String key, String value) {
- conf.set(key, value);
- try {
- initFormatter();
- } catch (Exception e) {
- System.err.println(ERROR_PREFIX + e.getMessage());
+ @VisibleForTesting
+ public String getCliSideVar(String key) {
+ if (SessionVars.exists(key)) {
+ ConfigKey configKey = SessionVars.get(key);
+ return get(configKey);
+ } else {
+ return get(key);
+ }
+ }
+
+ public void setCliSideVar(String key, String value) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(value);
+
+ boolean shouldReloadFormatter = false;
+
+ if (SessionVars.exists(key)) {
+ SessionVars configKey = SessionVars.get(key);
+ put(configKey, value);
+ shouldReloadFormatter = configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR;
+ } else {
+ set(key, value);
+
+ // It is hard to recognize it is a client side variable. So, we always reload formatter.
+ shouldReloadFormatter = true;
}
+
+ if (shouldReloadFormatter) {
+ try {
+ initFormatter();
+ } catch (Exception e) {
+ System.err.println(ERROR_PREFIX + e.getMessage());
+ }
+ }
+ }
+
+ public Map<String, TajoShellCommand> getCommands() {
+ return commands;
}
}
public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception {
this.conf = new TajoConf(c);
+ context = new TajoCliContext(conf);
this.sin = in;
this.reader = new ConsoleReader(sin, out);
this.reader.setExpandEvents(false);
@@ -161,13 +197,7 @@ public class TajoCli {
}
if (cmd.getOptionValues("conf") != null) {
- for (String eachParam: cmd.getOptionValues("conf")) {
- String[] tokens = eachParam.split("=");
- if (tokens.length != 2) {
- continue;
- }
- conf.set(tokens[0], tokens[1]);
- }
+ processConfVarCommand(cmd.getOptionValues("conf"));
}
// if there is no "-h" option,
@@ -196,19 +226,22 @@ public class TajoCli {
client = new TajoClient(conf, baseDatabase);
}
- context = new TajoCliContext();
context.setCurrentDatabase(client.getCurrentDatabase());
initHistory();
initCommands();
+ if (cmd.getOptionValues("conf") != null) {
+ processSessionVarCommand(cmd.getOptionValues("conf"));
+ }
+
if (cmd.hasOption("c")) {
- outputFormatter.setScirptMode();
+ displayFormatter.setScirptMode();
int exitCode = executeScript(cmd.getOptionValue("c"));
sout.flush();
System.exit(exitCode);
}
if (cmd.hasOption("f")) {
- outputFormatter.setScirptMode();
+ displayFormatter.setScirptMode();
cmd.getOptionValues("");
File sqlFile = new File(cmd.getOptionValue("f"));
if (sqlFile.exists()) {
@@ -226,13 +259,38 @@ public class TajoCli {
addShutdownHook();
}
+ private void processConfVarCommand(String[] confCommands) throws ServiceException {
+ for (String eachParam: confCommands) {
+ String[] tokens = eachParam.split("=");
+ if (tokens.length != 2) {
+ continue;
+ }
+
+ if (!SessionVars.exists(tokens[0])) {
+ conf.set(tokens[0], tokens[1]);
+ }
+ }
+ }
+
+ private void processSessionVarCommand(String[] confCommands) throws ServiceException {
+ for (String eachParam: confCommands) {
+ String[] tokens = eachParam.split("=");
+ if (tokens.length != 2) {
+ continue;
+ }
+
+ if (SessionVars.exists(tokens[0])) {
+ ((SetCommand)commands.get("\\set")).set(tokens[0], tokens[1]);
+ }
+ }
+ }
+
private void initFormatter() throws Exception {
- Class formatterClass = conf.getClass(ConfVars.CLI_OUTPUT_FORMATTER_CLASS.varname,
- DefaultTajoCliOutputFormatter.class);
- if (outputFormatter == null || !outputFormatter.getClass().equals(formatterClass)) {
- outputFormatter = (TajoCliOutputFormatter)formatterClass.newInstance();
+ Class formatterClass = context.getClass(SessionVars.CLI_FORMATTER_CLASS);
+ if (displayFormatter == null || !displayFormatter.getClass().equals(formatterClass)) {
+ displayFormatter = (TajoCliOutputFormatter)formatterClass.newInstance();
}
- outputFormatter.init(conf);
+ displayFormatter.init(context);
}
public TajoCliContext getContext() {
@@ -280,6 +338,9 @@ public class TajoCli {
throw new RuntimeException(e.getMessage());
}
commands.put(cmd.getCommand(), cmd);
+ for (String alias : cmd.getAliases()) {
+ commands.put(alias, cmd);
+ }
}
}
@@ -332,7 +393,7 @@ public class TajoCli {
exitCode = executeParsedResults(parsedResults);
currentPrompt = updatePrompt(parser.getState());
- if (exitCode != 0 && context.getConf().getBoolVar(ConfVars.CLI_ERROR_STOP)) {
+ if (exitCode != 0 && context.getBool(SessionVars.ON_ERROR_STOP)) {
return exitCode;
}
}
@@ -372,18 +433,18 @@ public class TajoCli {
try {
invoked.invoke(arguments);
} catch (IllegalArgumentException ige) {
- outputFormatter.printErrorMessage(sout, ige);
+ displayFormatter.printErrorMessage(sout, ige);
wasError = true;
return -1;
} catch (Exception e) {
- outputFormatter.printErrorMessage(sout, e);
+ displayFormatter.printErrorMessage(sout, e);
wasError = true;
return -1;
} finally {
context.getOutput().flush();
}
- if (wasError && context.getConf().getBoolVar(ConfVars.CLI_ERROR_STOP)) {
+ if (wasError && context.getBool(SessionVars.ON_ERROR_STOP)) {
break;
}
}
@@ -395,7 +456,7 @@ public class TajoCli {
long startTime = System.currentTimeMillis();
ClientProtos.SubmitQueryResponse response = client.executeQueryWithJson(json);
if (response == null) {
- outputFormatter.printErrorMessage(sout, "response is null");
+ displayFormatter.printErrorMessage(sout, "response is null");
wasError = true;
} else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
if (response.getIsForwarded()) {
@@ -403,7 +464,7 @@ public class TajoCli {
waitForQueryCompleted(queryId);
} else {
if (!response.hasTableDesc() && !response.hasResultSet()) {
- outputFormatter.printMessage(sout, "OK");
+ displayFormatter.printMessage(sout, "OK");
wasError = true;
} else {
localQueryCompleted(response, startTime);
@@ -411,7 +472,7 @@ public class TajoCli {
}
} else {
if (response.hasErrorMessage()) {
- outputFormatter.printErrorMessage(sout, response.getErrorMessage());
+ displayFormatter.printErrorMessage(sout, response.getErrorMessage());
wasError = true;
}
}
@@ -421,7 +482,7 @@ public class TajoCli {
long startTime = System.currentTimeMillis();
ClientProtos.SubmitQueryResponse response = client.executeQuery(statement);
if (response == null) {
- outputFormatter.printErrorMessage(sout, "response is null");
+ displayFormatter.printErrorMessage(sout, "response is null");
wasError = true;
} else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
if (response.getIsForwarded()) {
@@ -429,14 +490,14 @@ public class TajoCli {
waitForQueryCompleted(queryId);
} else {
if (!response.hasTableDesc() && !response.hasResultSet()) {
- outputFormatter.printMessage(sout, "OK");
+ displayFormatter.printMessage(sout, "OK");
} else {
localQueryCompleted(response, startTime);
}
}
} else {
if (response.hasErrorMessage()) {
- outputFormatter.printErrorMessage(sout, response.getErrorMessage());
+ displayFormatter.printErrorMessage(sout, response.getErrorMessage());
wasError = true;
}
}
@@ -454,13 +515,13 @@ public class TajoCli {
// non-forwarded INSERT INTO query does not have any query id.
// In this case, it just returns succeeded query information without printing the query results.
if (response.getMaxRowNum() < 0 && queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
- outputFormatter.printResult(sout, sin, desc, responseTime, res);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
} else {
res = TajoClient.createResultSet(client, response);
- outputFormatter.printResult(sout, sin, desc, responseTime, res);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
}
} catch (Throwable t) {
- outputFormatter.printErrorMessage(sout, t);
+ displayFormatter.printErrorMessage(sout, t);
wasError = true;
} finally {
if (res != null) {
@@ -495,7 +556,7 @@ public class TajoCli {
}
if (status.getState() == QueryState.QUERY_RUNNING || status.getState() == QueryState.QUERY_SUCCEEDED) {
- outputFormatter.printProgress(sout, status);
+ displayFormatter.printProgress(sout, status);
}
if (status.getState() != QueryState.QUERY_RUNNING &&
@@ -509,10 +570,10 @@ public class TajoCli {
}
if (status.getState() == QueryState.QUERY_ERROR || status.getState() == QueryState.QUERY_FAILED) {
- outputFormatter.printErrorMessage(sout, status);
+ displayFormatter.printErrorMessage(sout, status);
wasError = true;
} else if (status.getState() == QueryState.QUERY_KILLED) {
- outputFormatter.printKilledMessage(sout, queryId);
+ displayFormatter.printKilledMessage(sout, queryId);
wasError = true;
} else {
if (status.getState() == QueryState.QUERY_SUCCEEDED) {
@@ -521,15 +582,15 @@ public class TajoCli {
if (status.hasResult()) {
res = TajoClient.createResultSet(client, queryId, response);
TableDesc desc = new TableDesc(response.getTableDesc());
- outputFormatter.printResult(sout, sin, desc, responseTime, res);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
} else {
TableDesc desc = new TableDesc(response.getTableDesc());
- outputFormatter.printResult(sout, sin, desc, responseTime, res);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
}
}
}
} catch (Throwable t) {
- outputFormatter.printErrorMessage(sout, t);
+ displayFormatter.printErrorMessage(sout, t);
wasError = true;
} finally {
if (res != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java
index 0e91669..0ad89f2 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java
@@ -21,7 +21,6 @@ package org.apache.tajo.cli;
import org.apache.tajo.QueryId;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.client.QueryStatus;
-import org.apache.tajo.conf.TajoConf;
import java.io.InputStream;
import java.io.PrintWriter;
@@ -30,9 +29,9 @@ import java.sql.ResultSet;
public interface TajoCliOutputFormatter {
/**
* Initialize formatter
- * @param tajoConf
+ * @param context
*/
- public void init(TajoConf tajoConf);
+ public void init(TajoCli.TajoCliContext context);
/**
* print query result to console
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java
index 1ea2893..39f5377 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java
@@ -23,16 +23,24 @@ import org.apache.tajo.conf.TajoConf;
public abstract class TajoShellCommand {
public abstract String getCommand();
+ public String [] getAliases() {
+ return new String[] {};
+ }
public abstract void invoke(String [] command) throws Exception;
public abstract String getUsage();
public abstract String getDescription();
+ public void printHelp() {
+ context.getOutput().print(getCommand());
+ context.getOutput().print(" - ");
+ context.getOutput().println(getDescription());
+ }
protected TajoCli.TajoCliContext context;
protected TajoClient client;
protected int maxColumn;
public TajoShellCommand(TajoCli.TajoCliContext context) {
- maxColumn = context.getTajoClient().getConf().getIntVar(TajoConf.ConfVars.CLI_MAX_COLUMN);
+ maxColumn = context.getTajoClient().getConf().getIntVar(TajoConf.ConfVars.$CLI_MAX_COLUMN);
this.context = context;
client = context.getTajoClient();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java
new file mode 100644
index 0000000..b9d51ec
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+public interface ConfigKey {
+
+ // Client can set or change variables of this mode.
+ public static final int DEFAULT_MODE = 0;
+ // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables.
+ public static final int FROM_SHELL_ENV_MODE = 1;
+ // only TajoMaster is able to set and change variables of this mode.
+ public static final int SERVER_SIDE_VAR_MODE = 2;
+ // This type variable will be used only in cli side.
+ public static final int CLI_SIDE_VAR_MODE = 3;
+
+ public static enum ConfigType {
+ SYSTEM(""),
+ SESSION("$"),
+ QUERY("@"),
+ CLI("+");
+
+ private String prefix;
+
+ ConfigType(String prefix) {
+ this.prefix = prefix;
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+ }
+
+ public String keyname();
+
+ public ConfigType type();
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
new file mode 100644
index 0000000..220bd43
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.util.KeyValueSet;
+
+import static org.apache.tajo.ConfigKey.ConfigType;
+import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto;
+
+/**
+ * OverridableConf provides a consolidated config system. Tajo basically uses TajoConf, which is a extended class of
+ * Hadoop's Configuration system, However, TajoConf is only used for sharing static system configs, such as binding
+ * address of master and workers, system directories, other system parameters.
+ *
+ * For modifiable or instant configs, we use OverridableConf, which is a set of key-value pairs.
+ * OverridableConf provides more strong-typed way to set configs and its behavior is more clear than Configuration
+ * system.
+ *
+ * By default, OverridableConf recognizes following config types.
+ *
+ * <ul>
+ * <li>System Config - it comes from Hadoop's Configuration class. by tajo-site, catalog-site,
+ * catalog-default and TajoConf.</li>
+ * <li>Session variables - they are instantly configured by users.
+ * Each client session has it own set of session variables.</li>
+ * </ul>
+ *
+ * System configs and session variables can set the same config in the same time. System configs are usually used to set
+ * default configs, and session variables is user-specified configs. So, session variables can override system configs.
+ */
+public class OverridableConf extends KeyValueSet {
+ private static final Log LOG = LogFactory.getLog(OverridableConf.class);
+ private ConfigType [] configTypes;
+ private TajoConf conf;
+
+ public OverridableConf(final TajoConf conf, ConfigType...configTypes) {
+ this.conf = conf;
+ this.configTypes = configTypes;
+ }
+
+ public OverridableConf(final TajoConf conf, KeyValueSetProto proto, ConfigType...configTypes) {
+ super(proto);
+ this.conf = conf;
+ this.configTypes = configTypes;
+ }
+
+ public void setConf(TajoConf conf) {
+ this.conf = conf;
+ }
+
+ public TajoConf getConf() {
+ return conf;
+ }
+
+ public void setBool(ConfigKey key, boolean val) {
+ setBool(key.keyname(), val);
+ }
+
+ public boolean getBool(ConfigKey key, Boolean defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return getBool(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case QUERY:
+ return getBool(key.keyname());
+ case SESSION:
+ return getBool(key.keyname(), conf.getBoolVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getBoolVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public boolean getBool(ConfigKey key) {
+ return getBool(key, null);
+ }
+
+ public int getInt(ConfigKey key, Integer defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return getInt(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case SESSION:
+ return getInt(key.keyname(), conf.getIntVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getIntVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public int getInt(ConfigKey key) {
+ return getInt(key, null);
+ }
+
+ public long getLong(ConfigKey key, Long defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return getLong(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case SESSION:
+ return getLong(key.keyname(), conf.getLongVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getLongVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public long getLong(ConfigKey key) {
+ return getLong(key, null);
+ }
+
+ public float getFloat(ConfigKey key, Float defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return getFloat(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case SESSION:
+ return getFloat(key.keyname(), conf.getFloatVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getFloatVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public float getFloat(ConfigKey key) {
+ return getLong(key, null);
+ }
+
+ public void put(ConfigKey key, String val) {
+ set(key.keyname(), val);
+ }
+
+ private void assertRegisteredEnum(ConfigKey key) {
+ boolean registered = false;
+
+ for (ConfigType c : configTypes) {
+ registered = key.type() == c;
+ }
+
+ registered |= key.type() == ConfigType.SESSION || key.type() != ConfigType.SYSTEM;
+
+ Preconditions.checkArgument(registered, key.keyname() + " (" + key.type() + ") is not allowed in " +
+ getClass().getSimpleName());
+ }
+
+ public String get(ConfigKey key, String defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return get(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case SESSION:
+ return get(key.keyname(), conf.getVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public String get(ConfigKey key) {
+ return get(key, null);
+ }
+
+ public Class<?> getClass(ConfigKey key) {
+ assertRegisteredEnum(key);
+
+ String className = getTrimmed(key);
+ try {
+ return Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getTrimmed(ConfigKey key) {
+ String value = get(key);
+
+ if (null == value) {
+ return null;
+ } else {
+ return value.trim();
+ }
+ }
+
+ public boolean containsKey(ConfigKey key) {
+ return containsKey(key.keyname());
+ }
+
+ public boolean equalKey(ConfigKey key, String another) {
+ if (containsKey(key)) {
+ return get(key).equals(another);
+ } else {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
new file mode 100644
index 0000000..5503aaa
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+import static org.apache.tajo.SessionVars.VariableMode.*;
+import static org.apache.tajo.conf.TajoConf.ConfVars;
+
+public enum SessionVars implements ConfigKey {
+
+ // Common Suffix Naming Rules:
+ //
+ // * LIMIT - We use the suffix 'LIMIT' if the variable is threshold. So, if some value is greater or less than
+ // the variable with suffix 'LIMIT', some action will be different from before.
+ // * SIZE - The suffix 'SIZE' means a data volume like bytes or mega bytes.
+ // It should be used for user's desired volume.
+ // * ENABLED - The suffix 'ENABLED' means a true or false value. If it is true, it will enable some feature.
+ // Otherwise, the feature will be turned off.
+
+
+ //-------------------------------------------------------------------------------
+ // Server Side Only Variables
+ //-------------------------------------------------------------------------------
+ SESSION_ID(ConfVars.$EMPTY, "session variable", SERVER_SIDE_VAR),
+ SESSION_LAST_ACCESS_TIME(ConfVars.$EMPTY, "last access time", SERVER_SIDE_VAR),
+
+ USERNAME(ConfVars.USERNAME, "username", SERVER_SIDE_VAR),
+ CLIENT_HOST(ConfVars.$EMPTY, "client hostname", SERVER_SIDE_VAR),
+
+ CURRENT_DATABASE(ConfVars.$EMPTY, "current database", SERVER_SIDE_VAR),
+
+ //-------------------------------------------------------------------------------
+ // Client Side Variables
+ //-------------------------------------------------------------------------------
+
+ // Client --------------------------------------------------------
+ SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "session expiry time (secs)", DEFAULT),
+
+ // Command line interface and its behavior --------------------------------
+ CLI_COLUMNS(ConfVars.$CLI_MAX_COLUMN, "Sets the width for the wrapped format", CLI_SIDE_VAR),
+ CLI_FORMATTER_CLASS(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, "Sets the output format class to display results",
+ CLI_SIDE_VAR),
+ CLI_NULL_CHAR(ConfVars.$CLI_NULL_CHAR, "Sets the string to be printed in place of a null value.", CLI_SIDE_VAR),
+
+ CLI_PAGE_ROWS(ConfVars.$CLI_PRINT_PAUSE_NUM_RECORDS, "Sets the number of rows for paging", CLI_SIDE_VAR),
+ CLI_PAGING_ENABLED(ConfVars.$CLI_PRINT_PAUSE, "Enable paging of result display", CLI_SIDE_VAR),
+ CLI_DISPLAY_ERROR_TRACE(ConfVars.$CLI_PRINT_ERROR_TRACE, "Enable display of error trace", CLI_SIDE_VAR),
+
+ ON_ERROR_STOP(ConfVars.$CLI_ERROR_STOP, "tsql will exist if an error occurs.", CLI_SIDE_VAR),
+
+ // Timezone & Date ----------------------------------------------------------
+ TZ(ConfVars.$TIMEZONE, "Sets timezone", FROM_SHELL_ENV),
+ DATE_ORDER(ConfVars.$DATE_ORDER, "date order (default is YMD)", FROM_SHELL_ENV),
+
+ // Locales and Character set ------------------------------------------------
+ // TODO - they are reserved variables, and we should support them.
+ LANG(ConfVars.$EMPTY, "Language", FROM_SHELL_ENV),
+ LC_ALL(ConfVars.$EMPTY, "String sort order", FROM_SHELL_ENV),
+ LC_COLLATE(ConfVars.$EMPTY, "String sort order", FROM_SHELL_ENV),
+ LC_CTYPE(ConfVars.$EMPTY, "Character classification (What is a letter? Its upper-case equivalent?)", FROM_SHELL_ENV),
+ LC_MESSAGES(ConfVars.$EMPTY, "Language of messages", FROM_SHELL_ENV),
+ LC_MONETARY(ConfVars.$EMPTY, "Formatting of currency amounts", FROM_SHELL_ENV),
+ LC_NUMERIC(ConfVars.$EMPTY, "Formatting of numbers", FROM_SHELL_ENV),
+ LC_TIME(ConfVars.$EMPTY, "Formatting of dates and times", FROM_SHELL_ENV),
+
+
+ // Query and Optimization ---------------------------------------------------
+
+ // for distributed query strategies
+ BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "limited size (bytes) of broadcast table",
+ DEFAULT),
+
+ JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT),
+ SORT_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_SORT_TASK_VOLUME, "sort task input size (mb)", DEFAULT),
+ GROUPBY_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_GROUPBY_TASK_VOLUME, "group by task input size (mb)", DEFAULT),
+
+ JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join (mb)", DEFAULT),
+ GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort (mb)", DEFAULT),
+ TABLE_PARTITION_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME,
+ "shuffle output size for partition table write (mb)", DEFAULT),
+
+ // for physical Executors
+ EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT),
+ HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT),
+ INNER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD,
+ "limited size for hash inner join (mb)", DEFAULT),
+ OUTER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash outer join (mb)",
+ DEFAULT),
+ HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "limited size for hash groupby (mb)",
+ DEFAULT),
+ MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT),
+
+ NULL_CHAR(ConfVars.$CSVFILE_NULL, "null char of text file output", DEFAULT),
+
+ // Behavior Control ---------------------------------------------------------
+ ARITHABORT(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT,
+ "If true, a running query will be terminated when an overflow or divide-by-zero occurs.", DEFAULT),
+
+ //-------------------------------------------------------------------------------
+ // Only for Unit Testing
+ //-------------------------------------------------------------------------------
+ DEBUG_ENABLED(ConfVars.$DEBUG_ENABLED, "(debug only) debug mode enabled", DEFAULT),
+ TEST_BROADCAST_JOIN_ENABLED(ConfVars.$TEST_BROADCAST_JOIN_ENABLED, "(test only) broadcast enabled", TEST_VAR),
+ TEST_JOIN_OPT_ENABLED(ConfVars.$TEST_JOIN_OPT_ENABLED, "(test only) join optimization enabled", TEST_VAR),
+ TEST_FILTER_PUSHDOWN_ENABLED(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED, "filter push down enabled", TEST_VAR),
+ TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", TEST_VAR),
+ ;
+
+ public static Map<String, SessionVars> SESSION_VARS = Maps.newHashMap();
+ public static Map<String, SessionVars> DEPRECATED_SESSION_VARS = Maps.newHashMap();
+
+ static {
+ for (SessionVars var : SessionVars.values()) {
+ SESSION_VARS.put(var.keyname(), var);
+ DEPRECATED_SESSION_VARS.put(var.getConfVars().keyname(), var);
+ }
+ }
+
+ private final ConfVars key;
+ private final String description;
+ private final VariableMode mode;
+
+ public static enum VariableMode {
+ DEFAULT, // Client can set or change variables of this mode..
+ FROM_SHELL_ENV, // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables.
+ SERVER_SIDE_VAR, // only TajoMaster is able to set and change variables of this mode.
+ CLI_SIDE_VAR, // This type variable is used in CLI.
+ TEST_VAR // Only used for unit tests
+ }
+
+ SessionVars(ConfVars key, String description, VariableMode mode) {
+ this.key = key;
+ this.description = description;
+ this.mode = mode;
+ }
+
+ public String keyname() {
+ return name();
+ }
+
+ public ConfigType type() {
+ return ConfigType.SESSION;
+ }
+
+ public ConfVars getConfVars() {
+ return key;
+ }
+
+ public Class<?> getVarType() {
+ return key.valClass;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public VariableMode getMode() {
+ return mode;
+ }
+
+ public static boolean exists(String keyname) {
+ return SESSION_VARS.containsKey(keyname) || DEPRECATED_SESSION_VARS.containsKey(keyname);
+ }
+
+ public static boolean isDeprecated(String keyname) {
+ return DEPRECATED_SESSION_VARS.containsKey(keyname);
+ }
+
+ public static boolean isPublic(SessionVars var) {
+ return var.getMode() != SERVER_SIDE_VAR;
+ }
+
+ public static SessionVars get(String keyname) {
+ if (SESSION_VARS.containsKey(keyname)) {
+ return SESSION_VARS.get(keyname);
+ } else if (DEPRECATED_SESSION_VARS.containsKey(keyname)) {
+ return DEPRECATED_SESSION_VARS.get(keyname);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * rename deprecated name to current name if the name is deprecated.
+ *
+ * @param keyname session variable name
+ * @return The current session variable name
+ */
+ public static String handleDeprecatedName(String keyname) {
+ return SessionVars.exists(keyname) ? SessionVars.get(keyname).keyname() : keyname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index b75530b..2504c23 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.ConfigKey;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.util.TUtil;
@@ -81,7 +82,7 @@ public class TajoConf extends Configuration {
try {
if (CURRENT_TIMEZONE == null) {
TajoConf tajoConf = new TajoConf();
- CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.TAJO_TIMEZONE));
+ CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.$TIMEZONE));
}
return CURRENT_TIMEZONE;
} finally {
@@ -105,7 +106,7 @@ public class TajoConf extends Configuration {
try {
if (DATE_ORDER < 0) {
TajoConf tajoConf = new TajoConf();
- String dateOrder = tajoConf.getVar(ConfVars.TAJO_DATE_ORDER);
+ String dateOrder = tajoConf.getVar(ConfVars.$DATE_ORDER);
if ("YMD".equals(dateOrder)) {
DATE_ORDER = DateTimeConstants.DATEORDER_YMD;
} else if ("DMY".equals(dateOrder)) {
@@ -133,11 +134,13 @@ public class TajoConf extends Configuration {
}
}
- public static enum ConfVars {
+ public static enum ConfVars implements ConfigKey {
- //////////////////////////////////
+ ///////////////////////////////////////////////////////////////////////////////////////
// Tajo System Configuration
- //////////////////////////////////
+ //
+ // They are all static configs which are not changed or not overwritten at all.
+ ///////////////////////////////////////////////////////////////////////////////////////
// a username for a running Tajo cluster
ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/"),
@@ -196,23 +199,19 @@ public class TajoConf extends Configuration {
// Catalog
CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005"),
- //////////////////////////////////
- // for Yarn Resource Manager
- //////////////////////////////////
+
+ // for Yarn Resource Manager ----------------------------------------------
+
/** how many launching TaskRunners in parallel */
YARN_RM_QUERY_MASTER_MEMORY_MB("tajo.querymaster.memory-mb", 512),
YARN_RM_QUERY_MASTER_DISKS("tajo.yarn-rm.querymaster.disks", 1),
YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM("tajo.yarn-rm.parallel-task-runner-launcher-num", 16),
YARN_RM_WORKER_NUMBER_PER_NODE("tajo.yarn-rm.max-worker-num-per-node", 8),
- //////////////////////////////////
// Query Configuration
- //////////////////////////////////
QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60),
- //////////////////////////////////
- // Shuffle Configuration
- //////////////////////////////////
+ // Shuffle Configuration --------------------------------------------------
PULLSERVER_PORT("tajo.pullserver.port", 0),
SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false),
SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"),
@@ -221,61 +220,28 @@ public class TajoConf extends Configuration {
SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 120),
SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 20),
- //////////////////////////////////
- // Storage Configuration
- //////////////////////////////////
+
+ // Storage Configuration --------------------------------------------------
ROWFILE_SYNC_INTERVAL("rowfile.sync.interval", 100),
MINIMUM_SPLIT_SIZE("tajo.min.split.size", (long) 1),
// for RCFile
HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true),
- // for Storage Manager v2
+ // for Storage Manager ----------------------------------------------------
STORAGE_MANAGER_VERSION_2("tajo.storage-manager.v2", false),
STORAGE_MANAGER_DISK_SCHEDULER_MAX_READ_BYTES_PER_SLOT("tajo.storage-manager.max-read-bytes", 8 * 1024 * 1024),
STORAGE_MANAGER_DISK_SCHEDULER_REPORT_INTERVAL("tajo.storage-manager.disk-scheduler.report-interval", 60 * 1000),
STORAGE_MANAGER_CONCURRENCY_PER_DISK("tajo.storage-manager.disk-scheduler.per-disk-concurrency", 2),
- //////////////////////////////////////////
- // Distributed Query Execution Parameters
- //////////////////////////////////////////
- DIST_QUERY_BROADCAST_JOIN_AUTO("tajo.dist-query.join.auto-broadcast", true),
- DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576),
-
- DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128),
- DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128),
- DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128),
-
- DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128),
- DIST_QUERY_SORT_PARTITION_VOLUME("tajo.dist-query.sort.partition-volume-mb", 256),
- DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256),
-
- DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256),
-
- //////////////////////////////////
- // Physical Executors
- //////////////////////////////////
- EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1),
- EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L),
- EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8),
-
- EXECUTOR_INNER_JOIN_INMEMORY_HASH_TABLE_SIZE("tajo.executor.join.inner.in-memory-table-num", (long)1000000),
- EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-bytes",
- (long)256 * 1048576),
- EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-bytes",
- (long)256 * 1048576),
- EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes",
- (long)256 * 1048576),
- //////////////////////////////////
- // RPC
- //////////////////////////////////
+ // RPC --------------------------------------------------------------------
RPC_POOL_MAX_IDLE("tajo.rpc.pool.idle.max", 10),
- //Internal RPC Client
+ // Internal RPC Client
INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM("tajo.internal.rpc.client.worker-thread-num",
Runtime.getRuntime().availableProcessors() * 2),
- //Internal RPC Server
+ // Internal RPC Server
MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num",
Runtime.getRuntime().availableProcessors() * 2),
QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.querymaster.rpc.server.worker-thread-num",
@@ -296,58 +262,92 @@ public class TajoConf extends Configuration {
WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM("tajo.worker.service.rpc.server.worker-thread-num",
Runtime.getRuntime().availableProcessors() * 1),
- //////////////////////////////////
- // The Below is reserved
- //////////////////////////////////
-
- // GeoIP
- GEOIP_DATA("tajo.function.geoip-database-location", ""),
-
- //////////////////////////////////
- // Task Configuration
+ // Task Configuration -----------------------------------------------------
TASK_DEFAULT_MEMORY("tajo.task.memory-slot-mb.default", 512),
TASK_DEFAULT_DISK("tajo.task.disk-slot.default", 0.5f),
TASK_DEFAULT_SIZE("tajo.task.size-mb", 128),
- //////////////////////////////////
- //////////////////////////////////
- // User Session Configuration
- //////////////////////////////////
- CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour.
+ // Query and Optimization -------------------------------------------------
+ EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1),
+ EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8),
+
+ EXECUTOR_INNER_JOIN_INMEMORY_HASH_TABLE_SIZE("tajo.executor.join.inner.in-memory-table-num", (long)1000000),
- // Metrics
+ // Metrics ----------------------------------------------------------------
METRICS_PROPERTY_FILENAME("tajo.metrics.property.file", "tajo-metrics.properties"),
- //CLI
- CLI_MAX_COLUMN("tajo.cli.max_columns", 120),
- CLI_PRINT_PAUSE_NUM_RECORDS("tajo.cli.print.pause.num.records", 100),
- CLI_PRINT_PAUSE("tajo.cli.print.pause", true),
- CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true),
- CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"),
- CLI_NULL_CHAR("tajo.cli.nullchar", ""),
- CLI_ERROR_STOP("tajo.cli.error.stop", false),
+ // Misc -------------------------------------------------------------------
- //TIME & DATE
- TAJO_TIMEZONE("tajo.timezone", System.getProperty("user.timezone")),
- TAJO_DATE_ORDER("tajo.date.order", "YMD"),
+ // Geo IP
+ GEOIP_DATA("tajo.function.geoip-database-location", ""),
- //PLANNER
- PLANNER_USE_FILTER_PUSHDOWN("tajo.planner.use.filter.pushdown", true),
+ /////////////////////////////////////////////////////////////////////////////////
+ // User Session Configuration
+ //
+ // All session variables begin with dollor($) sign. They are default configs
+ // for session variables. Do not directly use the following configs. Instead,
+ // please use QueryContext in order to access session variables.
+ //
+ // Also, users can change the default values of session variables in tajo-site.xml.
+ /////////////////////////////////////////////////////////////////////////////////
+
+
+ $EMPTY("tajo._", ""),
+
+ // Query and Optimization ---------------------------------------------------
+
+ // for distributed query strategies
+ $DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576),
+
+ $DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128),
+ $DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128),
+ $DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128),
+
+ $DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128),
+ $DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256),
+ $DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256),
+
+ // for physical Executors
+ $EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L),
+ $EXECUTOR_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.common.in-memory-hash-threshold-bytes",
+ (long)256 * 1048576),
+ $EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-bytes",
+ (long)256 * 1048576),
+ $EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-bytes",
+ (long)256 * 1048576),
+ $EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes",
+ (long)256 * 1048576),
+ $MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite
- // FILE FORMAT
- CSVFILE_NULL("tajo.csvfile.null", "\\\\N"),
- //OPTIMIZER
- OPTIMIZER_JOIN_ENABLE("tajo.optimizer.join.enable", true),
+ // Client -----------------------------------------------------------------
+ $CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour.
- // DEBUG OPTION
- TAJO_DEBUG("tajo.debug", false),
+ // Command line interface and its behavior --------------------------------
+ $CLI_MAX_COLUMN("tajo.cli.max_columns", 120),
+ $CLI_NULL_CHAR("tajo.cli.nullchar", ""),
+ $CLI_PRINT_PAUSE_NUM_RECORDS("tajo.cli.print.pause.num.records", 100),
+ $CLI_PRINT_PAUSE("tajo.cli.print.pause", true),
+ $CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true),
+ $CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"),
+ $CLI_ERROR_STOP("tajo.cli.error.stop", false),
- // ONLY FOR TESTCASE
- TESTCASE_MIN_TASK_NUM("tajo.testcase.min.task.num", -1),
+ // Timezone & Date ----------------------------------------------------------
+ $TIMEZONE("tajo.timezone", System.getProperty("user.timezone")),
+ $DATE_ORDER("tajo.date.order", "YMD"),
- // behavior control
- BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false);
+ // FILE FORMAT
+ $CSVFILE_NULL("tajo.csvfile.null", "\\\\N"),
+
+ // Only for Debug and Testing
+ $DEBUG_ENABLED("tajo.debug.enabled", false),
+ $TEST_BROADCAST_JOIN_ENABLED("tajo.dist-query.join.auto-broadcast", true),
+ $TEST_JOIN_OPT_ENABLED("tajo.test.plan.join-optimization.enabled", true),
+ $TEST_FILTER_PUSHDOWN_ENABLED("tajo.test.plan.filter-pushdown.enabled", true),
+ $TEST_MIN_TASK_NUM("tajo.test.min-task-num", -1),
+
+ // Behavior Control ---------------------------------------------------------
+ $BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false);
;
public final String varname;
@@ -429,6 +429,16 @@ public class TajoConf extends Configuration {
String typeString() { return name().toUpperCase();}
abstract void checkType(String value) throws Exception;
}
+
+ @Override
+ public String keyname() {
+ return varname;
+ }
+
+ @Override
+ public ConfigType type() {
+ return ConfigType.SYSTEM;
+ }
}
public static int getIntVar(Configuration conf, ConfVars var) {
@@ -450,8 +460,12 @@ public class TajoConf extends Configuration {
}
public static long getLongVar(Configuration conf, ConfVars var) {
- assert (var.valClass == Long.class);
- return conf.getLong(var.varname, var.defaultLongVal);
+ assert (var.valClass == Long.class || var.valClass == Integer.class);
+ if (var.valClass == Integer.class) {
+ return conf.getInt(var.varname, var.defaultIntVal);
+ } else {
+ return conf.getLong(var.varname, var.defaultLongVal);
+ }
}
public static long getLongVar(Configuration conf, ConfVars var, long defaultVal) {
@@ -516,7 +530,6 @@ public class TajoConf extends Configuration {
}
public static String getVar(Configuration conf, ConfVars var) {
- assert (var.valClass == String.class);
return conf.get(var.varname, var.defaultVal);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
index 19acafc..874004b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
@@ -114,7 +114,7 @@ public abstract class Datum implements Comparable<Datum>, GsonObject {
}
protected static void initAbortWhenDivideByZero(TajoConf tajoConf) {
- abortWhenDivideByZero = tajoConf.getBoolVar(ConfVars.BEHAVIOR_ARITHMETIC_ABORT);
+ abortWhenDivideByZero = tajoConf.getBoolVar(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT);
}
public abstract int size();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
index a4f79d7..532e7cd 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
@@ -25,6 +25,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type;
public class NullDatum extends Datum {
private static NullDatum instance;
+ public static final String DEFAULT_TEXT = "";
private static final byte [] EMPTY_BYTES = new byte[0];
private static final DataType NULL_DATA_TYPE;