You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2014/08/11 10:11:30 UTC
[01/17] git commit: TAJO-978: RoundFloat8 should return Float8Datum
type. (Hyoungjun Kim via hyunsik)
Repository: tajo
Updated Branches:
refs/heads/index_support 5c0277fd8 -> fa8f037bb
TAJO-978: RoundFloat8 should return Float8Datum type. (Hyoungjun Kim via hyunsik)
Closes #96
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/650157c5
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/650157c5
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/650157c5
Branch: refs/heads/index_support
Commit: 650157c5907c43a2f9228cd9a3ff59619e24efb6
Parents: 8024f6a
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri Aug 1 15:06:14 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri Aug 1 15:11:46 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../tajo/engine/function/math/RoundFloat8.java | 39 ++++++++------------
.../tajo/engine/function/TestMathFunctions.java | 27 +++++++++-----
.../tajo/engine/query/TestSelectQuery.java | 21 +++++++++++
.../TestSelectQuery/testCaseWhenRound.sql | 8 ++++
.../TestSelectQuery/testCaseWhenRound.result | 22 +++++++++++
6 files changed, 87 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/650157c5/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index f423548..76cfff7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -97,6 +97,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-978: RoundFloat8 should return Float8Datum type. (Hyoungjun Kim via
+ hyunsik)
+
TAJO-957: ROUND should be support INT parameter. (Mai Hai Thanh via hyunsik)
TAJO-980: execution page in Web UI broken. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/650157c5/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java
index 56d40e4..d104431 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java
@@ -23,13 +23,14 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
-import org.apache.tajo.engine.eval.FunctionEval;
import org.apache.tajo.engine.function.GeneralFunction;
import org.apache.tajo.engine.function.annotation.Description;
import org.apache.tajo.engine.function.annotation.ParamTypes;
+import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.storage.Tuple;
-import java.text.NumberFormat;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
/**
* Function definition
@@ -46,9 +47,6 @@ import java.text.NumberFormat;
@ParamTypes(paramTypes = {TajoDataTypes.Type.INT8, TajoDataTypes.Type.INT4})}
)
public class RoundFloat8 extends GeneralFunction {
- private NumberFormat numberFormat;
- private boolean formatConstant;
-
public RoundFloat8() {
super(new Column[] {
new Column("value", TajoDataTypes.Type.FLOAT8),
@@ -57,11 +55,6 @@ public class RoundFloat8 extends GeneralFunction {
}
@Override
- public void init(FunctionEval.ParamType [] paramTypes) {
- formatConstant = paramTypes[1] == FunctionEval.ParamType.CONSTANT;
- }
-
- @Override
public Datum eval(Tuple params) {
Datum valueDatum = params.get(0);
Datum roundDatum = params.get(1);
@@ -70,23 +63,23 @@ public class RoundFloat8 extends GeneralFunction {
return NullDatum.get();
}
- if (numberFormat == null || !formatConstant) {
- numberFormat = NumberFormat.getInstance();
- numberFormat.setGroupingUsed(false);
- numberFormat.setMaximumFractionDigits(roundDatum.asInt4());
- }
-
double value = valueDatum.asFloat8();
- int roundPnt = roundDatum.asInt4();
- double roundNum;
+ int rountPoint = roundDatum.asInt4();
- if (value > 0) {
- roundNum = (long)(value * Math.pow(10, roundPnt) + 0.5d) / Math.pow(10, roundPnt);
+ if (Double.isNaN(value)) {
+ throw new InvalidOperationException("value is not a number.");
}
- else {
- roundNum = (long)(value * Math.pow(10, roundPnt) - 0.5d) / Math.pow(10, roundPnt);
+
+ if (Double.isInfinite(value)) {
+ throw new InvalidOperationException("value is infinite.");
}
- return DatumFactory.createText(numberFormat.format(roundNum));
+ try {
+ return DatumFactory.createFloat8(BigDecimal.valueOf(value).setScale(rountPoint,
+ RoundingMode.HALF_UP).doubleValue());
+ } catch (Exception e) {
+ throw new InvalidOperationException("RoundFloat8 eval error cause " + e.getMessage() + ", value=" + value +
+ ", round point=" + rountPoint);
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/650157c5/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java
index 41fcbfe..8068180 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java
@@ -20,13 +20,13 @@ package org.apache.tajo.engine.function;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.engine.eval.ExprTestBase;
+import org.apache.tajo.exception.InvalidOperationException;
import org.junit.Test;
import java.io.IOException;
-import java.sql.ResultSet;
import static org.apache.tajo.common.TajoDataTypes.Type.*;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
public class TestMathFunctions extends ExprTestBase {
@Test
@@ -440,19 +440,26 @@ public class TestMathFunctions extends ExprTestBase {
@Test
public void testRoundWithSpecifiedPrecision() throws IOException {
+ // divide zero
+ try {
+ testSimpleEval("select round(10.0/0.0,2) ", new String[]{""});
+ fail("10.0/0 should throw InvalidOperationException");
+ } catch (InvalidOperationException e) {
+ //success
+ }
+
testSimpleEval("select round(42.4382,2) ", new String[]{"42.44"});
testSimpleEval("select round(-42.4382,2) ", new String[]{"-42.44"});
- testSimpleEval("select round(-425,2) ", new String[]{"-425"});
- testSimpleEval("select round(425,2) ", new String[]{"425"});
+ testSimpleEval("select round(-425,2) ", new String[]{"-425.0"});
+ testSimpleEval("select round(425,2) ", new String[]{"425.0"});
- testSimpleEval("select round(1234567890,0) ", new String[]{"1234567890"});
- testSimpleEval("select round(1234567890,1) ", new String[]{"1234567890"});
- testSimpleEval("select round(1234567890,2) ", new String[]{"1234567890"});
+ testSimpleEval("select round(1234567890,0) ", new String[]{"1.23456789E9"});
+ testSimpleEval("select round(1234567890,1) ", new String[]{"1.23456789E9"});
+ testSimpleEval("select round(1234567890,2) ", new String[]{"1.23456789E9"});
testSimpleEval("select round(1.2345678901234567,13) ", new String[]{"1.2345678901235"});
- testSimpleEval("select round(1234567890.1234567,3) ", new String[]{"1234567890.123"});
- testSimpleEval("select round(1234567890.1234567,5) ", new String[]{"1234567890.12346"});
- //testSimpleEval("select round(1234567890.1234567890,7) ", new String[]{"1234567890.1234568"});
+ testSimpleEval("select round(1234567890.1234567,3) ", new String[]{"1.234567890123E9"});
+ testSimpleEval("select round(1234567890.1234567,5) ", new String[]{"1.23456789012346E9"});
Schema schema = new Schema();
schema.addColumn("col1", FLOAT8);
http://git-wip-us.apache.org/repos/asf/tajo/blob/650157c5/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index 8898067..2fd7b14 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -470,4 +470,25 @@ public class TestSelectQuery extends QueryTestCaseBase {
executeString("DROP TABLE table11 PURGE");
}
}
+
+ @Test
+ public void testCaseWhenRound() throws Exception {
+ /*
+ select *
+ from (select n_nationkey as key,
+ case
+ when n_nationkey > 6 then round((n_nationkey * 100 / 2.123) / (n_regionkey * 50 / 2.123), 2) else 100.0 end as val
+ from
+ nation
+ where
+ n_regionkey > 0 and n_nationkey > 0
+ ) a
+ order by
+ a.key
+ */
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/650157c5/tajo-core/src/test/resources/queries/TestSelectQuery/testCaseWhenRound.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testCaseWhenRound.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testCaseWhenRound.sql
new file mode 100644
index 0000000..cee905a
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testCaseWhenRound.sql
@@ -0,0 +1,8 @@
+select *
+ from (select n_nationkey as key,
+ case when n_nationkey > 6 then round((n_nationkey * 100 / 2.123) / (n_regionkey * 50 / 2.123), 2) else 100.0 end as val
+ from nation
+ where n_regionkey > 0
+ and n_nationkey > 0
+ ) a
+order by a.key
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/650157c5/tajo-core/src/test/resources/results/TestSelectQuery/testCaseWhenRound.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testCaseWhenRound.result b/tajo-core/src/test/resources/results/TestSelectQuery/testCaseWhenRound.result
new file mode 100644
index 0000000..2a50cd0
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectQuery/testCaseWhenRound.result
@@ -0,0 +1,22 @@
+key,val
+-------------------------------
+1,100.0
+2,100.0
+3,100.0
+4,100.0
+6,100.0
+7,4.67
+8,8.0
+9,9.0
+10,5.0
+11,5.5
+12,12.0
+13,6.5
+17,34.0
+18,18.0
+19,12.67
+20,10.0
+21,21.0
+22,14.67
+23,15.33
+24,48.0
[11/17] git commit: TAJO-999: SequenceFile key class need to be
compatible. (jaehwa)
Posted by ji...@apache.org.
TAJO-999: SequenceFile key class need to be compatible. (jaehwa)
Closes #110
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0603b49d
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0603b49d
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0603b49d
Branch: refs/heads/index_support
Commit: 0603b49dddb5d2b3c6f6447fa672e0ea03eae09e
Parents: fcc5da0
Author: Jaehwa Jung <bl...@apache.org>
Authored: Mon Aug 11 11:36:18 2014 +0900
Committer: Jaehwa Jung <bl...@apache.org>
Committed: Mon Aug 11 11:36:18 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../sequencefile/SequenceFileAppender.java | 20 +++++++++++---------
.../sequencefile/SequenceFileScanner.java | 20 ++++++++++++++------
.../tajo/storage/TestCompressionStorages.java | 12 ++++++++++++
.../org/apache/tajo/storage/TestStorages.java | 13 +++++++++++++
5 files changed, 52 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/0603b49d/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 864eaef..deb41a0 100644
--- a/CHANGES
+++ b/CHANGES
@@ -106,6 +106,8 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-999: SequenceFile key class need to be compatible. (jaehwa)
+
TAJO-994: 'count(distinct x)' function counts first null value. (hyunsik)
TAJO-996: Sometimes, scheduleFetchesByEvenDistributedVolumes loses
http://git-wip-us.apache.org/repos/asf/tajo/blob/0603b49d/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
index b150a9a..86d902a 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
@@ -25,11 +25,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.*;
import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.tajo.catalog.Schema;
@@ -73,7 +70,8 @@ public class SequenceFileAppender extends FileAppender {
long rowCount;
private boolean isShuffle;
- private static final BytesWritable EMPTY_KEY = new BytesWritable();
+
+ private Writable EMPTY_KEY;
public SequenceFileAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException {
super(conf, schema, meta, path);
@@ -128,20 +126,24 @@ public class SequenceFileAppender extends FileAppender {
throw new IOException(e);
}
- Class<? extends Writable> valueClass;
+ Class<? extends Writable> keyClass, valueClass;
if (serde instanceof BinarySerializerDeserializer) {
+ keyClass = BytesWritable.class;
+ EMPTY_KEY = new BytesWritable();
valueClass = BytesWritable.class;
} else {
+ keyClass = LongWritable.class;
+ EMPTY_KEY = new LongWritable();
valueClass = Text.class;
}
String type = this.meta.getOption(StorageConstants.COMPRESSION_TYPE, CompressionType.NONE.name());
if (type.equals(CompressionType.BLOCK.name())) {
- writer = SequenceFile.createWriter(fs, conf, path, BytesWritable.class, valueClass, CompressionType.BLOCK, codec);
+ writer = SequenceFile.createWriter(fs, conf, path, keyClass, valueClass, CompressionType.BLOCK, codec);
} else if (type.equals(CompressionType.RECORD.name())) {
- writer = SequenceFile.createWriter(fs, conf, path, BytesWritable.class, valueClass, CompressionType.RECORD, codec);
+ writer = SequenceFile.createWriter(fs, conf, path, keyClass, valueClass, CompressionType.RECORD, codec);
} else {
- writer = SequenceFile.createWriter(fs, conf, path, BytesWritable.class, valueClass, CompressionType.NONE, codec);
+ writer = SequenceFile.createWriter(fs, conf, path, keyClass, valueClass, CompressionType.NONE, codec);
}
if (enabledStats) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0603b49d/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
index 32d1d57..3c39841 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
@@ -24,10 +24,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -37,6 +35,7 @@ import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.BytesUtils;
+import org.apache.tajo.util.ReflectionUtil;
import java.io.IOException;
@@ -72,7 +71,7 @@ public class SequenceFileScanner extends FileScanner {
private int elementOffset, elementSize;
- private static final BytesWritable EMPTY_KEY = new BytesWritable();
+ private Writable EMPTY_KEY;
public SequenceFileScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) throws IOException {
super(conf, schema, meta, fragment);
@@ -120,8 +119,13 @@ public class SequenceFileScanner extends FileScanner {
String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
- if (serde instanceof BinarySerializerDeserializer)
+ if (serde instanceof BinarySerializerDeserializer) {
hasBinarySerDe = true;
+ }
+
+ Class<? extends Writable> keyClass = (Class<? extends Writable>)Class.forName(reader.getKeyClassName());
+ EMPTY_KEY = keyClass.newInstance();
+
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new IOException(e);
@@ -129,6 +133,10 @@ public class SequenceFileScanner extends FileScanner {
super.init();
}
+ public Writable getKey() {
+ return EMPTY_KEY;
+ }
+
private void prepareProjection(Column [] targets) {
projectionMap = new int[targets.length];
http://git-wip-us.apache.org/repos/asf/tajo/blob/0603b49d/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
index 9fe5721..61f4682 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
@@ -21,7 +21,10 @@ package org.apache.tajo.storage;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.*;
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
import org.apache.hadoop.util.NativeCodeLoader;
@@ -34,6 +37,7 @@ import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.storage.fragment.FileFragment;
+import org.apache.tajo.storage.sequencefile.SequenceFileScanner;
import org.apache.tajo.util.CommonTestingUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -44,6 +48,7 @@ import java.util.Arrays;
import java.util.Collection;
import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
@RunWith(Parameterized.class)
public class TestCompressionStorages {
@@ -228,6 +233,13 @@ public class TestCompressionStorages {
}
}
scanner.init();
+
+ if (storeType == StoreType.SEQUENCEFILE) {
+ assertTrue(scanner instanceof SequenceFileScanner);
+ Writable key = ((SequenceFileScanner) scanner).getKey();
+ assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
+ }
+
int tupleCnt = 0;
Tuple tuple;
while ((tuple = scanner.next()) != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0603b49d/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index 1cf1ecf..3bea740 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -21,6 +21,9 @@ package org.apache.tajo.storage;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Writable;
import org.apache.tajo.QueryId;
import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.catalog.CatalogUtil;
@@ -36,6 +39,7 @@ import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.datum.ProtobufDatumFactory;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.rcfile.RCFile;
+import org.apache.tajo.storage.sequencefile.SequenceFileScanner;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.KeyValueSet;
@@ -48,6 +52,7 @@ import java.util.Arrays;
import java.util.Collection;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@RunWith(Parameterized.class)
@@ -640,6 +645,10 @@ public class TestStorages {
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
+ assertTrue(scanner instanceof SequenceFileScanner);
+ Writable key = ((SequenceFileScanner) scanner).getKey();
+ assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
+
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
@@ -709,6 +718,10 @@ public class TestStorages {
Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
+ assertTrue(scanner instanceof SequenceFileScanner);
+ Writable key = ((SequenceFileScanner) scanner).getKey();
+ assertEquals(key.getClass().getCanonicalName(), BytesWritable.class.getCanonicalName());
+
Tuple retrieved;
while ((retrieved=scanner.next()) != null) {
for (int i = 0; i < tuple.size(); i++) {
[15/17] TAJO-928: Session variables should override query configs in
TajoConf.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
index 4d1cee1..0c3db6d 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
@@ -19,10 +19,13 @@
package org.apache.tajo.util;
import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
+import org.apache.tajo.OverridableConf;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.json.CommonGsonHelper;
import org.apache.tajo.json.GsonObject;
+import sun.misc.FloatingDecimal;
import java.util.HashMap;
import java.util.Map;
@@ -32,6 +35,9 @@ import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto;
import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto;
public class KeyValueSet implements ProtoObject<KeyValueSetProto>, Cloneable, GsonObject {
+ public static final String TRUE_STR = "true";
+ public static final String FALSE_STR = "false";
+
private KeyValueSetProto.Builder builder = KeyValueSetProto.newBuilder();
@Expose private Map<String,String> keyVals;
@@ -63,40 +69,133 @@ public class KeyValueSet implements ProtoObject<KeyValueSetProto>, Cloneable, Gs
public int size() {
return keyVals.size();
}
-
- public void put(String key, String val) {
- this.keyVals.put(key, val);
- }
public void putAll(Map<String, String> keyValues) {
if (keyValues != null) {
this.keyVals.putAll(keyValues);
}
}
-
- public void putAll(KeyValueSet keyValueSet) {
+
+ public void putAll(KeyValueSet keyValueSet) {
if (keyValueSet != null) {
- this.keyVals.putAll(keyValueSet.keyVals);
+ this.keyVals.putAll(keyValueSet.keyVals);
}
- }
-
- public String get(String key) {
- return this.keyVals.get(key);
- }
-
- public String get(String key, String defaultVal) {
- if(keyVals.containsKey(key))
- return keyVals.get(key);
- else {
- return defaultVal;
- }
- }
-
- public Map<String,String> getAllKeyValus() {
- return keyVals;
- }
+ }
+
+ public Map<String,String> getAllKeyValus() {
+ return keyVals;
+ }
+
+ public boolean containsKey(String key) {
+ return this.keyVals.containsKey(key);
+ }
+
+ public void set(String key, String val) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(val);
+
+ this.keyVals.put(key, val);
+ }
+
+ public String get(String key, String defaultVal) {
+ if(keyVals.containsKey(key)) {
+ return keyVals.get(key);
+ } else if (defaultVal != null) {
+ return defaultVal;
+ } else {
+ throw new IllegalArgumentException("No such a config key: " + key);
+ }
+ }
+
+ public String get(String key) {
+ return get(key, null);
+ }
+
+ public void setBool(String key, boolean val) {
+ set(key, val ? TRUE_STR : FALSE_STR);
+ }
+
+ public boolean getBool(String key, Boolean defaultVal) {
+ if (containsKey(key)) {
+ String strVal = get(key, null);
+ return strVal != null ? strVal.equalsIgnoreCase(TRUE_STR) : false;
+ } else if (defaultVal != null) {
+ return defaultVal;
+ } else {
+ return false;
+ }
+ }
+
+ public boolean getBool(String key) {
+ return getBool(key, null);
+ }
+
+ public void setInt(String key, int val) {
+ set(key, String.valueOf(val));
+ }
+
+ public int getInt(String key, Integer defaultVal) {
+ if (containsKey(key)) {
+ String strVal = get(key, null);
+ return Integer.parseInt(strVal);
+ } else if (defaultVal != null) {
+ return defaultVal;
+ } else {
+ throw new IllegalArgumentException("No such a config key: " + key);
+ }
+ }
+
+ public int getInt(String key) {
+ return getInt(key, null);
+ }
+
+ public void setLong(String key, long val) {
+ set(key, String.valueOf(val));
+ }
+
+ public long getLong(String key, Long defaultVal) {
+ if (containsKey(key)) {
+ String strVal = get(key, null);
+ return Long.parseLong(strVal);
+ } else if (defaultVal != null) {
+ return defaultVal;
+ } else {
+ throw new IllegalArgumentException("No such a config key: " + key);
+ }
+ }
+
+ public long getLong(String key) {
+ return getLong(key, null);
+ }
+
+ public void setFloat(String key, float val) {
+ set(key, String.valueOf(val));
+ }
+
+ public float getFloat(String key, Float defaultVal) {
+ if (containsKey(key)) {
+ String strVal = get(key, null);
+ try {
+ sun.misc.FloatingDecimal fd = FloatingDecimal.readJavaFormatString(strVal);
+ if (Float.MAX_VALUE < fd.doubleValue()) {
+ throw new IllegalStateException("Parsed value is overflow in float type");
+ }
+ return fd.floatValue();
+ } catch (NumberFormatException nfe) {
+ throw new IllegalArgumentException("No such a config key: " + key);
+ }
+ } else if (defaultVal != null) {
+ return defaultVal.floatValue();
+ } else {
+ throw new IllegalArgumentException("No such a config key: " + key);
+ }
+ }
+
+ public float getFloat(String key) {
+ return getFloat(key, null);
+ }
- public String delete(String key) {
+ public String remove(String key) {
return keyVals.remove(key);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java
index adf80b0..42623bd 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java
@@ -52,9 +52,9 @@ public class TestArithmeticOperator {
public void setUp() {
TajoConf tajoConf = new TajoConf();
if ("Zero_Exception".equals(option)) {
- tajoConf.setBoolVar(ConfVars.BEHAVIOR_ARITHMETIC_ABORT, true);
+ tajoConf.setBoolVar(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT, true);
} else {
- tajoConf.setBoolVar(ConfVars.BEHAVIOR_ARITHMETIC_ABORT, false);
+ tajoConf.setBoolVar(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT, false);
}
Datum.initAbortWhenDivideByZero(tajoConf);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java
index 3ffeeb0..a43cc1a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
@@ -35,7 +36,7 @@ import org.apache.tajo.engine.planner.logical.join.GreedyHeuristicJoinOrderAlgor
import org.apache.tajo.engine.planner.logical.join.JoinGraph;
import org.apache.tajo.engine.planner.logical.join.JoinOrderAlgorithm;
import org.apache.tajo.engine.planner.rewrite.*;
-import org.apache.tajo.master.session.Session;
+import org.apache.tajo.engine.query.QueryContext;
import java.util.LinkedHashSet;
import java.util.Set;
@@ -57,7 +58,7 @@ public class LogicalOptimizer {
public LogicalOptimizer(TajoConf systemConf) {
rulesBeforeJoinOpt = new BasicQueryRewriteEngine();
- if (systemConf.getBoolVar(ConfVars.PLANNER_USE_FILTER_PUSHDOWN)) {
+ if (systemConf.getBoolVar(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED)) {
rulesBeforeJoinOpt.addRewriteRule(new FilterPushDownRule());
}
@@ -84,13 +85,13 @@ public class LogicalOptimizer {
return optimize(null, plan);
}
- public LogicalNode optimize(Session session, LogicalPlan plan) throws PlanningException {
+ public LogicalNode optimize(QueryContext context, LogicalPlan plan) throws PlanningException {
rulesBeforeJoinOpt.rewrite(plan);
DirectedGraphCursor<String, BlockEdge> blockCursor =
new DirectedGraphCursor<String, BlockEdge>(plan.getQueryBlockGraph(), plan.getRootBlock().getName());
- if (session == null || "true".equals(session.getVariable(ConfVars.OPTIMIZER_JOIN_ENABLE.varname, "true"))) {
+ if (context == null || context.getBool(SessionVars.TEST_JOIN_OPT_ENABLED)) {
// default is true
while (blockCursor.hasNext()) {
optimizeJoinOrder(plan, blockCursor.nextBlock());
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
index 86bacef..ee65b2b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
@@ -67,10 +67,8 @@ public class LogicalPlan {
LogicalPlanner planner;
private boolean isExplain;
- private final String currentDatabase;
- public LogicalPlan(String currentDatabase, LogicalPlanner planner) {
- this.currentDatabase = currentDatabase;
+ public LogicalPlan(LogicalPlanner planner) {
this.planner = planner;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
index 84fe6c2..6ee0ff8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
@@ -26,10 +26,10 @@ import org.apache.tajo.engine.eval.FieldEval;
import org.apache.tajo.engine.exception.NoSuchColumnException;
import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock;
import org.apache.tajo.engine.planner.logical.*;
-import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
import org.apache.tajo.engine.planner.nameresolver.NameResolver;
+import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.engine.utils.SchemaUtil;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -42,18 +42,18 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
private ExprAnnotator annotator;
public static class PreprocessContext {
- public Session session;
+ public QueryContext queryContext;
public LogicalPlan plan;
public LogicalPlan.QueryBlock currentBlock;
- public PreprocessContext(Session session, LogicalPlan plan, LogicalPlan.QueryBlock currentBlock) {
- this.session = session;
+ public PreprocessContext(QueryContext queryContext, LogicalPlan plan, LogicalPlan.QueryBlock currentBlock) {
+ this.queryContext = queryContext;
this.plan = plan;
this.currentBlock = currentBlock;
}
public PreprocessContext(PreprocessContext context, LogicalPlan.QueryBlock currentBlock) {
- this.session = context.session;
+ this.queryContext = context.queryContext;
this.plan = context.plan;
this.currentBlock = currentBlock;
}
@@ -104,7 +104,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
if (CatalogUtil.isFQTableName(asteriskExpr.getQualifier())) {
qualifier = asteriskExpr.getQualifier();
} else {
- qualifier = CatalogUtil.buildFQName(ctx.session.getCurrentDatabase(), asteriskExpr.getQualifier());
+ qualifier = CatalogUtil.buildFQName(ctx.queryContext.getCurrentDatabase(), asteriskExpr.getQualifier());
}
relationOp = block.getRelation(qualifier);
@@ -359,7 +359,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
if (CatalogUtil.isFQTableName(expr.getName())) {
actualRelationName = relation.getName();
} else {
- actualRelationName = CatalogUtil.buildFQName(ctx.session.getCurrentDatabase(), relation.getName());
+ actualRelationName = CatalogUtil.buildFQName(ctx.queryContext.getCurrentDatabase(), relation.getName());
}
TableDesc desc = catalog.getTableDesc(actualRelationName);
@@ -388,7 +388,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
// a table subquery should be dealt as a relation.
TableSubQueryNode node = ctx.plan.createNode(TableSubQueryNode.class);
- node.init(CatalogUtil.buildFQName(ctx.session.getCurrentDatabase(), expr.getName()), child);
+ node.init(CatalogUtil.buildFQName(ctx.queryContext.getCurrentDatabase(), expr.getName()), child);
ctx.currentBlock.addRelation(node);
return node;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java
index bb8192f..6512ae0 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java
@@ -24,7 +24,7 @@ import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.logical.*;
-import org.apache.tajo.master.session.Session;
+import org.apache.tajo.engine.query.QueryContext;
import java.util.Stack;
@@ -38,17 +38,17 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
}
public static class Context {
- Session session;
+ QueryContext queryContext;
VerificationState state;
- public Context(Session session, VerificationState state) {
- this.session = session;
+ public Context(QueryContext queryContext, VerificationState state) {
+ this.queryContext = this.queryContext;
this.state = state;
}
}
- public VerificationState verify(Session session, VerificationState state, LogicalPlan plan) throws PlanningException {
- Context context = new Context(session, state);
+ public VerificationState verify(QueryContext queryContext, VerificationState state, LogicalPlan plan) throws PlanningException {
+ Context context = new Context(queryContext, state);
visit(context, plan, plan.getRootBlock());
return context.state;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index a4820cb..35df11f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -42,17 +42,16 @@ import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock;
import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.engine.utils.SchemaUtil;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.StorageUtil;
-import org.apache.tajo.util.Pair;
import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import java.util.*;
import static org.apache.tajo.algebra.CreateTable.PartitionType;
-
import static org.apache.tajo.engine.planner.ExprNormalizer.ExprNormalizedResult;
import static org.apache.tajo.engine.planner.LogicalPlan.BlockType;
import static org.apache.tajo.engine.planner.LogicalPlanPreprocessor.PreprocessContext;
@@ -75,7 +74,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
}
public static class PlanContext {
- Session session;
+ QueryContext queryContext;
LogicalPlan plan;
// transient data for each query block
@@ -83,15 +82,15 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
boolean debugOrUnitTests;
- public PlanContext(Session session, LogicalPlan plan, QueryBlock block, boolean debugOrUnitTests) {
- this.session = session;
+ public PlanContext(QueryContext context, LogicalPlan plan, QueryBlock block, boolean debugOrUnitTests) {
+ this.queryContext = context;
this.plan = plan;
this.queryBlock = block;
this.debugOrUnitTests = debugOrUnitTests;
}
public PlanContext(PlanContext context, QueryBlock block) {
- this.session = context.session;
+ this.queryContext = context.queryContext;
this.plan = context.plan;
this.queryBlock = block;
this.debugOrUnitTests = context.debugOrUnitTests;
@@ -109,21 +108,21 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
* @param expr A relational algebraic expression for a query.
* @return A logical plan
*/
- public LogicalPlan createPlan(Session session, Expr expr) throws PlanningException {
- return createPlan(session, expr, false);
+ public LogicalPlan createPlan(QueryContext context, Expr expr) throws PlanningException {
+ return createPlan(context, expr, false);
}
@VisibleForTesting
- public LogicalPlan createPlan(Session session, Expr expr, boolean debug) throws PlanningException {
+ public LogicalPlan createPlan(QueryContext queryContext, Expr expr, boolean debug) throws PlanningException {
- LogicalPlan plan = new LogicalPlan(session.getCurrentDatabase(), this);
+ LogicalPlan plan = new LogicalPlan(this);
QueryBlock rootBlock = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK);
- PreprocessContext preProcessorCtx = new PreprocessContext(session, plan, rootBlock);
+ PreprocessContext preProcessorCtx = new PreprocessContext(queryContext, plan, rootBlock);
preprocessor.visit(preProcessorCtx, new Stack<Expr>(), expr);
plan.resetGeneratedId();
- PlanContext context = new PlanContext(session, plan, plan.getRootBlock(), debug);
+ PlanContext context = new PlanContext(queryContext, plan, plan.getRootBlock(), debug);
LogicalNode topMostNode = this.visit(context, new Stack<Expr>(), expr);
// Add Root Node
@@ -1424,7 +1423,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
databaseName = CatalogUtil.extractQualifier(expr.getTableName());
tableName = CatalogUtil.extractSimpleName(expr.getTableName());
} else {
- databaseName = context.session.getCurrentDatabase();
+ databaseName = context.queryContext.getCurrentDatabase();
tableName = expr.getTableName();
}
TableDesc desc = catalog.getTableDesc(databaseName, tableName);
@@ -1624,7 +1623,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
if (CatalogUtil.isFQTableName(parentTableName) == false) {
parentTableName =
- CatalogUtil.buildFQName(context.session.getCurrentDatabase(),
+ CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(),
parentTableName);
}
TableDesc parentTableDesc = catalog.getTableDesc(parentTableName);
@@ -1657,7 +1656,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setTableName(expr.getTableName());
} else {
createTableNode.setTableName(
- CatalogUtil.buildFQName(context.session.getCurrentDatabase(), expr.getTableName()));
+ CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), expr.getTableName()));
}
// This is CREATE TABLE <tablename> LIKE <parentTable>
if(expr.getLikeParentTableName() != null)
@@ -1753,7 +1752,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
CreateTable.ColumnPartition partition = (CreateTable.ColumnPartition) expr;
String partitionExpression = Joiner.on(',').join(partition.getColumns());
- partitionMethodDesc = new PartitionMethodDesc(context.session.getCurrentDatabase(), tableName,
+ partitionMethodDesc = new PartitionMethodDesc(context.queryContext.getCurrentDatabase(), tableName,
CatalogProtos.PartitionType.COLUMN, partitionExpression, convertColumnsToSchema(partition.getColumns()));
} else {
throw new PlanningException(String.format("Not supported PartitonType: %s", expr.getPartitionType()));
@@ -1816,7 +1815,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
if (CatalogUtil.isFQTableName(dropTable.getTableName())) {
qualified = dropTable.getTableName();
} else {
- qualified = CatalogUtil.buildFQName(context.session.getCurrentDatabase(), dropTable.getTableName());
+ qualified = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), dropTable.getTableName());
}
dropTableNode.init(qualified, dropTable.isIfExists(), dropTable.isPurge());
return dropTableNode;
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index 6678e46..9f533e2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -28,6 +28,7 @@ import com.google.common.collect.ObjectArrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -37,6 +38,7 @@ import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.engine.planner.physical.*;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.InternalException;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer;
@@ -57,7 +59,6 @@ import java.util.Stack;
import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType;
-import static org.apache.tajo.conf.TajoConf.ConfVars;
import static org.apache.tajo.ipc.TajoWorkerProtocol.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
import static org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty;
import static org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty.EnforceType;
@@ -68,7 +69,6 @@ import static org.apache.tajo.ipc.TajoWorkerProtocol.SortEnforce;
public class PhysicalPlannerImpl implements PhysicalPlanner {
private static final Log LOG = LogFactory.getLog(PhysicalPlannerImpl.class);
private static final int UNGENERATED_PID = -1;
- private final long INNER_JOIN_INMEMORY_HASH_THRESHOLD;
protected final TajoConf conf;
protected final AbstractStorageManager sm;
@@ -76,8 +76,6 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
public PhysicalPlannerImpl(final TajoConf conf, final AbstractStorageManager sm) {
this.conf = conf;
this.sm = sm;
-
- this.INNER_JOIN_INMEMORY_HASH_THRESHOLD = conf.getLongVar(ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD);
}
public PhysicalExec createPlan(final TaskAttemptContext context, final LogicalNode logicalPlan)
@@ -258,7 +256,16 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
throws IOException {
String [] lineage = PlannerUtil.getRelationLineage(node);
long volume = estimateSizeRecursive(context, lineage);
- boolean inMemoryInnerJoinFlag = volume <= INNER_JOIN_INMEMORY_HASH_THRESHOLD;
+ boolean inMemoryInnerJoinFlag = false;
+
+ QueryContext queryContext = context.getQueryContext();
+
+ if (queryContext.containsKey(SessionVars.INNER_HASH_JOIN_SIZE_LIMIT)) {
+ inMemoryInnerJoinFlag = volume <= context.getQueryContext().getLong(SessionVars.INNER_HASH_JOIN_SIZE_LIMIT);
+ } else {
+ inMemoryInnerJoinFlag = volume <= context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT);
+ }
+
LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.",
context.getTaskId().toString(),
(left ? "Left" : "Right"),
@@ -470,8 +477,17 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
PhysicalExec leftExec, PhysicalExec rightExec) throws IOException {
String [] rightLineage = PlannerUtil.getRelationLineage(plan.getRightChild());
long rightTableVolume = estimateSizeRecursive(context, rightLineage);
+ boolean hashJoin;
- if (rightTableVolume < conf.getLongVar(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)) {
+ QueryContext queryContext = context.getQueryContext();
+
+ if (queryContext.containsKey(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT)) {
+ hashJoin = rightTableVolume < queryContext.getLong(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT);
+ } else {
+ hashJoin = rightTableVolume < queryContext.getLong(SessionVars.HASH_JOIN_SIZE_LIMIT);
+ }
+
+ if (hashJoin) {
// we can implement left outer join using hash join, using the right operand as the build relation
LOG.info("Left Outer Join (" + plan.getPID() +") chooses [Hash Join].");
return new HashLeftOuterJoinExec(context, plan, leftExec, rightExec);
@@ -488,8 +504,18 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
//if the left operand is small enough => implement it as a left outer hash join with exchanged operators (note:
// blocking, but merge join is blocking as well)
String [] outerLineage4 = PlannerUtil.getRelationLineage(plan.getLeftChild());
- long outerSize = estimateSizeRecursive(context, outerLineage4);
- if (outerSize < conf.getLongVar(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)){
+ long leftTableVolume = estimateSizeRecursive(context, outerLineage4);
+ boolean hashJoin;
+
+ QueryContext queryContext = context.getQueryContext();
+
+ if (queryContext.containsKey(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT)) {
+ hashJoin = leftTableVolume < queryContext.getLong(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT);
+ } else {
+ hashJoin = leftTableVolume < queryContext.getLong(SessionVars.HASH_JOIN_SIZE_LIMIT);
+ }
+
+ if (hashJoin){
LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Hash Join].");
return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec);
} else {
@@ -971,7 +997,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
String [] outerLineage = PlannerUtil.getRelationLineage(groupbyNode.getChild());
long estimatedSize = estimateSizeRecursive(context, outerLineage);
- final long threshold = conf.getLongVar(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD);
+ final long threshold = context.getQueryContext().getLong(SessionVars.HASH_GROUPBY_SIZE_LIMIT);
// if the relation size is less than the threshold,
// the hash aggregation will be used.
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java
index 2d6c095..75dcc18 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java
@@ -24,7 +24,7 @@ import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.master.session.Session;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.util.TUtil;
import java.util.Set;
@@ -38,17 +38,17 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor <PreLogicalPlanVe
}
public static class Context {
- Session session;
+ QueryContext queryContext;
VerificationState state;
- public Context(Session session, VerificationState state) {
- this.session = session;
+ public Context(QueryContext queryContext, VerificationState state) {
+ this.queryContext = queryContext;
this.state = state;
}
}
- public VerificationState verify(Session session, VerificationState state, Expr expr) throws PlanningException {
- Context context = new Context(session, state);
+ public VerificationState verify(QueryContext queryContext, VerificationState state, Expr expr) throws PlanningException {
+ Context context = new Context(queryContext, state);
visit(context, new Stack<Expr>(), expr);
return context.state;
}
@@ -127,7 +127,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor <PreLogicalPlanVe
if (CatalogUtil.isFQTableName(tableName)) {
qualifiedName = tableName;
} else {
- qualifiedName = CatalogUtil.buildFQName(context.session.getCurrentDatabase(), tableName);
+ qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), tableName);
}
if (!catalog.existsTable(qualifiedName)) {
@@ -143,7 +143,10 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor <PreLogicalPlanVe
if (CatalogUtil.isFQTableName(tableName)) {
qualifiedName = tableName;
} else {
- qualifiedName = CatalogUtil.buildFQName(context.session.getCurrentDatabase(), tableName);
+ qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), tableName);
+ }
+ if(qualifiedName == null) {
+ System.out.println("A");
}
if (catalog.existsTable(qualifiedName)) {
context.state.addVerification(String.format("relation \"%s\" already exists", qualifiedName));
@@ -246,7 +249,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor <PreLogicalPlanVe
if (expr.hasTableName()) {
String qualifiedName = expr.getTableName();
if (TajoConstants.EMPTY_STRING.equals(CatalogUtil.extractQualifier(expr.getTableName()))) {
- qualifiedName = CatalogUtil.buildFQName(context.session.getCurrentDatabase(),
+ qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(),
expr.getTableName());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 2daf799..432589b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
@@ -120,9 +121,8 @@ public class GlobalPlanner {
LogicalNode inputPlan = PlannerUtil.clone(masterPlan.getLogicalPlan(),
masterPlan.getLogicalPlan().getRootBlock().getRoot());
- boolean autoBroadcast = conf.getBoolVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO);
- if (autoBroadcast) {
-
+ boolean broadcastEnabled = masterPlan.getContext().getBool(SessionVars.TEST_BROADCAST_JOIN_ENABLED);
+ if (broadcastEnabled) {
// pre-visit the master plan in order to find tables to be broadcasted
// this visiting does not make any execution block and change plan.
BroadcastJoinMarkCandidateVisitor markCandidateVisitor = new BroadcastJoinMarkCandidateVisitor();
@@ -268,11 +268,11 @@ public class GlobalPlanner {
MasterPlan masterPlan = context.plan;
ExecutionBlock currentBlock;
- boolean autoBroadcast = conf.getBoolVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO);
- long broadcastThreshold = conf.getLongVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD);
+ boolean broadcastEnabled = context.getPlan().getContext().getBool(SessionVars.TEST_BROADCAST_JOIN_ENABLED);
+ long broadcastTableSizeLimit = context.getPlan().getContext().getLong(SessionVars.BROADCAST_TABLE_SIZE_LIMIT);
// to check when the tajo.dist-query.join.broadcast.auto property is true
- if (autoBroadcast && joinNode.isCandidateBroadcast()) {
+ if (broadcastEnabled && joinNode.isCandidateBroadcast()) {
LogicalNode leftNode = joinNode.getLeftChild();
LogicalNode rightNode = joinNode.getRightChild();
@@ -293,7 +293,7 @@ public class GlobalPlanner {
// Checking Left Side of Join
if (ScanNode.isScanNode(leftNode)) {
ScanNode scanNode = (ScanNode)leftNode;
- if (joinNode.getJoinType() == JoinType.LEFT_OUTER || getTableVolume(scanNode) >= broadcastThreshold) {
+ if (joinNode.getJoinType() == JoinType.LEFT_OUTER || getTableVolume(scanNode) >= broadcastTableSizeLimit) {
numLargeTables++;
} else {
leftBroadcast = true;
@@ -306,7 +306,7 @@ public class GlobalPlanner {
// Checking Right Side OF Join
if (ScanNode.isScanNode(rightNode)) {
ScanNode scanNode = (ScanNode)rightNode;
- if (joinNode.getJoinType() == JoinType.RIGHT_OUTER || getTableVolume(scanNode) >= broadcastThreshold) {
+ if (joinNode.getJoinType() == JoinType.RIGHT_OUTER || getTableVolume(scanNode) >= broadcastTableSizeLimit) {
numLargeTables++;
} else {
rightBroadcast = true;
@@ -331,7 +331,7 @@ public class GlobalPlanner {
}
JoinNode broadcastJoinNode = (JoinNode)eachNode;
ScanNode scanNode = broadcastJoinNode.getRightChild();
- if (getTableVolume(scanNode) < broadcastThreshold) {
+ if (getTableVolume(scanNode) < broadcastTableSizeLimit) {
broadcastTargetScanNodes.add(scanNode);
blockJoinNode = broadcastJoinNode;
LOG.info("The table " + scanNode.getCanonicalName() + " ("
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
index f714758..31cb3b6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.IOUtils;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
@@ -38,6 +39,7 @@ import org.apache.tajo.storage.*;
import org.apache.tajo.storage.Scanner;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
+import org.apache.tajo.unit.StorageUnit;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -111,7 +113,7 @@ public class ExternalSortExec extends SortExec {
throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2");
}
// TODO - sort buffer and core num should be changed to use the allocated container resource.
- this.sortBufferBytesNum = context.getConf().getLongVar(ConfVars.EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE) * 1048576L;
+ this.sortBufferBytesNum = context.getQueryContext().getLong(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB;
this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM);
this.executorService = Executors.newFixedThreadPool(this.allocatedCoreNum);
this.inMemoryTable = new ArrayList<Tuple>(100000);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
index b1d0400..e73cc2f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
@@ -18,9 +18,9 @@
package org.apache.tajo.engine.planner.physical;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.logical.InsertNode;
import org.apache.tajo.engine.planner.logical.PersistentStoreNode;
import org.apache.tajo.storage.Appender;
@@ -59,7 +59,7 @@ public class StoreTableExec extends UnaryPhysicalExec {
appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta,
createTableNode.getTableSchema(), context.getOutputPath());
} else {
- String nullChar = context.getQueryContext().get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal);
+ String nullChar = context.getQueryContext().get(SessionVars.NULL_CHAR);
meta.putOption(StorageConstants.CSVFILE_NULL, nullChar);
appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema,
context.getOutputPath());
@@ -77,7 +77,7 @@ public class StoreTableExec extends UnaryPhysicalExec {
while((tuple = child.next()) != null) {
appender.addTuple(tuple);
}
-
+
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java
index 79d6cb3..f4160e4 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java
@@ -19,94 +19,90 @@
package org.apache.tajo.engine.query;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.ConfigKey;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.logical.NodeType;
+import org.apache.tajo.master.session.Session;
import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto;
-public class QueryContext extends KeyValueSet {
- public static final String COMMAND_TYPE = "tajo.query.command";
-
- public static final String STAGING_DIR = "tajo.query.staging_dir";
-
- public static final String USER_NAME = "tajo.query.username";
-
- public static final String OUTPUT_TABLE_NAME = "tajo.query.output.table";
- public static final String OUTPUT_TABLE_PATH = "tajo.query.output.path";
- public static final String OUTPUT_PARTITIONS = "tajo.query.output.partitions";
- public static final String OUTPUT_OVERWRITE = "tajo.query.output.overwrite";
- public static final String OUTPUT_AS_DIRECTORY = "tajo.query.output.asdirectory";
+/**
+ * QueryContent is a overridable config, and it provides a set of various configs for a query instance.
+ */
+public class QueryContext extends OverridableConf {
+ public static enum QueryVars implements ConfigKey {
+ COMMAND_TYPE,
+ STAGING_DIR,
+ OUTPUT_TABLE_NAME,
+ OUTPUT_TABLE_PATH,
+ OUTPUT_PARTITIONS,
+ OUTPUT_OVERWRITE,
+ OUTPUT_AS_DIRECTORY,
+ OUTPUT_PER_FILE_SIZE,
+ ;
- public static final String TRUE_VALUE = "1";
- public static final String FALSE_VALUE = "0";
+ QueryVars() {
+ }
- public QueryContext() {}
+ @Override
+ public String keyname() {
+ return name().toLowerCase();
+ }
- public QueryContext(KeyValueSetProto proto) {
- super(proto);
+ @Override
+ public ConfigType type() {
+ return ConfigType.QUERY;
+ }
}
- public void put(TajoConf.ConfVars key, String value) {
- put(key.varname, value);
+ public QueryContext(TajoConf conf) {
+ super(conf, ConfigKey.ConfigType.QUERY);
}
- public String get(TajoConf.ConfVars key) {
- return get(key.varname);
+ public QueryContext(TajoConf conf, Session session) {
+ super(conf);
+ putAll(session.getAllVariables());
}
- public String get(String key) {
- return super.get(key);
+ public QueryContext(TajoConf conf, KeyValueSetProto proto) {
+ super(conf, proto, ConfigKey.ConfigType.QUERY);
}
- public void setBool(String key, boolean val) {
- put(key, val ? TRUE_VALUE : FALSE_VALUE);
- }
+ //-----------------------------------------------------------------------------------------------
+ // Query Config Specified Section
+ //-----------------------------------------------------------------------------------------------
- public boolean getBool(String key) {
- String strVal = get(key);
- return strVal != null ? strVal.equals(TRUE_VALUE) : false;
+ public String getCurrentDatabase() {
+ return get(SessionVars.CURRENT_DATABASE);
}
public void setUser(String username) {
- put(USER_NAME, username);
+ put(SessionVars.USERNAME, username);
}
public String getUser() {
- return get(USER_NAME);
+ return get(SessionVars.USERNAME);
}
public void setStagingDir(Path path) {
- put(STAGING_DIR, path.toUri().toString());
+ put(QueryVars.STAGING_DIR, path.toUri().toString());
}
public Path getStagingDir() {
- String strVal = get(STAGING_DIR);
+ String strVal = get(QueryVars.STAGING_DIR);
return strVal != null ? new Path(strVal) : null;
}
/**
- * The fact that QueryContext has an output table means this query has a target table.
- * In other words, this query is 'CREATE TABLE' or 'INSERT (OVERWRITE) INTO <table name>' statement.
- * This config is not set if a query has INSERT (OVERWRITE) INTO LOCATION '/path/..'.
- */
- public boolean hasOutputTable() {
- return get(OUTPUT_TABLE_NAME) != null;
- }
-
- /**
* Set a target table name
*
* @param tableName The target table name
*/
public void setOutputTable(String tableName) {
- put(OUTPUT_TABLE_NAME, tableName);
- }
-
- public String getOutputTable() {
- String strVal = get(OUTPUT_TABLE_NAME);
- return strVal != null ? strVal : null;
+ put(QueryVars.OUTPUT_TABLE_NAME, tableName);
}
/**
@@ -116,52 +112,64 @@ public class QueryContext extends KeyValueSet {
* @return
*/
public boolean hasOutputPath() {
- return get(OUTPUT_TABLE_PATH) != null;
+ return containsKey(QueryVars.OUTPUT_TABLE_PATH);
}
public void setOutputPath(Path path) {
- put(OUTPUT_TABLE_PATH, path.toUri().toString());
+ put(QueryVars.OUTPUT_TABLE_PATH, path.toUri().toString());
}
public Path getOutputPath() {
- String strVal = get(OUTPUT_TABLE_PATH);
+ String strVal = get(QueryVars.OUTPUT_TABLE_PATH);
return strVal != null ? new Path(strVal) : null;
}
public boolean hasPartition() {
- return get(OUTPUT_PARTITIONS) != null;
+ return containsKey(QueryVars.OUTPUT_PARTITIONS);
}
public void setPartitionMethod(PartitionMethodDesc partitionMethodDesc) {
- put(OUTPUT_PARTITIONS, partitionMethodDesc != null ? partitionMethodDesc.toJson() : null);
+ put(QueryVars.OUTPUT_PARTITIONS, partitionMethodDesc != null ? partitionMethodDesc.toJson() : null);
}
public PartitionMethodDesc getPartitionMethod() {
- return PartitionMethodDesc.fromJson(get(OUTPUT_PARTITIONS));
+ return PartitionMethodDesc.fromJson(get(QueryVars.OUTPUT_PARTITIONS));
}
public void setOutputOverwrite() {
- setBool(OUTPUT_OVERWRITE, true);
+ setBool(QueryVars.OUTPUT_OVERWRITE, true);
}
public boolean isOutputOverwrite() {
- return getBool(OUTPUT_OVERWRITE);
+ return getBool(QueryVars.OUTPUT_OVERWRITE);
}
public void setFileOutput() {
- setBool(OUTPUT_AS_DIRECTORY, true);
+ setBool(QueryVars.OUTPUT_AS_DIRECTORY, true);
+ }
+
+ public boolean containsKey(ConfigKey key) {
+ return containsKey(key.keyname());
+ }
+
+ public boolean equalKey(ConfigKey key, String another) {
+ if (containsKey(key)) {
+ return get(key).equals(another);
+ } else {
+ return false;
+ }
}
- public boolean isFileOutput() {
- return getBool(OUTPUT_AS_DIRECTORY);
+ public boolean isCommandType(NodeType commandType) {
+ return equalKey(QueryVars.COMMAND_TYPE, commandType.name());
}
public void setCommandType(NodeType nodeType) {
- put(COMMAND_TYPE, nodeType.name());
+ put(QueryVars.COMMAND_TYPE, nodeType.name());
}
public NodeType getCommandType() {
- String strVal = get(COMMAND_TYPE);
+ String strVal = get(QueryVars.COMMAND_TYPE);
return strVal != null ? NodeType.valueOf(strVal) : null;
}
@@ -170,7 +178,7 @@ public class QueryContext extends KeyValueSet {
}
public boolean isCreateTable() {
- return getCommandType() == NodeType.CREATE_TABLE;
+ return isCommandType(NodeType.CREATE_TABLE);
}
public void setInsert() {
@@ -178,14 +186,6 @@ public class QueryContext extends KeyValueSet {
}
public boolean isInsert() {
- return getCommandType() == NodeType.INSERT;
- }
-
- public void setHiveQueryMode() {
- setBool("hive.query.mode", true);
- }
-
- public boolean isHiveQueryMode() {
- return getBool("hive.query.mode");
+ return isCommandType(NodeType.INSERT);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java
index f1af2ff..56df48d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.query;
import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.ipc.TajoWorkerProtocol;
@@ -193,7 +194,7 @@ public class QueryUnitRequestImpl implements QueryUnitRequest {
if (!p.hasQueryContext()) {
return null;
}
- this.queryContext = new QueryContext(p.getQueryContext());
+ this.queryContext = new QueryContext(new TajoConf(), p.getQueryContext());
return this.queryContext;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 73f3cf5..37a56ba 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
import org.apache.tajo.QueryId;
import org.apache.tajo.QueryIdFactory;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.algebra.AlterTablespaceSetType;
import org.apache.tajo.algebra.Expr;
@@ -116,40 +117,25 @@ public class GlobalEngine extends AbstractService {
public SubmitQueryResponse executeQuery(Session session, String query, boolean isJson) {
LOG.info("Query: " + query);
- QueryContext queryContext = new QueryContext();
- queryContext.putAll(session.getAllVariables());
+ QueryContext queryContext = new QueryContext(context.getConf(), session);
Expr planningContext;
try {
if (isJson) {
planningContext = buildExpressionFromJson(query);
} else {
- // setting environment variables
- String [] cmds = query.split(" ");
- if(cmds != null) {
- if(cmds[0].equalsIgnoreCase("set")) {
- String[] params = cmds[1].split("=");
- context.getConf().set(params[0], params[1]);
- SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder();
- responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME));
- responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
- return responseBuilder.build();
- }
- }
-
planningContext = buildExpressionFromSql(queryContext, query);
}
String jsonExpr = planningContext.toJson();
- LogicalPlan plan = createLogicalPlan(session, planningContext);
+ LogicalPlan plan = createLogicalPlan(queryContext, planningContext);
SubmitQueryResponse response = executeQueryInternal(queryContext, session, plan, query, jsonExpr);
return response;
} catch (Throwable t) {
context.getSystemMetrics().counter("Query", "errorQuery").inc();
LOG.error("\nStack Trace:\n" + StringUtils.stringifyException(t));
SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder();
- responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME));
+ responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME));
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
responseBuilder.setIsForwarded(true);
responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR);
@@ -183,11 +169,11 @@ public class GlobalEngine extends AbstractService {
SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder();
responseBuilder.setIsForwarded(false);
- responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME));
+ responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME));
if (PlannerUtil.checkIfDDLPlan(rootNode)) {
context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
- updateQuery(session, rootNode.getChild());
+ updateQuery(queryContext, rootNode.getChild());
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
@@ -310,7 +296,7 @@ public class GlobalEngine extends AbstractService {
}
TaskAttemptContext taskAttemptContext =
- new TaskAttemptContext(context.getConf(), queryContext, null, (CatalogProtos.FragmentProto[]) null, stagingDir);
+ new TaskAttemptContext(queryContext, null, (CatalogProtos.FragmentProto[]) null, stagingDir);
taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000"));
EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild());
@@ -390,7 +376,7 @@ public class GlobalEngine extends AbstractService {
}
- public QueryId updateQuery(Session session, String sql, boolean isJson) throws IOException, SQLException, PlanningException {
+ public QueryId updateQuery(QueryContext queryContext, String sql, boolean isJson) throws IOException, SQLException, PlanningException {
try {
LOG.info("SQL: " + sql);
@@ -402,13 +388,13 @@ public class GlobalEngine extends AbstractService {
expr = analyzer.parse(sql);
}
- LogicalPlan plan = createLogicalPlan(session, expr);
+ LogicalPlan plan = createLogicalPlan(queryContext, expr);
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
if (!PlannerUtil.checkIfDDLPlan(rootNode)) {
throw new SQLException("This is not update query:\n" + sql);
} else {
- updateQuery(session, rootNode.getChild());
+ updateQuery(queryContext, rootNode.getChild());
return QueryIdFactory.NULL_QUERY_ID;
}
} catch (Exception e) {
@@ -417,46 +403,46 @@ public class GlobalEngine extends AbstractService {
}
}
- private boolean updateQuery(Session session, LogicalNode root) throws IOException {
+ private boolean updateQuery(QueryContext queryContext, LogicalNode root) throws IOException {
switch (root.getType()) {
case CREATE_DATABASE:
CreateDatabaseNode createDatabase = (CreateDatabaseNode) root;
- createDatabase(session, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists());
+ createDatabase(queryContext, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists());
return true;
case DROP_DATABASE:
DropDatabaseNode dropDatabaseNode = (DropDatabaseNode) root;
- dropDatabase(session, dropDatabaseNode.getDatabaseName(), dropDatabaseNode.isIfExists());
+ dropDatabase(queryContext, dropDatabaseNode.getDatabaseName(), dropDatabaseNode.isIfExists());
return true;
case CREATE_TABLE:
CreateTableNode createTable = (CreateTableNode) root;
- createTable(session, createTable, createTable.isIfNotExists());
+ createTable(queryContext, createTable, createTable.isIfNotExists());
return true;
case DROP_TABLE:
DropTableNode dropTable = (DropTableNode) root;
- dropTable(session, dropTable.getTableName(), dropTable.isIfExists(), dropTable.isPurge());
+ dropTable(queryContext, dropTable.getTableName(), dropTable.isIfExists(), dropTable.isPurge());
return true;
case ALTER_TABLESPACE:
AlterTablespaceNode alterTablespace = (AlterTablespaceNode) root;
- alterTablespace(session, alterTablespace);
+ alterTablespace(queryContext, alterTablespace);
return true;
case ALTER_TABLE:
AlterTableNode alterTable = (AlterTableNode) root;
- alterTable(session,alterTable);
+ alterTable(queryContext,alterTable);
return true;
case TRUNCATE_TABLE:
TruncateTableNode truncateTable = (TruncateTableNode) root;
- truncateTable(session, truncateTable);
+ truncateTable(queryContext, truncateTable);
return true;
default:
throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson());
}
}
- private LogicalPlan createLogicalPlan(Session session, Expr expression) throws PlanningException {
+ private LogicalPlan createLogicalPlan(QueryContext queryContext, Expr expression) throws PlanningException {
VerificationState state = new VerificationState();
- preVerifier.verify(session, state, expression);
+ preVerifier.verify(queryContext, state, expression);
if (!state.verified()) {
StringBuilder sb = new StringBuilder();
for (String error : state.getErrorMessages()) {
@@ -465,19 +451,19 @@ public class GlobalEngine extends AbstractService {
throw new VerifyException(sb.toString());
}
- LogicalPlan plan = planner.createPlan(session, expression);
+ LogicalPlan plan = planner.createPlan(queryContext, expression);
if (LOG.isDebugEnabled()) {
LOG.debug("=============================================");
LOG.debug("Non Optimized Query: \n" + plan.toString());
LOG.debug("=============================================");
}
LOG.info("Non Optimized Query: \n" + plan.toString());
- optimizer.optimize(session, plan);
+ optimizer.optimize(queryContext, plan);
LOG.info("=============================================");
LOG.info("Optimized Query: \n" + plan.toString());
LOG.info("=============================================");
- annotatedPlanVerifier.verify(session, state, plan);
+ annotatedPlanVerifier.verify(queryContext, state, plan);
if (!state.verified()) {
StringBuilder sb = new StringBuilder();
@@ -493,7 +479,7 @@ public class GlobalEngine extends AbstractService {
/**
* Alter a given table
*/
- public void alterTablespace(final Session session, final AlterTablespaceNode alterTablespace) {
+ public void alterTablespace(final QueryContext queryContext, final AlterTablespaceNode alterTablespace) {
final CatalogService catalog = context.getCatalog();
final String spaceName = alterTablespace.getTablespaceName();
@@ -517,7 +503,7 @@ public class GlobalEngine extends AbstractService {
/**
* Alter a given table
*/
- public void alterTable(final Session session, final AlterTableNode alterTable) throws IOException {
+ public void alterTable(final QueryContext queryContext, final AlterTableNode alterTable) throws IOException {
final CatalogService catalog = context.getCatalog();
final String tableName = alterTable.getTableName();
@@ -529,7 +515,7 @@ public class GlobalEngine extends AbstractService {
databaseName = split[0];
simpleTableName = split[1];
} else {
- databaseName = session.getCurrentDatabase();
+ databaseName = queryContext.getCurrentDatabase();
simpleTableName = tableName;
}
final String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName);
@@ -572,7 +558,8 @@ public class GlobalEngine extends AbstractService {
if (existColumnName(qualifiedName, alterTable.getNewColumnName())) {
throw new ColumnNameAlreadyExistException(alterTable.getNewColumnName());
}
- catalog.alterTable(CatalogUtil.renameColumn(qualifiedName, alterTable.getColumnName(), alterTable.getNewColumnName(), AlterTableType.RENAME_COLUMN));
+ catalog.alterTable(CatalogUtil.renameColumn(qualifiedName, alterTable.getColumnName(),
+ alterTable.getNewColumnName(), AlterTableType.RENAME_COLUMN));
break;
case ADD_COLUMN:
if (existColumnName(qualifiedName, alterTable.getAddNewColumn().getSimpleName())) {
@@ -588,7 +575,8 @@ public class GlobalEngine extends AbstractService {
/**
* Truncate table a given table
*/
- public void truncateTable(final Session session, final TruncateTableNode truncateTableNode) throws IOException {
+ public void truncateTable(final QueryContext queryContext, final TruncateTableNode truncateTableNode)
+ throws IOException {
List<String> tableNames = truncateTableNode.getTableNames();
final CatalogService catalog = context.getCatalog();
@@ -602,7 +590,7 @@ public class GlobalEngine extends AbstractService {
databaseName = split[0];
simpleTableName = split[1];
} else {
- databaseName = session.getCurrentDatabase();
+ databaseName = queryContext.getCurrentDatabase();
simpleTableName = eachTableName;
}
final String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName);
@@ -641,7 +629,7 @@ public class GlobalEngine extends AbstractService {
return tableDesc.getSchema().containsByName(columnName) ? true : false;
}
- private TableDesc createTable(Session session, CreateTableNode createTable, boolean ifNotExists) throws IOException {
+ private TableDesc createTable(QueryContext queryContext, CreateTableNode createTable, boolean ifNotExists) throws IOException {
TableMeta meta;
if (createTable.hasOptions()) {
@@ -659,7 +647,7 @@ public class GlobalEngine extends AbstractService {
databaseName = CatalogUtil.extractQualifier(createTable.getTableName());
tableName = CatalogUtil.extractSimpleName(createTable.getTableName());
} else {
- databaseName = session.getCurrentDatabase();
+ databaseName = queryContext.getCurrentDatabase();
tableName = createTable.getTableName();
}
@@ -668,11 +656,11 @@ public class GlobalEngine extends AbstractService {
createTable.setPath(tablePath);
}
- return createTableOnPath(session, createTable.getTableName(), createTable.getTableSchema(),
+ return createTableOnPath(queryContext, createTable.getTableName(), createTable.getTableSchema(),
meta, createTable.getPath(), createTable.isExternal(), createTable.getPartitionMethod(), ifNotExists);
}
- public TableDesc createTableOnPath(Session session, String tableName, Schema schema, TableMeta meta,
+ public TableDesc createTableOnPath(QueryContext queryContext, String tableName, Schema schema, TableMeta meta,
Path path, boolean isExternal, PartitionMethodDesc partitionDesc,
boolean ifNotExists)
throws IOException {
@@ -683,7 +671,7 @@ public class GlobalEngine extends AbstractService {
databaseName = splitted[0];
simpleTableName = splitted[1];
} else {
- databaseName = session.getCurrentDatabase();
+ databaseName = queryContext.getCurrentDatabase();
simpleTableName = tableName;
}
String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName);
@@ -736,7 +724,7 @@ public class GlobalEngine extends AbstractService {
}
}
- public boolean createDatabase(@Nullable Session session, String databaseName,
+ public boolean createDatabase(@Nullable QueryContext queryContext, String databaseName,
@Nullable String tablespace,
boolean ifNotExists) throws IOException {
@@ -768,7 +756,7 @@ public class GlobalEngine extends AbstractService {
return true;
}
- public boolean dropDatabase(Session session, String databaseName, boolean ifExists) {
+ public boolean dropDatabase(QueryContext queryContext, String databaseName, boolean ifExists) {
boolean exists = catalog.existDatabase(databaseName);
if(!exists) {
@@ -780,7 +768,7 @@ public class GlobalEngine extends AbstractService {
}
}
- if (session.getCurrentDatabase().equals(databaseName)) {
+ if (queryContext.getCurrentDatabase().equals(databaseName)) {
throw new RuntimeException("ERROR: Cannot drop the current open database");
}
@@ -795,7 +783,7 @@ public class GlobalEngine extends AbstractService {
* @param tableName to be dropped
* @param purge Remove all data if purge is true.
*/
- public boolean dropTable(Session session, String tableName, boolean ifExists, boolean purge) {
+ public boolean dropTable(QueryContext queryContext, String tableName, boolean ifExists, boolean purge) {
CatalogService catalog = context.getCatalog();
String databaseName;
@@ -805,7 +793,7 @@ public class GlobalEngine extends AbstractService {
databaseName = splitted[0];
simpleTableName = splitted[1];
} else {
- databaseName = session.getCurrentDatabase();
+ databaseName = queryContext.getCurrentDatabase();
simpleTableName = tableName;
}
String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 97f59ef..7d80a88 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -34,6 +34,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.ClientProtos;
import org.apache.tajo.ipc.ClientProtos.*;
import org.apache.tajo.ipc.TajoMasterClientProtocol;
@@ -193,7 +194,8 @@ public class TajoMasterClientService extends AbstractService {
}
@Override
- public BoolProto existSessionVariable(RpcController controller, SessionedStringProto request) throws ServiceException {
+ public BoolProto existSessionVariable(RpcController controller, SessionedStringProto request)
+ throws ServiceException {
try {
String value = context.getSessionManager().getVariable(request.getSessionId().getId(), request.getValue());
if (value != null) {
@@ -278,9 +280,16 @@ public class TajoMasterClientService extends AbstractService {
try {
Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+ QueryContext queryContext = new QueryContext(conf, session);
+ if (queryContext.getCurrentDatabase() == null) {
+ for (Map.Entry<String,String> e : queryContext.getAllKeyValus().entrySet()) {
+ System.out.println(e.getKey() + "=" + e.getValue());
+ }
+ }
+
UpdateQueryResponse.Builder builder = UpdateQueryResponse.newBuilder();
try {
- context.getGlobalEngine().updateQuery(session, request.getQuery(), request.getIsJson());
+ context.getGlobalEngine().updateQuery(queryContext, request.getQuery(), request.getIsJson());
builder.setResultCode(ResultCode.OK);
return builder.build();
} catch (Exception e) {
@@ -539,7 +548,9 @@ public class TajoMasterClientService extends AbstractService {
public BoolProto createDatabase(RpcController controller, SessionedStringProto request) throws ServiceException {
try {
Session session = context.getSessionManager().getSession(request.getSessionId().getId());
- if (context.getGlobalEngine().createDatabase(session, request.getValue(), null, false)) {
+ QueryContext queryContext = new QueryContext(conf, session);
+
+ if (context.getGlobalEngine().createDatabase(queryContext, request.getValue(), null, false)) {
return BOOL_TRUE;
} else {
return BOOL_FALSE;
@@ -567,8 +578,9 @@ public class TajoMasterClientService extends AbstractService {
public BoolProto dropDatabase(RpcController controller, SessionedStringProto request) throws ServiceException {
try {
Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+ QueryContext queryContext = new QueryContext(conf, session);
- if (context.getGlobalEngine().dropDatabase(session, request.getValue(), false)) {
+ if (context.getGlobalEngine().dropDatabase(queryContext, request.getValue(), false)) {
return BOOL_TRUE;
} else {
return BOOL_FALSE;
@@ -605,6 +617,10 @@ public class TajoMasterClientService extends AbstractService {
tableName = request.getValue();
}
+ if (databaseName == null) {
+ System.out.println("A");
+ }
+
if (catalog.existsTable(databaseName, tableName)) {
return BOOL_TRUE;
} else {
@@ -672,6 +688,7 @@ public class TajoMasterClientService extends AbstractService {
throws ServiceException {
try {
Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+ QueryContext queryContext = new QueryContext(conf, session);
Path path = new Path(request.getPath());
FileSystem fs = path.getFileSystem(conf);
@@ -689,7 +706,7 @@ public class TajoMasterClientService extends AbstractService {
TableDesc desc;
try {
- desc = context.getGlobalEngine().createTableOnPath(session, request.getName(), schema,
+ desc = context.getGlobalEngine().createTableOnPath(queryContext, request.getName(), schema,
meta, path, true, partitionDesc, false);
} catch (Exception e) {
return TableResponse.newBuilder()
@@ -715,7 +732,9 @@ public class TajoMasterClientService extends AbstractService {
public BoolProto dropTable(RpcController controller, DropTableRequest dropTable) throws ServiceException {
try {
Session session = context.getSessionManager().getSession(dropTable.getSessionId().getId());
- context.getGlobalEngine().dropTable(session, dropTable.getName(), false, dropTable.getPurge());
+ QueryContext queryContext = new QueryContext(conf, session);
+
+ context.getGlobalEngine().dropTable(queryContext, dropTable.getName(), false, dropTable.getPurge());
return BOOL_TRUE;
} catch (Throwable t) {
throw new ServiceException(t);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 8bb3dde..8111ef6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.state.*;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoProtos.QueryState;
import org.apache.tajo.catalog.CatalogService;
@@ -38,7 +39,6 @@ import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.ExecutionBlockCursor;
@@ -636,7 +636,7 @@ public class Query implements EventHandler<QueryEvent> {
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
TableMeta meta = lastStage.getTableMeta();
- String nullChar = queryContext.get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal);
+ String nullChar = queryContext.get(SessionVars.NULL_CHAR);
meta.putOption(StorageConstants.CSVFILE_NULL, nullChar);
TableStats stats = lastStage.getResultStats();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
index 25af82f..aed69b2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
@@ -29,10 +29,12 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tajo.QueryId;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.global.GlobalPlanner;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.TajoAsyncDispatcher;
@@ -83,6 +85,8 @@ public class QueryMaster extends CompositeService implements EventHandler {
private QueryMasterContext queryMasterContext;
+ private QueryContext queryContext;
+
private QueryHeartbeatThread queryHeartbeatThread;
private FinishedQueryMasterTaskCleanThread finishedQueryMasterTaskCleanThread;
@@ -362,12 +366,9 @@ public class QueryMaster extends CompositeService implements EventHandler {
try {
queryMasterTask.stop();
- //if (!systemConf.get(CommonTestingUtil.TAJO_TEST, "FALSE").equalsIgnoreCase("TRUE")
- // && !workerContext.isYarnContainerMode()) {
- if (!getContext().getConf().getBoolVar(TajoConf.ConfVars.TAJO_DEBUG)) {
+ if (!queryContext.getBool(SessionVars.DEBUG_ENABLED)) {
cleanup(queryId);
}
- //}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
@@ -408,6 +409,8 @@ public class QueryMaster extends CompositeService implements EventHandler {
queryMasterTask.start();
}
+ queryContext = event.getQueryContext();
+
synchronized(queryMasterTasks) {
queryMasterTasks.put(event.getQueryId(), queryMasterTask);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
index f52d143..ec975d8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
@@ -204,7 +204,6 @@ public class QueryMasterManagerService extends CompositeService
try {
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(
new QueryId(report.getId().getQueryUnitId().getExecutionBlockId().getQueryId()));
- // queryMaster terminated by internal error before task has not done
if (queryMasterTask != null) {
queryMasterTask.getEventHandler().handle(new TaskCompletionEvent(report));
}
@@ -239,7 +238,8 @@ public class QueryMasterManagerService extends CompositeService
LOG.info("Receive executeQuery request:" + queryId);
queryMaster.handle(new QueryStartEvent(queryId,
new Session(request.getSession()),
- new QueryContext(request.getQueryContext()), request.getExprInJson().getValue(),
+ new QueryContext(workerContext.getQueryMaster().getContext().getConf(),
+ request.getQueryContext()), request.getExprInJson().getValue(),
request.getLogicalPlanJson().getValue()));
done.run(TajoWorker.TRUE_PROTO);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
index 071e5d4..5885a1d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
@@ -328,8 +328,8 @@ public class QueryMasterTask extends CompositeService {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(systemConf);
Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class);
- LogicalPlan plan = planner.createPlan(session, expr);
- optimizer.optimize(session, plan);
+ LogicalPlan plan = planner.createPlan(queryContext, expr);
+ optimizer.optimize(queryContext, plan);
GlobalEngine.DistributedQueryHookManager hookManager = new GlobalEngine.DistributedQueryHookManager();
hookManager.addHook(new GlobalEngine.InsertHook());
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index fa1ed4c..940170c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -25,12 +25,12 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.StatisticsUtil;
import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.PlannerUtil;
import org.apache.tajo.engine.planner.PlanningException;
import org.apache.tajo.engine.planner.RangePartitionAlgorithm;
@@ -50,6 +50,7 @@ import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.TupleRange;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.unit.StorageUnit;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.util.TajoIdUtils;
import org.apache.tajo.worker.FetchImpl;
@@ -373,8 +374,7 @@ public class Repartitioner {
// Getting the desire number of join tasks according to the volumn
// of a larger table
int largerIdx = stats[0] >= stats[1] ? 0 : 1;
- int desireJoinTaskVolumn = subQuery.getContext().getConf().
- getIntVar(ConfVars.DIST_QUERY_JOIN_TASK_VOLUME);
+ int desireJoinTaskVolumn = subQuery.getMasterPlan().getContext().getInt(SessionVars.JOIN_TASK_INPUT_SIZE);
// calculate the number of tasks according to the data size
int mb = (int) Math.ceil((double) stats[largerIdx] / 1048576);
@@ -858,17 +858,17 @@ public class Repartitioner {
// Scattered hash shuffle hashes the key columns and groups the hash keys associated with
// the same hash key. Then, if the volume of a group is larger
- // than DIST_QUERY_TABLE_PARTITION_VOLUME, it divides the group into more than two sub groups
- // according to DIST_QUERY_TABLE_PARTITION_VOLUME (default size = 256MB).
+ // than $DIST_QUERY_TABLE_PARTITION_VOLUME, it divides the group into more than two sub groups
+ // according to $DIST_QUERY_TABLE_PARTITION_VOLUME (default size = 256MB).
// As a result, each group size always becomes the less than or equal
- // to DIST_QUERY_TABLE_PARTITION_VOLUME. Finally, each subgroup is assigned to a query unit.
+ // to $DIST_QUERY_TABLE_PARTITION_VOLUME. Finally, each subgroup is assigned to a query unit.
// It is usually used for writing partitioned tables.
public static void scheduleScatteredHashShuffleFetches(TaskSchedulerContext schedulerContext,
SubQuery subQuery, Map<ExecutionBlockId, List<IntermediateEntry>> intermediates,
String tableName) {
int i = 0;
- long splitVolume = ((long) 1048576) * subQuery.getContext().getConf().
- getIntVar(ConfVars.DIST_QUERY_TABLE_PARTITION_VOLUME); // in bytes
+ long splitVolume = StorageUnit.MB *
+ subQuery.getMasterPlan().getContext().getLong(SessionVars.TABLE_PARTITION_PER_SHUFFLE_SIZE);
long sumNumBytes = 0L;
Map<Integer, List<FetchImpl>> fetches = new HashMap<Integer, List<FetchImpl>>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index 17efa21..b6fe9da 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -32,10 +32,7 @@ import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.*;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryIdFactory;
-import org.apache.tajo.QueryUnitId;
-import org.apache.tajo.TajoIdProtos;
+import org.apache.tajo.*;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
@@ -742,7 +739,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
LOG.info(subQuery.getId() + ", Bigger Table's volume is approximately " + mb + " MB");
int taskNum = (int) Math.ceil((double) mb /
- conf.getIntVar(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME));
+ conf.getIntVar(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME));
int totalMem = getClusterTotalMemory(subQuery);
LOG.info(subQuery.getId() + ", Total memory of cluster is " + totalMem + " MB");
@@ -750,8 +747,8 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
// determine the number of task
taskNum = Math.min(taskNum, slots);
- if (conf.getIntVar(ConfVars.TESTCASE_MIN_TASK_NUM) > 0) {
- taskNum = conf.getIntVar(ConfVars.TESTCASE_MIN_TASK_NUM);
+ if (conf.getIntVar(ConfVars.$TEST_MIN_TASK_NUM) > 0) {
+ taskNum = conf.getIntVar(ConfVars.$TEST_MIN_TASK_NUM);
LOG.warn("!!!!! TESTCASE MODE !!!!!");
}
@@ -795,7 +792,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
LOG.info(subQuery.getId() + ", Table's volume is approximately " + mb + " MB");
// determine the number of task
int taskNumBySize = (int) Math.ceil((double) mb /
- conf.getIntVar(ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME));
+ conf.getIntVar(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME));
int totalMem = getClusterTotalMemory(subQuery);
@@ -1110,7 +1107,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
stopScheduler();
releaseContainers();
- if (!getContext().getConf().getBoolVar(TajoConf.ConfVars.TAJO_DEBUG)) {
+ if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) {
List<ExecutionBlock> childs = getMasterPlan().getChilds(getId());
List<TajoIdProtos.ExecutionBlockIdProto> ebIds = Lists.newArrayList();
for (ExecutionBlock executionBlock : childs){
[13/17] TAJO-928: Session variables should override query configs in
TajoConf.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
index 1a8a90e..548d43a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
@@ -35,7 +35,6 @@ import org.apache.tajo.engine.planner.logical.JoinNode;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -61,7 +60,7 @@ public class TestFullOuterHashJoinExec {
private LogicalPlanner planner;
private AbstractStorageManager sm;
private Path testDir;
- private static Session session = LocalTajoTestingUtility.createDummySession();
+ private QueryContext defaultContext;
private TableDesc dep3;
private TableDesc job3;
@@ -234,11 +233,10 @@ public class TestFullOuterHashJoinExec {
phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path);
catalog.createTable(phone3);
-
-
-
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
+
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
@After
@@ -260,7 +258,7 @@ public class TestFullOuterHashJoinExec {
@Test
public final void testFullOuterHashJoinExec0() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
@@ -270,7 +268,7 @@ public class TestFullOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec0");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -297,7 +295,7 @@ public class TestFullOuterHashJoinExec {
@Test
public final void testFullOuterHashJoinExec1() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
@@ -307,7 +305,7 @@ public class TestFullOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuter_HashJoinExec1");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -333,7 +331,7 @@ public class TestFullOuterHashJoinExec {
@Test
public final void testFullOuterHashJoinExec2() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[2]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
@@ -343,7 +341,7 @@ public class TestFullOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec2");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -370,7 +368,7 @@ public class TestFullOuterHashJoinExec {
@Test
public final void testFullOuterHashJoinExec3() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[3]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
@@ -381,7 +379,7 @@ public class TestFullOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec3");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged,
workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
index 50e5906..1b9f7aa 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
@@ -36,7 +36,6 @@ import org.apache.tajo.engine.planner.logical.JoinNode;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -62,7 +61,7 @@ public class TestFullOuterMergeJoinExec {
private LogicalPlanner planner;
private AbstractStorageManager sm;
private Path testDir;
- private static final Session session = LocalTajoTestingUtility.createDummySession();
+ private QueryContext defaultContext;
private TableDesc dep3;
private TableDesc dep4;
@@ -279,6 +278,8 @@ public class TestFullOuterMergeJoinExec {
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
+
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
@After
@@ -304,7 +305,7 @@ public class TestFullOuterMergeJoinExec {
@Test
public final void testFullOuterMergeJoin0() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -316,7 +317,7 @@ public class TestFullOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin0");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -341,7 +342,7 @@ public class TestFullOuterMergeJoinExec {
@Test
public final void testFullOuterMergeJoin1() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -353,7 +354,7 @@ public class TestFullOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin1");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -378,7 +379,7 @@ public class TestFullOuterMergeJoinExec {
@Test
public final void testFullOuterMergeJoin2() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[2]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -390,7 +391,7 @@ public class TestFullOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin2");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -416,7 +417,7 @@ public class TestFullOuterMergeJoinExec {
@Test
public final void testFullOuterMergeJoin3() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[3]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -428,7 +429,7 @@ public class TestFullOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin3");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -456,7 +457,7 @@ public class TestFullOuterMergeJoinExec {
@Test
public final void testFullOuterMergeJoin4() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[4]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -469,7 +470,7 @@ public class TestFullOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin4");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -495,7 +496,7 @@ public class TestFullOuterMergeJoinExec {
@Test
public final void testFullOuterMergeJoin5() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[5]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -508,7 +509,7 @@ public class TestFullOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin5");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index 794ca79..7055e44 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@ -154,11 +154,11 @@ public class TestHashAntiJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashAntiJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
optimizer.optimize(plan);
LogicalNode rootNode = plan.getRootBlock().getRoot();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
index 0e07536..a8828ab 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
@@ -20,6 +20,7 @@ package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.LocalTajoTestingUtility;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -36,7 +37,6 @@ import org.apache.tajo.engine.planner.logical.JoinNode;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -61,7 +61,7 @@ public class TestHashJoinExec {
private LogicalPlanner planner;
private AbstractStorageManager sm;
private Path testDir;
- private final Session session = LocalTajoTestingUtility.createDummySession();
+ private QueryContext defaultContext;
private TableDesc employee;
private TableDesc people;
@@ -126,6 +126,7 @@ public class TestHashJoinExec {
catalog.createTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
@After
@@ -140,8 +141,9 @@ public class TestHashJoinExec {
@Test
public final void testHashInnerJoin() throws IOException, PlanningException {
+
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
@@ -152,7 +154,7 @@ public class TestHashJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -182,7 +184,7 @@ public class TestHashJoinExec {
@Test
public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
@@ -195,13 +197,12 @@ public class TestHashJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
- TajoConf localConf = new TajoConf(conf);
- localConf.setLongVar(TajoConf.ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD, 100l);
- PhysicalPlannerImpl phyPlanner = new PhysicalPlannerImpl(localConf, sm);
+ ctx.getQueryContext().setLong(SessionVars.HASH_JOIN_SIZE_LIMIT.keyname(), 100l);
+ PhysicalPlannerImpl phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
ProjectionExec proj = (ProjectionExec) exec;
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index 835260f..6373f23 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@ -158,11 +158,11 @@ public class TestHashSemiJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashSemiJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
optimizer.optimize(plan);
LogicalNode rootNode = plan.getRootBlock().getRoot();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
index bb25875..ef740e3 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
@@ -35,7 +35,6 @@ import org.apache.tajo.engine.planner.logical.JoinNode;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -62,7 +61,7 @@ public class TestLeftOuterHashJoinExec {
private LogicalPlanner planner;
private AbstractStorageManager sm;
private Path testDir;
- private Session session = LocalTajoTestingUtility.createDummySession();
+ private QueryContext defaultContext;
private TableDesc dep3;
private TableDesc job3;
@@ -239,6 +238,7 @@ public class TestLeftOuterHashJoinExec {
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
@After
@@ -262,7 +262,7 @@ public class TestLeftOuterHashJoinExec {
@Test
public final void testLeftOuterHashJoinExec0() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
@@ -272,7 +272,7 @@ public class TestLeftOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterHashJoinExec0");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -303,11 +303,11 @@ public class TestLeftOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec1");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
@@ -344,11 +344,11 @@ public class TestLeftOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec2");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[2]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
@@ -386,11 +386,11 @@ public class TestLeftOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec3");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[3]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
@@ -428,11 +428,11 @@ public class TestLeftOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec4");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[4]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
@@ -444,12 +444,10 @@ public class TestLeftOuterHashJoinExec {
assertEquals(1, 0);
}
else{
- Tuple tuple;
int count = 0;
- int i = 1;
exec.init();
- while ((tuple = exec.next()) != null) {
+ while (exec.next() != null) {
//TODO check contents
count = count + 1;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
index e935f57..2a8d9f0 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
@@ -36,7 +36,6 @@ import org.apache.tajo.engine.planner.PlanningException;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -60,7 +59,7 @@ public class TestLeftOuterNLJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private static final Session session = LocalTajoTestingUtility.createDummySession();
+ private QueryContext defaultContext;
private AbstractStorageManager sm;
private Path testDir;
@@ -236,6 +235,8 @@ public class TestLeftOuterNLJoinExec {
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
+
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
@After
@@ -261,11 +262,11 @@ public class TestLeftOuterNLJoinExec {
FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterNLJoinExec0");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
@@ -303,11 +304,11 @@ public class TestLeftOuterNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec1");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
@@ -347,11 +348,11 @@ public class TestLeftOuterNLJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec2");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[2]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
@@ -392,11 +393,11 @@ public class TestLeftOuterNLJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec3");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[3]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
@@ -436,11 +437,11 @@ public class TestLeftOuterNLJoinExec {
FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec4");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[4]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
index a4449fa..ed19e4f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
@@ -154,7 +154,7 @@ public class TestMergeJoinExec {
@Test
public final void testMergeInnerJoin() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
LogicalNode root = plan.getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(root, NodeType.JOIN);
@@ -166,7 +166,7 @@ public class TestMergeJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testMergeInnerJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
index 019929d..1120bbe 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
@@ -152,12 +152,12 @@ public class TestNLJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLCrossJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
- Expr context = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),
- context).getRootBlock().getRoot();
+ Expr expr = analyzer.parse(QUERIES[0]);
+ LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf),
+ expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
@@ -181,13 +181,12 @@ public class TestNLJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLInnerJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), merged, workDir);
ctx.setEnforcer(new Enforcer());
- Expr context = analyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),
- context).getRootBlock().getRoot();
- //LogicalOptimizer.optimize(ctx, plan);
+ Expr expr = analyzer.parse(QUERIES[1]);
+ LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf),
+ expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index a823d2b..a184a9a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -85,6 +85,7 @@ public class TestPhysicalPlanner {
private static AbstractStorageManager sm;
private static Path testDir;
private static Session session = LocalTajoTestingUtility.createDummySession();
+ private static QueryContext defaultContext;
private static TableDesc employee = null;
private static TableDesc score = null;
@@ -168,6 +169,7 @@ public class TestPhysicalPlanner {
planner = new LogicalPlanner(catalog);
optimizer = new LogicalOptimizer(conf);
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null);
}
@@ -201,12 +203,12 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
employee.getPath(), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanPlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
LogicalNode rootNode =plan.getRootBlock().getRoot();
optimizer.optimize(plan);
@@ -232,12 +234,12 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
employee.getPath(), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanWithFilterPlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[16]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
LogicalNode rootNode =plan.getRootBlock().getRoot();
optimizer.optimize(plan);
@@ -261,12 +263,12 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByPlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[7]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
optimizer.optimize(plan);
LogicalNode rootNode = plan.getRootBlock().getRoot();
@@ -293,12 +295,12 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir(
"target/test-data/testHashGroupByPlanWithALLField");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[15]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
@@ -322,12 +324,12 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortGroupByPlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[]{frags[0]}, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[7]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
@@ -382,14 +384,14 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
ctx.setOutputPath(new Path(workDir, "grouped1"));
Expr context = analyzer.parse(CreateTableAsStmts[0]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
@@ -424,14 +426,14 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithRCFile");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
ctx.setOutputPath(new Path(workDir, "grouped2"));
Expr context = analyzer.parse(CreateTableAsStmts[1]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.RCFILE);
@@ -465,14 +467,14 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
ctx.setOutputPath(new Path(workDir, "grouped3"));
Expr context = analyzer.parse(CreateTableAsStmts[2]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -483,7 +485,7 @@ public class TestPhysicalPlanner {
public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOException, PlanningException {
Expr context = analyzer.parse(CreateTableAsStmts[2]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalRootNode rootNode = (LogicalRootNode) optimizer.optimize(plan);
CreateTableNode createTableNode = rootNode.getChild();
Enforcer enforcer = new Enforcer();
@@ -492,7 +494,7 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(enforcer);
@@ -507,7 +509,7 @@ public class TestPhysicalPlanner {
public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOException, PlanningException {
Expr context = analyzer.parse(CreateTableAsStmts[2]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalRootNode rootNode = (LogicalRootNode) optimizer.optimize(plan);
CreateTableNode createTableNode = rootNode.getChild();
Enforcer enforcer = new Enforcer();
@@ -516,7 +518,7 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(enforcer);
@@ -533,11 +535,11 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testPartitionedStorePlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
id, new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[7]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
int numPartitions = 3;
Column key1 = new Column("default.score.deptname", Type.TEXT);
@@ -594,11 +596,11 @@ public class TestPhysicalPlanner {
Path workDir = CommonTestingUtil.getTestDir(
"target/test-data/testPartitionedStorePlanWithEmptyGroupingSet");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
id, new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[14]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
LogicalNode rootNode = plan.getRootBlock().getRoot();
int numPartitions = 1;
DataChannel dataChannel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
@@ -648,12 +650,12 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testAggregationFunction");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[8]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
// Set all aggregation functions to the first phase mode
@@ -679,12 +681,12 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCountFunction");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[9]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
// Set all aggregation functions to the first phase mode
@@ -707,12 +709,12 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByWithNullValue");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[11]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
@@ -732,12 +734,12 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testUnionPlan");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[0]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
LogicalRootNode root = (LogicalRootNode) rootNode;
UnionNode union = plan.createNode(UnionNode.class);
@@ -760,11 +762,11 @@ public class TestPhysicalPlanner {
@Test
public final void testEvalExpr() throws IOException, PlanningException {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEvalExpr");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] { }, workDir);
Expr expr = analyzer.parse(QUERIES[12]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
@@ -777,7 +779,7 @@ public class TestPhysicalPlanner {
assertTrue(7.0d == tuple.get(1).asFloat8());
expr = analyzer.parse(QUERIES[13]);
- plan = planner.createPlan(session, expr);
+ plan = planner.createPlan(defaultContext, expr);
rootNode = optimizer.optimize(plan);
phyPlanner = new PhysicalPlannerImpl(conf, sm);
@@ -797,11 +799,11 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
Expr context = analyzer.parse(createIndexStmt[0]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
@@ -825,12 +827,12 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testDuplicateEliminate");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(duplicateElimination[0]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
@@ -859,12 +861,12 @@ public class TestPhysicalPlanner {
employee.getPath(), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testIndexedStoreExec");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(SORT_QUERY[0]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT);
@@ -953,7 +955,7 @@ public class TestPhysicalPlanner {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortEnforcer");
Expr context = analyzer.parse(SORT_QUERY[0]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
optimizer.optimize(plan);
LogicalNode rootNode = plan.getRootBlock().getRoot();
@@ -961,7 +963,7 @@ public class TestPhysicalPlanner {
Enforcer enforcer = new Enforcer();
enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.IN_MEMORY_SORT);
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(enforcer);
@@ -975,7 +977,7 @@ public class TestPhysicalPlanner {
assertTrue(exec instanceof MemSortExec);
context = analyzer.parse(SORT_QUERY[0]);
- plan = planner.createPlan(session, context);
+ plan = planner.createPlan(defaultContext, context);
optimizer.optimize(plan);
rootNode = plan.getRootBlock().getRoot();
@@ -983,7 +985,7 @@ public class TestPhysicalPlanner {
enforcer = new Enforcer();
enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.MERGE_SORT);
- ctx = new TaskAttemptContext(conf, new QueryContext(),
+ ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(enforcer);
@@ -1003,7 +1005,7 @@ public class TestPhysicalPlanner {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByEnforcer");
Expr context = analyzer.parse(QUERIES[7]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
optimizer.optimize(plan);
LogicalNode rootNode = plan.getRootBlock().getRoot();
@@ -1011,7 +1013,7 @@ public class TestPhysicalPlanner {
Enforcer enforcer = new Enforcer();
enforcer.enforceHashAggregation(groupByNode.getPID());
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(enforcer);
@@ -1025,7 +1027,7 @@ public class TestPhysicalPlanner {
assertNotNull(PhysicalPlanUtil.findExecutor(exec, HashAggregateExec.class));
context = analyzer.parse(QUERIES[7]);
- plan = planner.createPlan(session, context);
+ plan = planner.createPlan(defaultContext, context);
optimizer.optimize(plan);
rootNode = plan.getRootBlock().getRoot();
@@ -1033,7 +1035,7 @@ public class TestPhysicalPlanner {
enforcer = new Enforcer();
enforcer.enforceSortAggregation(groupByNode.getPID(), null);
- ctx = new TaskAttemptContext(conf, new QueryContext(),
+ ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index ed6cb4e..f649dac 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -139,11 +139,11 @@ public class TestProgressExternalSortExec {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
LogicalNode rootNode = plan.getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
index de90f70..506555e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
@@ -36,7 +36,6 @@ import org.apache.tajo.engine.planner.PlanningException;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -62,7 +61,7 @@ public class TestRightOuterHashJoinExec {
private LogicalPlanner planner;
private AbstractStorageManager sm;
private Path testDir;
- private static Session session = LocalTajoTestingUtility.createDummySession();
+ private QueryContext defaultContext;
private TableDesc dep3;
private TableDesc job3;
@@ -214,6 +213,7 @@ public class TestRightOuterHashJoinExec {
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
@After
@@ -237,11 +237,11 @@ public class TestRightOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec0");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
@@ -278,11 +278,11 @@ public class TestRightOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec1");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
@@ -319,11 +319,11 @@ public class TestRightOuterHashJoinExec {
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec2");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[2]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, plan);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
index d971073..0ce74eb 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
@@ -35,7 +35,6 @@ import org.apache.tajo.engine.planner.logical.JoinNode;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -61,7 +60,8 @@ public class TestRightOuterMergeJoinExec {
private LogicalPlanner planner;
private AbstractStorageManager sm;
private Path testDir;
- private static final Session session = LocalTajoTestingUtility.createDummySession();
+ private QueryContext defaultContext;
+
private TableDesc dep3;
private TableDesc dep4;
@@ -276,10 +276,10 @@ public class TestRightOuterMergeJoinExec {
phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path);
catalog.createTable(phone3);
-
-
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
+
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
@After
@@ -305,7 +305,7 @@ public class TestRightOuterMergeJoinExec {
@Test
public final void testRightOuterMergeJoin0() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -315,7 +315,7 @@ public class TestRightOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin0");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -340,7 +340,7 @@ public class TestRightOuterMergeJoinExec {
@Test
public final void testRightOuter_MergeJoin1() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -352,7 +352,7 @@ public class TestRightOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin1");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -376,7 +376,7 @@ public class TestRightOuterMergeJoinExec {
@Test
public final void testRightOuterMergeJoin2() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[2]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -388,7 +388,7 @@ public class TestRightOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin2");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -412,7 +412,7 @@ public class TestRightOuterMergeJoinExec {
@Test
public final void testRightOuter_MergeJoin3() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[3]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -424,7 +424,7 @@ public class TestRightOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin3");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -449,7 +449,7 @@ public class TestRightOuterMergeJoinExec {
@Test
public final void testRightOuter_MergeJoin4() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[4]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -462,7 +462,7 @@ public class TestRightOuterMergeJoinExec {
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin4");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -486,7 +486,7 @@ public class TestRightOuterMergeJoinExec {
@Test
public final void testRightOuterMergeJoin5() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[5]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
@@ -499,7 +499,7 @@ public class TestRightOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin5");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index da6fb34..4d4cc3d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -110,12 +110,12 @@ public class TestSortExec {
public final void testNext() throws IOException, PlanningException {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestSortExec");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility
.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[0]);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), context);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
index fe9f990..2b62c0e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
@@ -291,8 +291,8 @@ public class TestGroupByQuery extends QueryTestCaseBase {
// case9
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("id", Type.TEXT);
@@ -345,8 +345,8 @@ public class TestGroupByQuery extends QueryTestCaseBase {
public final void testDistinctAggregationCaseByCase3() throws Exception {
// first distinct is smaller than second distinct.
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("col1", Type.TEXT);
@@ -375,8 +375,8 @@ public class TestGroupByQuery extends QueryTestCaseBase {
public final void testDistinctAggregationCaseByCase4() throws Exception {
// Reproduction case for TAJO-994
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("col1", Type.TEXT);
@@ -562,8 +562,8 @@ public class TestGroupByQuery extends QueryTestCaseBase {
@Test
public final void testNumShufflePartition() throws Exception {
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("col1", Type.TEXT);
@@ -592,7 +592,7 @@ public class TestGroupByQuery extends QueryTestCaseBase {
TajoTestingCluster.createTable("testnumshufflepartition", schema, tableOptions, data.toArray(new String[]{}), 3);
try {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME.varname, "2");
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.varname, "2");
ResultSet res = executeString(
"select col1 \n" +
",count(distinct col2) as cnt1\n" +
@@ -646,8 +646,8 @@ public class TestGroupByQuery extends QueryTestCaseBase {
assertEquals(2, partitionIds.size());
executeString("DROP TABLE testnumshufflepartition PURGE").close();
} finally {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME.varname,
- ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.varname,
+ ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.defaultVal);
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
index 349ad0c..2625136 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
@@ -48,9 +48,9 @@ import static org.junit.Assert.*;
public class TestJoinBroadcast extends QueryTestCaseBase {
public TestJoinBroadcast() throws Exception {
super(TajoConstants.DEFAULT_DATABASE_NAME);
- testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "true");
+ testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "true");
testingCluster.setAllTajoDaemonConfValue(
- TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "" + (5 * 1024));
+ TajoConf.ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "" + (5 * 1024));
executeDDL("create_lineitem_large_ddl.sql", "lineitem_large");
executeDDL("create_customer_large_ddl.sql", "customer_large");
@@ -575,8 +575,8 @@ public class TestJoinBroadcast extends QueryTestCaseBase {
@Test
public final void testLeftOuterJoinLeftSideSmallTable() throws Exception {
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
[03/17] git commit: TAJO-987: Hash shuffle should be balanced
according to intermediate volumes.
Posted by ji...@apache.org.
TAJO-987: Hash shuffle should be balanced according to intermediate volumes.
Closes #101
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/eeaf379a
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/eeaf379a
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/eeaf379a
Branch: refs/heads/index_support
Commit: eeaf379a48030dd819a6daf2040c779379543ac8
Parents: 072b5a3
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 4 12:08:51 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 4 12:08:51 2014 +0900
----------------------------------------------------------------------
CHANGES | 5 +-
.../tajo/master/querymaster/Repartitioner.java | 111 +++++++++++++++++--
.../apache/tajo/master/TestRepartitioner.java | 53 +++++++++
3 files changed, 160 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/eeaf379a/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 705e91e..d6ac279 100644
--- a/CHANGES
+++ b/CHANGES
@@ -29,7 +29,10 @@ Release 0.9.0 - unreleased
IMPROVEMENT
- TAJO-955: Add database selection submit button in catalogview.jsp for text
+ TAJO-987: Hash shuffle should be balanced according to intermediate
+ volumes. (hyunsik)
+
+ TAJO-955: Add database selection submit button in catalogview.jsp for text
based browse. (Hyoungjun Kim via jihoon)
TAJO-956: CONCAT should be support multiple params and null param.
http://git-wip-us.apache.org/repos/asf/tajo/blob/eeaf379a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index f86106f..1fa3f11 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -18,6 +18,7 @@
package org.apache.tajo.master.querymaster;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.primitives.Ints;
import org.apache.commons.logging.Log;
@@ -48,6 +49,7 @@ import org.apache.tajo.storage.AbstractStorageManager;
import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.TupleRange;
import org.apache.tajo.storage.fragment.FileFragment;
+import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.util.TajoIdUtils;
import org.apache.tajo.worker.FetchImpl;
@@ -59,9 +61,7 @@ import java.net.URI;
import java.util.*;
import java.util.Map.Entry;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.HASH_SHUFFLE;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.RANGE_SHUFFLE;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.SCATTERED_HASH_SHUFFLE;
+import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.*;
/**
* Repartitioner creates non-leaf tasks and shuffles intermediate data.
@@ -675,6 +675,31 @@ public class Repartitioner {
}
}
+ @VisibleForTesting
+ public static class FetchGroupMeta {
+ long totalVolume;
+ List<FetchImpl> fetchUrls;
+
+ public FetchGroupMeta(long volume, FetchImpl fetchUrls) {
+ this.totalVolume = volume;
+ this.fetchUrls = Lists.newArrayList(fetchUrls);
+ }
+
+ public FetchGroupMeta addFetche(FetchImpl fetches) {
+ this.fetchUrls.add(fetches);
+ return this;
+ }
+
+ public void increaseVolume(long volume) {
+ this.totalVolume += volume;
+ }
+
+ public long getVolume() {
+ return totalVolume;
+ }
+
+ }
+
public static void scheduleHashShuffledFetches(TaskSchedulerContext schedulerContext, MasterPlan masterPlan,
SubQuery subQuery, DataChannel channel,
int maxNum) {
@@ -689,7 +714,7 @@ public class Repartitioner {
SubQuery.scheduleFragments(subQuery, fragments);
Map<QueryUnit.PullHost, List<IntermediateEntry>> hashedByHost;
- Map<Integer, Collection<FetchImpl>> finalFetches = new HashMap<Integer, Collection<FetchImpl>>();
+ Map<Integer, FetchGroupMeta> finalFetches = new HashMap<Integer, FetchGroupMeta>();
Map<ExecutionBlockId, List<IntermediateEntry>> intermediates = new HashMap<ExecutionBlockId,
List<IntermediateEntry>>();
@@ -717,10 +742,15 @@ public class Repartitioner {
FetchImpl fetch = new FetchImpl(e.getKey(), channel.getShuffleType(),
block.getId(), interm.getKey(), e.getValue());
+ long volumeSum = 0;
+ for (IntermediateEntry ie : e.getValue()) {
+ volumeSum += ie.getVolume();
+ }
+
if (finalFetches.containsKey(interm.getKey())) {
- finalFetches.get(interm.getKey()).add(fetch);
+ finalFetches.get(interm.getKey()).addFetche(fetch).increaseVolume(volumeSum);
} else {
- finalFetches.put(interm.getKey(), TUtil.newList(fetch));
+ finalFetches.put(interm.getKey(), new FetchGroupMeta(volumeSum, fetch));
}
}
}
@@ -756,12 +786,77 @@ public class Repartitioner {
scan.getTableName());
} else {
schedulerContext.setEstimatedTaskNum(determinedTaskNum);
- // divide fetch uris into the the proper number of tasks in a round robin manner.
- scheduleFetchesByRoundRobin(subQuery, finalFetches, scan.getTableName(), determinedTaskNum);
+ // divide fetch uris into the the proper number of tasks according to volumes
+ scheduleFetchesByEvenDistributedVolumes(subQuery, finalFetches, scan.getTableName(), determinedTaskNum);
LOG.info(subQuery.getId() + ", DeterminedTaskNum : " + determinedTaskNum);
}
}
+ public static Pair<Long [], Map<String, List<FetchImpl>>[]> makeEvenDistributedFetchImpl(
+ Map<Integer, FetchGroupMeta> partitions, String tableName, int num) {
+
+ // Sort fetchGroupMeta in a descending order of data volumes.
+ List<FetchGroupMeta> fetchGroupMetaList = Lists.newArrayList(partitions.values());
+ Collections.sort(fetchGroupMetaList, new Comparator<FetchGroupMeta>() {
+ @Override
+ public int compare(FetchGroupMeta o1, FetchGroupMeta o2) {
+ return o1.getVolume() < o2.getVolume() ? 1 : (o1.getVolume() > o2.getVolume() ? -1 : 0);
+ }
+ });
+
+ // Initialize containers
+ Map<String, List<FetchImpl>>[] fetchesArray = new Map[num];
+ Long [] assignedVolumes = new Long[num];
+ // initialization
+ for (int i = 0; i < num; i++) {
+ fetchesArray[i] = new HashMap<String, List<FetchImpl>>();
+ assignedVolumes[i] = 0l;
+ }
+
+ // This algorithm assignes bigger first manner by using a sorted iterator. It is a kind of greedy manner.
+ // Its complexity is O(n). Since FetchGroup can be more than tens of thousands, we should consider its complexity.
+ // In terms of this point, it will show reasonable performance and results. even though it is not an optimal
+ // algorithm.
+ Iterator<FetchGroupMeta> iterator = fetchGroupMetaList.iterator();
+
+ int p = 0;
+ while(iterator.hasNext()) {
+ while (p < num && iterator.hasNext()) {
+ FetchGroupMeta fetchGroupMeta = iterator.next();
+ assignedVolumes[p] += fetchGroupMeta.getVolume();
+
+ TUtil.putCollectionToNestedList(fetchesArray[p], tableName, fetchGroupMeta.fetchUrls);
+ p++;
+ }
+
+ p = num - 1;
+ while (p > 0 && iterator.hasNext()) {
+ FetchGroupMeta fetchGroupMeta = iterator.next();
+ assignedVolumes[p] += fetchGroupMeta.getVolume();
+
+ // While the current one is smaller than next one, it adds additional fetches to current one.
+ while(iterator.hasNext() && assignedVolumes[p - 1] > assignedVolumes[p]) {
+ FetchGroupMeta additionalFetchGroup = iterator.next();
+ assignedVolumes[p] += additionalFetchGroup.getVolume();
+ TUtil.putCollectionToNestedList(fetchesArray[p], tableName, additionalFetchGroup.fetchUrls);
+ }
+
+ p--;
+ }
+ }
+
+ return new Pair<Long[], Map<String, List<FetchImpl>>[]>(assignedVolumes, fetchesArray);
+ }
+
+ public static void scheduleFetchesByEvenDistributedVolumes(SubQuery subQuery, Map<Integer, FetchGroupMeta> partitions,
+ String tableName, int num) {
+ Map<String, List<FetchImpl>>[] fetchsArray = makeEvenDistributedFetchImpl(partitions, tableName, num).getSecond();
+ // Schedule FetchImpls
+ for (Map<String, List<FetchImpl>> eachFetches : fetchsArray) {
+ SubQuery.scheduleFetches(subQuery, eachFetches);
+ }
+ }
+
// Scattered hash shuffle hashes the key columns and groups the hash keys associated with
// the same hash key. Then, if the volume of a group is larger
// than DIST_QUERY_TABLE_PARTITION_VOLUME, it divides the group into more than two sub groups
http://git-wip-us.apache.org/repos/asf/tajo/blob/eeaf379a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
index 0ccaebe..009c02e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
@@ -18,11 +18,14 @@
package org.apache.tajo.master;
+import com.google.common.collect.Maps;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
import org.apache.tajo.TestTajoIds;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.querymaster.QueryUnit;
+import org.apache.tajo.master.querymaster.Repartitioner;
+import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.FetchImpl;
import org.jboss.netty.handler.codec.http.QueryStringDecoder;
@@ -35,6 +38,8 @@ import java.util.List;
import java.util.Map;
import static junit.framework.Assert.assertEquals;
+import static org.apache.tajo.master.querymaster.Repartitioner.FetchGroupMeta;
+import static org.junit.Assert.assertTrue;
public class TestRepartitioner {
@Test
@@ -83,4 +88,52 @@ public class TestRepartitioner {
}
return ret;
}
+
+ @Test
+ public void testScheduleFetchesByEvenDistributedVolumes() {
+ Map<Integer, FetchGroupMeta> fetchGroups = Maps.newHashMap();
+ String tableName = "test1";
+
+
+ fetchGroups.put(0, new FetchGroupMeta(100, new FetchImpl()));
+ fetchGroups.put(1, new FetchGroupMeta(80, new FetchImpl()));
+ fetchGroups.put(2, new FetchGroupMeta(70, new FetchImpl()));
+ fetchGroups.put(3, new FetchGroupMeta(30, new FetchImpl()));
+ fetchGroups.put(4, new FetchGroupMeta(10, new FetchImpl()));
+ fetchGroups.put(5, new FetchGroupMeta(5, new FetchImpl()));
+
+ Pair<Long [], Map<String, List<FetchImpl>>[]> results;
+
+ results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 1);
+ long expected [] = {100 + 80 + 70 + 30 + 10 + 5};
+ assertFetchVolumes(expected, results.getFirst());
+
+ results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 2);
+ long expected0 [] = {130, 165};
+ assertFetchVolumes(expected0, results.getFirst());
+
+ results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 3);
+ long expected1 [] = {100, 95, 100};
+ assertFetchVolumes(expected1, results.getFirst());
+
+ results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 4);
+ long expected2 [] = {100, 80, 70, 45};
+ assertFetchVolumes(expected2, results.getFirst());
+
+ results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 5);
+ long expected3 [] = {100, 80, 70, 30, 15};
+ assertFetchVolumes(expected3, results.getFirst());
+
+ results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 6);
+ long expected4 [] = {100, 80, 70, 30, 10, 5};
+ assertFetchVolumes(expected4, results.getFirst());
+ }
+
+ private static void assertFetchVolumes(long [] expected, Long [] results) {
+ assertEquals("the lengths of volumes are mismatch", expected.length, results.length);
+
+ for (int i = 0; i < expected.length; i++) {
+ assertTrue(expected[i] + " is expected, but " + results[i], expected[i] == results[i]);
+ }
+ }
}
[06/17] git commit: TAJO-989: Cleanup of child blocks after parent
execution block is complete. (jinho)
Posted by ji...@apache.org.
TAJO-989: Cleanup of child blocks after parent execution block is complete. (jinho)
Closes #103
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0f3412a7
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0f3412a7
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0f3412a7
Branch: refs/heads/index_support
Commit: 0f3412a74bb3c565df1259b19630bc17e1bc69e0
Parents: ae38468
Author: jinossy <ji...@gmail.com>
Authored: Tue Aug 5 11:58:16 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Tue Aug 5 11:58:16 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../tajo/master/querymaster/QueryMaster.java | 30 +++++++++-
.../tajo/master/querymaster/SubQuery.java | 20 ++++++-
.../tajo/worker/TajoWorkerManagerService.java | 13 +++++
.../main/java/org/apache/tajo/worker/Task.java | 6 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 21 ++++++-
.../src/main/proto/TajoWorkerProtocol.proto | 5 ++
.../apache/tajo/worker/TestDeletionService.java | 61 ++++++++++++++++++++
8 files changed, 151 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/0f3412a7/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 61593a2..be71bf4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -29,6 +29,9 @@ Release 0.9.0 - unreleased
IMPROVEMENT
+ TAJO-989: Cleanup of child blocks after parent execution block is complete
+ (jinho)
+
TAJO-966: Range partition should support split of multiple characters.
(hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/0f3412a7/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
index f173c24..25af82f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
@@ -18,6 +18,7 @@
package org.apache.tajo.master.querymaster;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -28,6 +29,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tajo.QueryId;
+import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.global.GlobalPlanner;
@@ -162,6 +164,30 @@ public class QueryMaster extends CompositeService implements EventHandler {
}
}
+ protected void cleanupExecutionBlock(List<TajoIdProtos.ExecutionBlockIdProto> executionBlockIds) {
+ LOG.info("cleanup executionBlocks : " + executionBlockIds);
+ NettyClientBase rpc = null;
+ List<TajoMasterProtocol.WorkerResourceProto> workers = getAllWorker();
+ TajoWorkerProtocol.ExecutionBlockListProto.Builder builder = TajoWorkerProtocol.ExecutionBlockListProto.newBuilder();
+ builder.addAllExecutionBlockId(Lists.newArrayList(executionBlockIds));
+ TajoWorkerProtocol.ExecutionBlockListProto executionBlockListProto = builder.build();
+ for (TajoMasterProtocol.WorkerResourceProto worker : workers) {
+ try {
+ if (worker.getPeerRpcPort() == 0) continue;
+
+ rpc = connPool.getConnection(NetUtils.createSocketAddr(worker.getHost(), worker.getPeerRpcPort()),
+ TajoWorkerProtocol.class, true);
+ TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub();
+
+ tajoWorkerProtocolService.cleanupExecutionBlocks(null, executionBlockListProto, NullCallback.get());
+ } catch (Exception e) {
+ LOG.error(e.getMessage());
+ } finally {
+ connPool.releaseConnection(rpc);
+ }
+ }
+ }
+
private void cleanup(QueryId queryId) {
LOG.info("cleanup query resources : " + queryId);
NettyClientBase rpc = null;
@@ -338,7 +364,9 @@ public class QueryMaster extends CompositeService implements EventHandler {
queryMasterTask.stop();
//if (!systemConf.get(CommonTestingUtil.TAJO_TEST, "FALSE").equalsIgnoreCase("TRUE")
// && !workerContext.isYarnContainerMode()) {
- cleanup(queryId); // TODO We will support yarn mode
+ if (!getContext().getConf().getBoolVar(TajoConf.ConfVars.TAJO_DEBUG)) {
+ cleanup(queryId);
+ }
//}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0f3412a7/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index f2e9dd5..17efa21 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.util.Records;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
@@ -1004,7 +1005,8 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
LOG.info("SubQuery (" + subQuery.getId() + ") has " + subQuery.containers.size() + " containers!");
subQuery.eventHandler.handle(
new TaskRunnerGroupEvent(EventType.CONTAINER_REMOTE_LAUNCH,
- subQuery.getId(), allocationEvent.getAllocatedContainer()));
+ subQuery.getId(), allocationEvent.getAllocatedContainer())
+ );
subQuery.eventHandler.handle(new SubQueryEvent(subQuery.getId(), SubQueryEventType.SQ_START));
} catch (Throwable t) {
@@ -1107,6 +1109,20 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
private void cleanup() {
stopScheduler();
releaseContainers();
+
+ if (!getContext().getConf().getBoolVar(TajoConf.ConfVars.TAJO_DEBUG)) {
+ List<ExecutionBlock> childs = getMasterPlan().getChilds(getId());
+ List<TajoIdProtos.ExecutionBlockIdProto> ebIds = Lists.newArrayList();
+ for (ExecutionBlock executionBlock : childs){
+ ebIds.add(executionBlock.getId().getProto());
+ }
+
+ try {
+ getContext().getQueryMasterContext().getQueryMaster().cleanupExecutionBlock(ebIds);
+ } catch (Throwable e) {
+ LOG.error(e);
+ }
+ }
}
private static class SubQueryCompleteTransition
@@ -1114,7 +1130,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
@Override
public SubQueryState transition(SubQuery subQuery, SubQueryEvent subQueryEvent) {
- // TODO - Commit subQuery & do cleanup
+ // TODO - Commit subQuery
// TODO - records succeeded, failed, killed completed task
// TODO - records metrics
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0f3412a7/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
index 13ef15d..e77da70 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
+import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.TajoIdProtos;
@@ -151,4 +152,16 @@ public class TajoWorkerManagerService extends CompositeService
workerContext.cleanup(new QueryId(request).toString());
done.run(TajoWorker.TRUE_PROTO);
}
+
+ @Override
+ public void cleanupExecutionBlocks(RpcController controller, TajoWorkerProtocol.ExecutionBlockListProto request,
+ RpcCallback<PrimitiveProtos.BoolProto> done) {
+ for (TajoIdProtos.ExecutionBlockIdProto executionBlockIdProto : request.getExecutionBlockIdList()) {
+ String inputDir = TaskRunner.getBaseInputDir(new ExecutionBlockId(executionBlockIdProto)).toString();
+ workerContext.cleanup(inputDir);
+ String outputDir = TaskRunner.getBaseOutputDir(new ExecutionBlockId(executionBlockIdProto)).toString();
+ workerContext.cleanup(outputDir);
+ }
+ done.run(TajoWorker.TRUE_PROTO);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0f3412a7/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
index 230c63a..3a4536a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
@@ -794,12 +794,10 @@ public class Task {
}
}
}
+
public static Path getTaskAttemptDir(QueryUnitAttemptId quid) {
Path workDir =
- StorageUtil.concatPath(
- quid.getQueryUnitId().getExecutionBlockId().getQueryId().toString(),
- "in",
- quid.getQueryUnitId().getExecutionBlockId().toString(),
+ StorageUtil.concatPath(TaskRunner.getBaseInputDir(quid.getQueryUnitId().getExecutionBlockId()),
String.valueOf(quid.getQueryUnitId().getId()),
String.valueOf(quid.getId()));
return workDir;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0f3412a7/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
index 3fcee06..9676192 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
@@ -45,6 +45,7 @@ import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.NullCallback;
import org.apache.tajo.rpc.RpcConnectionPool;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.TajoIdUtils;
import java.net.InetSocketAddress;
@@ -172,6 +173,24 @@ public class TaskRunner extends AbstractService {
return executionBlockId + "," + containerId;
}
+ public static Path getBaseOutputDir(ExecutionBlockId executionBlockId){
+ Path workDir =
+ StorageUtil.concatPath(
+ executionBlockId.getQueryId().toString(),
+ "output",
+ String.valueOf(executionBlockId.getId()));
+ return workDir;
+ }
+
+ public static Path getBaseInputDir(ExecutionBlockId executionBlockId) {
+ Path workDir =
+ StorageUtil.concatPath(
+ executionBlockId.getQueryId().toString(),
+ "in",
+ executionBlockId.toString());
+ return workDir;
+ }
+
@Override
public void init(Configuration conf) {
this.systemConf = (TajoConf)conf;
@@ -182,7 +201,7 @@ public class TaskRunner extends AbstractService {
localFS = FileSystem.getLocal(conf);
// the base dir for an output dir
- baseDir = queryId.toString() + "/output" + "/" + executionBlockId.getId();
+ baseDir = getBaseOutputDir(executionBlockId).toString();
// initialize LocalDirAllocator
lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0f3412a7/tajo-core/src/main/proto/TajoWorkerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
index ce8ce86..dc2b1d7 100644
--- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto
+++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
@@ -179,6 +179,10 @@ message RunExecutionBlockRequestProto {
optional string queryOutputPath = 6;
}
+message ExecutionBlockListProto {
+ repeated ExecutionBlockIdProto executionBlockId = 1;
+}
+
service TajoWorkerProtocolService {
rpc ping (QueryUnitAttemptIdProto) returns (BoolProto);
@@ -186,6 +190,7 @@ service TajoWorkerProtocolService {
rpc executeExecutionBlock(RunExecutionBlockRequestProto) returns (BoolProto);
rpc killTaskAttempt(QueryUnitAttemptIdProto) returns (BoolProto);
rpc cleanup(QueryIdProto) returns (BoolProto);
+ rpc cleanupExecutionBlocks(ExecutionBlockListProto) returns (BoolProto);
}
message EnforceProperty {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0f3412a7/tajo-core/src/test/java/org/apache/tajo/worker/TestDeletionService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestDeletionService.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestDeletionService.java
new file mode 100644
index 0000000..98251c1
--- /dev/null
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestDeletionService.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.worker;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.util.CommonTestingUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestDeletionService {
+ DeletionService deletionService;
+
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ @After
+ public void tearDown() {
+ if(deletionService != null){
+ deletionService.stop();
+ }
+ }
+
+ @Test
+ public final void testTemporalDirectory() throws IOException, InterruptedException {
+ int delay = 1;
+ deletionService = new DeletionService(1, delay);
+ FileSystem fs = FileSystem.getLocal(new Configuration());
+ Path tempPath = CommonTestingUtil.getTestDir();
+ assertTrue(fs.exists(tempPath));
+ deletionService.delete(tempPath);
+ assertTrue(fs.exists(tempPath));
+
+ Thread.sleep(delay * 2 * 1000);
+ assertFalse(fs.exists(tempPath));
+ }
+}
[14/17] TAJO-928: Session variables should override query configs in
TajoConf.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java b/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java
index a67b6c8..cdf552d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java
@@ -19,6 +19,7 @@
package org.apache.tajo.master.session;
import com.google.common.collect.ImmutableMap;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.common.ProtoObject;
@@ -30,17 +31,21 @@ import static org.apache.tajo.ipc.TajoWorkerProtocol.SessionProto;
public class Session implements SessionConstants, ProtoObject<SessionProto> {
private final String sessionId;
private final String userName;
+ private String currentDatabase;
private final Map<String, String> sessionVariables;
// transient status
private volatile long lastAccessTime;
- private volatile String currentDatabase;
public Session(String sessionId, String userName, String databaseName) {
this.sessionId = sessionId;
this.userName = userName;
+ this.currentDatabase = databaseName;
this.lastAccessTime = System.currentTimeMillis();
+
this.sessionVariables = new HashMap<String, String>();
+ sessionVariables.put(SessionVars.SESSION_ID.keyname(), sessionId);
+ sessionVariables.put(SessionVars.USERNAME.keyname(), userName);
selectDatabase(databaseName);
}
@@ -71,64 +76,61 @@ public class Session implements SessionConstants, ProtoObject<SessionProto> {
public void setVariable(String name, String value) {
synchronized (sessionVariables) {
- sessionVariables.put(name, value);
+ sessionVariables.put(SessionVars.handleDeprecatedName(name), value);
}
}
public String getVariable(String name) throws NoSuchSessionVariableException {
synchronized (sessionVariables) {
if (sessionVariables.containsKey(name)) {
- return sessionVariables.get(name);
+ return sessionVariables.get(SessionVars.handleDeprecatedName(name));
} else {
throw new NoSuchSessionVariableException(name);
}
}
}
- public String getVariable(String name, String defaultValue) {
- synchronized (sessionVariables) {
- if (sessionVariables.containsKey(name)) {
- return sessionVariables.get(name);
- } else {
- return defaultValue;
- }
- }
- }
-
public void removeVariable(String name) {
synchronized (sessionVariables) {
- sessionVariables.remove(name);
+ sessionVariables.remove(SessionVars.handleDeprecatedName(name));
}
}
public synchronized Map<String, String> getAllVariables() {
synchronized (sessionVariables) {
+ sessionVariables.put(SessionVars.SESSION_ID.keyname(), sessionId);
+ sessionVariables.put(SessionVars.USERNAME.keyname(), userName);
+ sessionVariables.put(SessionVars.SESSION_LAST_ACCESS_TIME.keyname(), String.valueOf(lastAccessTime));
+ sessionVariables.put(SessionVars.CURRENT_DATABASE.keyname(), currentDatabase);
return ImmutableMap.copyOf(sessionVariables);
}
}
- public void selectDatabase(String databaseName) {
+ public synchronized void selectDatabase(String databaseName) {
this.currentDatabase = databaseName;
}
- public String getCurrentDatabase() {
- return this.currentDatabase;
+ public synchronized String getCurrentDatabase() {
+ return currentDatabase;
}
@Override
public SessionProto getProto() {
SessionProto.Builder builder = SessionProto.newBuilder();
- builder.setSessionId(sessionId);
- builder.setUsername(userName);
- builder.setCurrentDatabase(currentDatabase);
+ builder.setSessionId(getSessionId());
+ builder.setUsername(getUserName());
+ builder.setCurrentDatabase(getCurrentDatabase());
builder.setLastAccessTime(lastAccessTime);
KeyValueSet variables = new KeyValueSet();
- variables.putAll(this.sessionVariables);
- builder.setVariables(variables.getProto());
- return builder.build();
+
+ synchronized (sessionVariables) {
+ variables.putAll(this.sessionVariables);
+ builder.setVariables(variables.getProto());
+ return builder.build();
+ }
}
public String toString() {
- return "user=" + userName + ",id=" + sessionId;
+ return "user=" + getUserName() + ",id=" + getSessionId() +",last_atime=" + getLastAccessTime();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/master/session/SessionLivelinessMonitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/session/SessionLivelinessMonitor.java b/tajo-core/src/main/java/org/apache/tajo/master/session/SessionLivelinessMonitor.java
index 483920f..912f769 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/session/SessionLivelinessMonitor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/session/SessionLivelinessMonitor.java
@@ -40,7 +40,7 @@ public class SessionLivelinessMonitor extends AbstractLivelinessMonitor<String>
TajoConf systemConf = (TajoConf) conf;
// seconds
- int expireIntvl = systemConf.getIntVar(TajoConf.ConfVars.CLIENT_SESSION_EXPIRY_TIME);
+ int expireIntvl = systemConf.getIntVar(TajoConf.ConfVars.$CLIENT_SESSION_EXPIRY_TIME);
setExpireInterval(expireIntvl);
setMonitorInterval(expireIntvl / 3);
super.serviceInit(conf);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
index 195b35e..200892a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
@@ -148,7 +148,7 @@ public class Task {
this.taskDir = StorageUtil.concatPath(taskRunnerContext.getBaseDir(),
taskId.getQueryUnitId().getId() + "_" + taskId.getId());
- this.context = new TaskAttemptContext(systemConf, queryContext, taskId,
+ this.context = new TaskAttemptContext(queryContext, taskId,
request.getFragments().toArray(new FragmentProto[request.getFragments().size()]), taskDir);
this.context.setDataChannel(request.getDataChannel());
this.context.setEnforcer(request.getEnforcer());
@@ -537,7 +537,7 @@ public class Task {
FetcherHistoryProto.Builder builder = FetcherHistoryProto.newBuilder();
for (Fetcher fetcher : fetcherRunners) {
// TODO store the fetcher histories
- if (systemConf.getBoolVar(TajoConf.ConfVars.TAJO_DEBUG)) {
+ if (systemConf.getBoolVar(TajoConf.ConfVars.$DEBUG_ENABLED)) {
builder.setStartTime(fetcher.getStartTime());
builder.setFinishTime(fetcher.getFinishTime());
builder.setFileLength(fetcher.getFileLen());
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
index db4af45..211f953 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
@@ -50,7 +50,6 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
*/
public class TaskAttemptContext {
private static final Log LOG = LogFactory.getLog(TaskAttemptContext.class);
- private final TajoConf conf;
private final Map<String, List<FragmentProto>> fragmentMap = Maps.newHashMap();
private TaskAttemptState state;
@@ -76,10 +75,9 @@ public class TaskAttemptContext {
/** a output volume for each partition */
private Map<Integer, Long> partitionOutputVolume;
- public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryUnitAttemptId queryId,
+ public TaskAttemptContext(final QueryContext queryContext, final QueryUnitAttemptId queryId,
final FragmentProto[] fragments,
final Path workDir) {
- this.conf = conf;
this.queryContext = queryContext;
this.queryId = queryId;
@@ -104,15 +102,15 @@ public class TaskAttemptContext {
}
@VisibleForTesting
- public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryUnitAttemptId queryId,
+ public TaskAttemptContext(final QueryContext queryContext, final QueryUnitAttemptId queryId,
final Fragment [] fragments, final Path workDir) {
- this(conf, queryContext, queryId, FragmentConvertor.toFragmentProtoArray(fragments), workDir);
+ this(queryContext, queryId, FragmentConvertor.toFragmentProtoArray(fragments), workDir);
}
public TajoConf getConf() {
- return this.conf;
+ return queryContext.getConf();
}
-
+
public TaskAttemptState getState() {
return this.state;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/main/resources/webapps/worker/querydetail.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/querydetail.jsp b/tajo-core/src/main/resources/webapps/worker/querydetail.jsp
index c0bee9b..3c40e99 100644
--- a/tajo-core/src/main/resources/webapps/worker/querydetail.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/querydetail.jsp
@@ -29,6 +29,8 @@
<%@ page import="org.apache.tajo.worker.TajoWorker" %>
<%@ page import="java.text.SimpleDateFormat" %>
<%@ page import="java.util.List" %>
+<%@ page import="java.util.Map" %>
+<%@ page import="org.apache.tajo.SessionVars" %>
<%
QueryId queryId = TajoIdUtils.parseQueryId(request.getParameter("queryId"));
@@ -93,6 +95,14 @@ for(SubQuery eachSubQuery: subQueries) {
%>
</table>
<p/>
+ <h3>Applied Session Variables</h3>
+ <table width="100%" border="1" class="border_table">
+ <%for(Map.Entry<String,String> entry: query.getPlan().getContext().getAllKeyValus().entrySet()) {
+ if (SessionVars.exists(entry.getKey()) && SessionVars.isPublic(SessionVars.get(entry.getKey()))) {
+ %> <tr><td width="200"><%=entry.getKey()%></td><td><%=entry.getValue()%></td> <%
+ }
+ } %>
+ </table>
<hr/>
<h3>Logical Plan</h3>
<pre style="white-space:pre-wrap;"><%=query.getPlan().getLogicalPlan().toString()%></pre>
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
index 84522e1..271ba70 100644
--- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
+++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
@@ -31,6 +31,7 @@ import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.global.MasterPlan;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.master.session.Session;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TajoIdUtils;
@@ -69,6 +70,10 @@ public class LocalTajoTestingUtility {
return new Session(UUID.randomUUID().toString(), dummyUserInfo.getUserName(), TajoConstants.DEFAULT_DATABASE_NAME);
}
+ public static QueryContext createDummyContext(TajoConf conf) {
+ return new QueryContext(conf, createDummySession());
+ }
+
/**
* for test
* @return The generated QueryId
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index 9d15732..8cd1bff 100644
--- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -214,7 +214,7 @@ public class QueryTestCaseBase {
} catch (ServiceException e) {
e.printStackTrace();
}
- testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "false");
+ testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
}
protected TajoClient getClient() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/TpchTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TpchTestBase.java b/tajo-core/src/test/java/org/apache/tajo/TpchTestBase.java
index 5ed8821..aacae36 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TpchTestBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TpchTestBase.java
@@ -91,7 +91,7 @@ public class TpchTestBase {
private void setUp() throws Exception {
util = new LocalTajoTestingUtility();
KeyValueSet opt = new KeyValueSet();
- opt.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ opt.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
util.setup(names, paths, schemas, opt);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java
index e13eeef..bd2b4e1 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java
@@ -18,6 +18,9 @@
package org.apache.tajo.cli;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.TpchTestBase;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
@@ -27,10 +30,14 @@ import org.apache.tajo.datum.Int4Datum;
import org.apache.tajo.datum.TextDatum;
import org.apache.tajo.jdbc.MetaDataTuple;
import org.apache.tajo.jdbc.TajoMetaDataResultSet;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
+import java.io.ByteArrayOutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.net.URL;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Arrays;
@@ -39,6 +46,38 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
public class TestDefaultCliOutputFormatter {
+ protected static final TpchTestBase testBase;
+ protected static final TajoTestingCluster cluster;
+
+ /** the base path of result directories */
+ protected static final Path resultBasePath;
+ static {
+ testBase = TpchTestBase.getInstance();
+ cluster = testBase.getTestingCluster();
+ URL resultBaseURL = ClassLoader.getSystemResource("results");
+ resultBasePath = new Path(resultBaseURL.toString());
+ }
+
+ private TajoConf conf;
+ private TajoCli tajoCli;
+ private TajoCli.TajoCliContext cliContext;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = cluster.getConfiguration();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ tajoCli = new TajoCli(conf, new String[]{}, System.in, out);
+ cliContext = tajoCli.getContext();
+ }
+
+ @After
+ public void tearDown() {
+ if (tajoCli != null) {
+ tajoCli.close();
+ }
+ }
+
+
@Test
public void testParseErrorMessage() {
String message = "java.sql.SQLException: ERROR: no such a table: table1";
@@ -65,9 +104,10 @@ public class TestDefaultCliOutputFormatter {
@Test
public void testPrintResultInsertStatement() throws Exception {
- TajoConf tajoConf = new TajoConf();
+
+
DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter();
- outputFormatter.init(tajoConf);
+ outputFormatter.init(cliContext);
float responseTime = 10.1f;
long numBytes = 102;
@@ -89,9 +129,8 @@ public class TestDefaultCliOutputFormatter {
@Test
public void testPrintResultSelectStatement() throws Exception {
- TajoConf tajoConf = new TajoConf();
DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter();
- outputFormatter.init(tajoConf);
+ outputFormatter.init(cliContext);
float responseTime = 10.1f;
long numBytes = 102;
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java
index 0631b6e..5795f0c 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java
@@ -23,15 +23,16 @@ import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.ConfigKey;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.TpchTestBase;
import org.apache.tajo.client.QueryStatus;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.FileUtil;
-import org.apache.tajo.util.NetUtils;
import org.junit.After;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
@@ -41,9 +42,7 @@ import java.io.File;
import java.io.PrintWriter;
import java.net.URL;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class TestTajoCli {
protected static final TpchTestBase testBase;
@@ -60,6 +59,7 @@ public class TestTajoCli {
private TajoCli tajoCli;
private Path currentResultPath;
+ private ByteArrayOutputStream out;
@Rule
public TestName name = new TestName();
@@ -69,6 +69,12 @@ public class TestTajoCli {
currentResultPath = new Path(resultBasePath, className);
}
+ @Before
+ public void setUp() throws Exception {
+ out = new ByteArrayOutputStream();
+ tajoCli = new TajoCli(cluster.getConfiguration(), new String[]{}, System.in, out);
+ }
+
@After
public void tearDown() {
if (tajoCli != null) {
@@ -76,6 +82,40 @@ public class TestTajoCli {
}
}
+ private static void setVar(TajoCli cli, ConfigKey key, String val) throws Exception {
+ cli.executeMetaCommand("\\set " + key.keyname() +" " + val);
+ }
+
+ private static void assertSessionVar(TajoCli cli, String key, String expectedVal) {
+ assertEquals(cli.getContext().getCliSideVar(key), expectedVal);
+ }
+
+ private void assertOutputResult(String actual) throws Exception {
+ assertOutputResult(name.getMethodName() + ".result", actual);
+ }
+
+ private void assertOutputResult(String expectedResultFile, String actual) throws Exception {
+ assertOutputResult(expectedResultFile, actual, null, null);
+ }
+
+ private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues)
+ throws Exception {
+ FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+ Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile);
+ assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile));
+
+ String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
+
+ if (paramKeys != null) {
+ for (int i = 0; i < paramKeys.length; i++) {
+ if (i < paramValues.length) {
+ expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]);
+ }
+ }
+ }
+ assertEquals(expectedResult.trim(), actual.trim());
+ }
+
@Test
public void testParseParam() throws Exception {
String[] args = new String[]{"-f", "test.sql", "--param", "test1=10", "--param", "test2=20"};
@@ -112,10 +152,9 @@ public class TestTajoCli {
assertEquals("tajo.executor.join.inner.in-memory-table-num=256", confValues[1]);
TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
-
- tajoCli = new TajoCli(tajoConf, args, System.in, System.out);
- assertEquals("false", tajoCli.getContext().getConf().get("tajo.cli.print.pause"));
- assertEquals("256", tajoCli.getContext().getConf().get("tajo.executor.join.inner.in-memory-table-num"));
+ TajoCli testCli = new TajoCli(tajoConf, args, System.in, System.out);
+ assertEquals("false", testCli.getContext().get(SessionVars.CLI_PAGING_ENABLED));
+ assertEquals("256", testCli.getContext().getConf().get("tajo.executor.join.inner.in-memory-table-num"));
}
@Test
@@ -131,10 +170,7 @@ public class TestTajoCli {
@Test
public void testLocalQueryWithoutFrom() throws Exception {
String sql = "select 'abc', '123'; select substr('123456', 1,3);";
- TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
+ setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
tajoCli.executeScript(sql);
String consoleResult = new String(out.toByteArray());
@@ -151,11 +187,7 @@ public class TestTajoCli {
databaseName = "TEST_CONNECTION_DATABASE";
}
String sql = "create database \"" + databaseName + "\";";
- TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
tajoCli.executeScript(sql);
tajoCli.executeMetaCommand("\\c " + databaseName);
@@ -179,11 +211,7 @@ public class TestTajoCli {
String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4);";
- TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
-
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
+ setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
tajoCli.executeScript(sql);
tajoCli.executeMetaCommand("\\d " + tableName);
@@ -211,79 +239,51 @@ public class TestTajoCli {
" c_custkey,\n" +
" orders.o_orderkey;\n";
- TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
-
- tajoConf.setVar(ConfVars.CLI_NULL_CHAR, "");
-
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
+ setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
tajoCli.executeScript(sql);
String consoleResult = new String(out.toByteArray());
assertOutputResult(consoleResult);
}
- @Test
- public void testSelectResultWithNullTrue() throws Exception {
+ private void verifySelectResultWithNullTrue() throws Exception {
String sql =
"select\n" +
- " c_custkey,\n" +
- " orders.o_orderkey,\n" +
- " orders.o_orderstatus \n" +
- "from\n" +
- " orders full outer join customer on c_custkey = o_orderkey\n" +
- "order by\n" +
- " c_custkey,\n" +
- " orders.o_orderkey;\n";
+ " c_custkey,\n" +
+ " orders.o_orderkey,\n" +
+ " orders.o_orderstatus \n" +
+ "from\n" +
+ " orders full outer join customer on c_custkey = o_orderkey\n" +
+ "order by\n" +
+ " c_custkey,\n" +
+ " orders.o_orderkey;\n";
- TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
- tajoConf.setVar(ConfVars.CLI_NULL_CHAR, "testnull");
+ setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
+ assertSessionVar(tajoCli, SessionVars.CLI_NULL_CHAR.keyname(), "testnull");
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
tajoCli.executeScript(sql);
String consoleResult = new String(out.toByteArray());
assertOutputResult(consoleResult);
}
- private void assertOutputResult(String actual) throws Exception {
- assertOutputResult(name.getMethodName() + ".result", actual);
- }
-
- private void assertOutputResult(String expectedResultFile, String actual) throws Exception {
- assertOutputResult(expectedResultFile, actual, null, null);
+ @Test
+ public void testSelectResultWithNullTrueDeprecated() throws Exception {
+ setVar(tajoCli, TajoConf.ConfVars.$CLI_NULL_CHAR, "testnull");
+ verifySelectResultWithNullTrue();
}
- private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues)
- throws Exception {
- FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
- Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile);
- assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile));
-
- String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
-
- if (paramKeys != null) {
- for (int i = 0; i < paramKeys.length; i++) {
- if (i < paramValues.length) {
- expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]);
- }
- }
- }
- assertEquals(expectedResult, actual);
+ @Test
+ public void testSelectResultWithNullTrue() throws Exception {
+ setVar(tajoCli, SessionVars.CLI_NULL_CHAR, "testnull");
+ verifySelectResultWithNullTrue();
}
- @Test
- public void testStopWhenError() throws Exception {
- TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
+ private void verifyStopWhenError() throws Exception {
+ setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
- tajoCli.executeMetaCommand("\\set tajo.cli.error.stop true");
+ assertSessionVar(tajoCli, SessionVars.ON_ERROR_STOP.keyname(), "true");
tajoCli.executeScript("select count(*) from lineitem; " +
"select count(*) from lineitem2; " +
@@ -296,20 +296,20 @@ public class TestTajoCli {
@Test
public void testGetConf() throws Exception {
TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
+ setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
ByteArrayOutputStream out = new ByteArrayOutputStream();
tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
tajoCli.executeMetaCommand("\\getconf tajo.rootdir");
String consoleResult = new String(out.toByteArray());
- assertEquals(consoleResult, tajoCli.getContext().getConf().getVar(ConfVars.ROOT_DIR) + "\n");
+ assertEquals(consoleResult, tajoCli.getContext().getConf().getVar(TajoConf.ConfVars.ROOT_DIR) + "\n");
}
@Test
public void testShowMasters() throws Exception {
TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
- tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
+ setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
ByteArrayOutputStream out = new ByteArrayOutputStream();
tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
@@ -317,12 +317,30 @@ public class TestTajoCli {
String consoleResult = new String(out.toByteArray());
- String masterAddress = tajoCli.getContext().getConf().getVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS);
+ String masterAddress = tajoCli.getContext().getConf().getVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS);
String host = masterAddress.split(":")[0];
assertEquals(consoleResult, host + "\n");
}
+ @Test
+ public void testStopWhenErrorDeprecated() throws Exception {
+ tajoCli.executeMetaCommand("\\set tajo.cli.error.stop true");
+ verifyStopWhenError();
+ }
+
+ @Test
+ public void testStopWhenError() throws Exception {
+ tajoCli.executeMetaCommand("\\set ON_ERROR_STOP true");
+ verifyStopWhenError();
+ }
+
+ @Test
+ public void testHelpSessionVars() throws Exception {
+ tajoCli.executeMetaCommand("\\help set");
+ assertOutputResult(new String(out.toByteArray()));
+ }
+
public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputFormatter {
@Override
protected String getResponseTimeReadable(float responseTime) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 732f02c..3d6ed2b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -33,7 +33,6 @@ import org.apache.tajo.catalog.FunctionDesc;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.ipc.ClientProtos;
import org.apache.tajo.jdbc.TajoResultSet;
import org.apache.tajo.storage.StorageConstants;
@@ -176,7 +175,8 @@ public class TestTajoClient {
String key = prefixName + i;
String val = prefixValue + i;
- assertEquals(i, client.getAllSessionVariables().size());
+ // Basically,
+ assertEquals(i + 4, client.getAllSessionVariables().size());
assertFalse(client.getAllSessionVariables().containsKey(key));
assertFalse(client.existSessionVariable(key));
@@ -184,7 +184,7 @@ public class TestTajoClient {
map.put(key, val);
client.updateSessionVariables(map);
- assertEquals(i + 1, client.getAllSessionVariables().size());
+ assertEquals(i + 5, client.getAllSessionVariables().size());
assertTrue(client.getAllSessionVariables().containsKey(key));
assertTrue(client.existSessionVariable(key));
}
@@ -690,7 +690,7 @@ public class TestTajoClient {
TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
Map<String, String> variables = new HashMap<String, String>();
- variables.put(ConfVars.CSVFILE_NULL.varname, "\\\\T");
+ variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T");
client.updateSessionVariables(variables);
TajoResultSet res = (TajoResultSet)client.executeQueryAndGetResult(sql);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index ad80ddf..7f9436a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -32,6 +32,7 @@ import org.apache.tajo.datum.*;
import org.apache.tajo.engine.json.CoreGsonHelper;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.planner.*;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.engine.utils.SchemaUtil;
import org.apache.tajo.master.TajoMaster;
import org.apache.tajo.master.session.Session;
@@ -107,22 +108,24 @@ public class ExprTestBase {
InvalidStatementException {
Session session = LocalTajoTestingUtility.createDummySession();
+ QueryContext context = new QueryContext(util.getConfiguration(), session);
+
List<ParsedResult> parsedResults = SimpleParser.parseScript(query);
if (parsedResults.size() > 1) {
throw new RuntimeException("this query includes two or more statements.");
}
Expr expr = analyzer.parse(parsedResults.get(0).getHistoryStatement());
VerificationState state = new VerificationState();
- preLogicalPlanVerifier.verify(session, state, expr);
+ preLogicalPlanVerifier.verify(context, state, expr);
if (state.getErrorMessages().size() > 0) {
if (!condition && state.getErrorMessages().size() > 0) {
throw new PlanningException(state.getErrorMessages().get(0));
}
assertFalse(state.getErrorMessages().get(0), true);
}
- LogicalPlan plan = planner.createPlan(session, expr, true);
+ LogicalPlan plan = planner.createPlan(context, expr, true);
optimizer.optimize(plan);
- annotatedPlanVerifier.verify(session, state, plan);
+ annotatedPlanVerifier.verify(context, state, plan);
if (state.getErrorMessages().size() > 0) {
assertFalse(state.getErrorMessages().get(0), true);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
index 7bb619d..35f5aa2 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
@@ -40,9 +40,9 @@ import org.apache.tajo.engine.planner.Target;
import org.apache.tajo.engine.planner.logical.GroupbyNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.InternalException;
import org.apache.tajo.master.TajoMaster;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.util.CommonTestingUtil;
import org.junit.AfterClass;
@@ -67,7 +67,7 @@ public class TestEvalTreeUtil {
static EvalNode expr3;
static SQLAnalyzer analyzer;
static LogicalPlanner planner;
- static Session session = LocalTajoTestingUtility.createDummySession();
+ static QueryContext defaultContext;
public static class TestSum extends GeneralFunction {
private Integer x;
@@ -123,6 +123,8 @@ public class TestEvalTreeUtil {
"select name, score, age from people where test_sum(score * age, 50)", // 2
};
+ defaultContext = LocalTajoTestingUtility.createDummyContext(util.getConfiguration());
+
expr1 = getRootSelection(QUERIES[0]);
expr2 = getRootSelection(QUERIES[1]);
expr3 = getRootSelection(QUERIES[2]);
@@ -137,7 +139,7 @@ public class TestEvalTreeUtil {
Expr expr = analyzer.parse(query);
LogicalPlan plan = null;
try {
- plan = planner.createPlan(session, expr);
+ plan = planner.createPlan(defaultContext, expr);
} catch (PlanningException e) {
e.printStackTrace();
}
@@ -149,7 +151,7 @@ public class TestEvalTreeUtil {
Expr block = analyzer.parse(query);
LogicalPlan plan = null;
try {
- plan = planner.createPlan(session, block);
+ plan = planner.createPlan(defaultContext, block);
} catch (PlanningException e) {
e.printStackTrace();
}
@@ -225,7 +227,7 @@ public class TestEvalTreeUtil {
@Test
public final void testGetContainExprs() throws CloneNotSupportedException, PlanningException {
Expr expr = analyzer.parse(QUERIES[1]);
- LogicalPlan plan = planner.createPlan(session, expr, true);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr, true);
Target [] targets = plan.getRootBlock().getRawTargets();
Column col1 = new Column("default.people.score", TajoDataTypes.Type.INT4);
Collection<EvalNode> exprs =
@@ -303,7 +305,7 @@ public class TestEvalTreeUtil {
assertTrue(7.0d == node.eval(null, null).asFloat8());
Expr expr = analyzer.parse(QUERIES[1]);
- LogicalPlan plan = planner.createPlan(session, expr, true);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr, true);
targets = plan.getRootBlock().getRawTargets();
Column col1 = new Column("default.people.score", TajoDataTypes.Type.INT4);
Collection<EvalNode> exprs =
@@ -343,7 +345,7 @@ public class TestEvalTreeUtil {
public final void testFindDistinctAggFunctions() throws PlanningException {
String query = "select sum(score) + max(age) from people";
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
GroupbyNode groupByNode = plan.getRootBlock().getNode(NodeType.GROUP_BY);
EvalNode [] aggEvals = groupByNode.getAggFunctions();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
index 773382d..82e7f21 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
@@ -29,8 +29,8 @@ import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.engine.function.builtin.SumInt;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.planner.logical.*;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.master.TajoMaster;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
import org.junit.AfterClass;
@@ -48,7 +48,7 @@ public class TestLogicalOptimizer {
private static SQLAnalyzer sqlAnalyzer;
private static LogicalPlanner planner;
private static LogicalOptimizer optimizer;
- private static Session session = LocalTajoTestingUtility.createDummySession();
+ private static QueryContext defaultContext;
@BeforeClass
public static void setUp() throws Exception {
@@ -101,6 +101,8 @@ public class TestLogicalOptimizer {
sqlAnalyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
optimizer = new LogicalOptimizer(util.getConfiguration());
+
+ defaultContext = LocalTajoTestingUtility.createDummyContext(util.getConfiguration());
}
@AfterClass
@@ -121,7 +123,7 @@ public class TestLogicalOptimizer {
public final void testProjectionPushWithNaturalJoin() throws PlanningException, CloneNotSupportedException {
// two relations
Expr expr = sqlAnalyzer.parse(QUERIES[4]);
- LogicalPlan newPlan = planner.createPlan(session, expr);
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
LogicalNode plan = newPlan.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
LogicalRootNode root = (LogicalRootNode) plan;
@@ -148,7 +150,7 @@ public class TestLogicalOptimizer {
public final void testProjectionPushWithInnerJoin() throws PlanningException {
// two relations
Expr expr = sqlAnalyzer.parse(QUERIES[5]);
- LogicalPlan newPlan = planner.createPlan(session, expr);
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
optimizer.optimize(newPlan);
}
@@ -156,7 +158,7 @@ public class TestLogicalOptimizer {
public final void testProjectionPush() throws CloneNotSupportedException, PlanningException {
// two relations
Expr expr = sqlAnalyzer.parse(QUERIES[2]);
- LogicalPlan newPlan = planner.createPlan(session, expr);
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
LogicalNode plan = newPlan.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
@@ -178,7 +180,7 @@ public class TestLogicalOptimizer {
@Test
public final void testOptimizeWithGroupBy() throws CloneNotSupportedException, PlanningException {
Expr expr = sqlAnalyzer.parse(QUERIES[3]);
- LogicalPlan newPlan = planner.createPlan(session, expr);
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
LogicalNode plan = newPlan.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
@@ -205,7 +207,7 @@ public class TestLogicalOptimizer {
public final void testPushable() throws CloneNotSupportedException, PlanningException {
// two relations
Expr expr = sqlAnalyzer.parse(QUERIES[0]);
- LogicalPlan newPlan = planner.createPlan(session, expr);
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
LogicalNode plan = newPlan.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
@@ -236,7 +238,7 @@ public class TestLogicalOptimizer {
// Scan Pushable Test
expr = sqlAnalyzer.parse(QUERIES[1]);
- newPlan = planner.createPlan(session, expr);
+ newPlan = planner.createPlan(defaultContext, expr);
plan = newPlan.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
@@ -258,7 +260,7 @@ public class TestLogicalOptimizer {
@Test
public final void testInsertInto() throws CloneNotSupportedException, PlanningException {
Expr expr = sqlAnalyzer.parse(TestLogicalPlanner.insertStatements[0]);
- LogicalPlan newPlan = planner.createPlan(session, expr);
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
optimizer.optimize(newPlan);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java
index 2f1e0f9..934beef 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java
@@ -18,7 +18,6 @@
package org.apache.tajo.engine.planner;
-import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.benchmark.TPCH;
import org.apache.tajo.catalog.*;
@@ -88,7 +87,7 @@ public class TestLogicalPlan {
@Test
public final void testQueryBlockGraph() {
- LogicalPlan plan = new LogicalPlan(LocalTajoTestingUtility.createDummySession().getCurrentDatabase(), planner);
+ LogicalPlan plan = new LogicalPlan(planner);
LogicalPlan.QueryBlock root = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK);
LogicalPlan.QueryBlock new1 = plan.newQueryBlock();
LogicalPlan.QueryBlock new2 = plan.newQueryBlock();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index 6d0f7e0..73f72be 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -37,6 +37,7 @@ import org.apache.tajo.engine.function.builtin.SumInt;
import org.apache.tajo.engine.json.CoreGsonHelper;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.planner.logical.*;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.master.TajoMaster;
import org.apache.tajo.master.session.Session;
import org.apache.tajo.util.CommonTestingUtil;
@@ -154,8 +155,10 @@ public class TestLogicalPlanner {
@Test
public final void testSingleRelation() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[0]);
- LogicalPlan planNode = planner.createPlan(session, expr);
+ LogicalPlan planNode = planner.createPlan(qc, expr);
LogicalNode plan = planNode.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
TestLogicalNode.testCloneLogicalNode(plan);
@@ -186,9 +189,11 @@ public class TestLogicalPlanner {
@Test
public final void testImplicityJoinPlan() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
// two relations
Expr expr = sqlAnalyzer.parse(QUERIES[1]);
- LogicalPlan planNode = planner.createPlan(session, expr);
+ LogicalPlan planNode = planner.createPlan(qc, expr);
LogicalNode plan = planNode.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
@@ -221,7 +226,7 @@ public class TestLogicalPlanner {
// three relations
expr = sqlAnalyzer.parse(QUERIES[2]);
- plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
TestLogicalNode.testCloneLogicalNode(plan);
@@ -273,9 +278,10 @@ public class TestLogicalPlanner {
@Test
public final void testNaturalJoinPlan() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
// two relations
Expr context = sqlAnalyzer.parse(JOINS[0]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot();
testJsonSerDerObject(plan);
assertSchema(expectedJoinSchema, plan.getOutSchema());
@@ -304,9 +310,10 @@ public class TestLogicalPlanner {
@Test
public final void testInnerJoinPlan() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
// two relations
Expr expr = sqlAnalyzer.parse(JOINS[1]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
LogicalNode root = plan.getRootBlock().getRoot();
testJsonSerDerObject(root);
assertSchema(expectedJoinSchema, root.getOutSchema());
@@ -336,9 +343,11 @@ public class TestLogicalPlanner {
@Test
public final void testOuterJoinPlan() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
// two relations
Expr expr = sqlAnalyzer.parse(JOINS[2]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
assertSchema(expectedJoinSchema, plan.getOutSchema());
@@ -369,9 +378,11 @@ public class TestLogicalPlanner {
@Test
public final void testGroupby() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
// without 'having clause'
Expr context = sqlAnalyzer.parse(QUERIES[7]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
LogicalRootNode root = (LogicalRootNode) plan;
@@ -380,7 +391,7 @@ public class TestLogicalPlanner {
// with having clause
context = sqlAnalyzer.parse(QUERIES[3]);
- plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ plan = planner.createPlan(qc, context).getRootBlock().getRoot();
TestLogicalNode.testCloneLogicalNode(plan);
assertEquals(NodeType.ROOT, plan.getType());
@@ -411,7 +422,8 @@ public class TestLogicalPlanner {
public final void testMultipleJoin() throws IOException, PlanningException {
Expr expr = sqlAnalyzer.parse(
FileUtil.readTextFile(new File("src/test/resources/queries/TestJoinQuery/testTPCHQ2Join.sql")));
- LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr).getRootBlock().getRoot();
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
Schema expected = tpch.getOutSchema("q2");
assertSchema(expected, plan.getOutSchema());
@@ -448,8 +460,8 @@ public class TestLogicalPlanner {
qualMap.put(entry.getKey(), Boolean.TRUE);
}
} else if (rightType == EvalType.ROW_CONSTANT) {
- RowConstantEval rightField = (RowConstantEval)qual.getRightExpr();
- RowConstantEval rightJoinField = (RowConstantEval)entry.getKey().getRightExpr();
+ RowConstantEval rightField = qual.getRightExpr();
+ RowConstantEval rightJoinField = entry.getKey().getRightExpr();
if (leftJoinField.getColumnRef().getQualifiedName().equals(leftField.getColumnRef().getQualifiedName())) {
assertEquals(rightField.getValues().length, rightJoinField.getValues().length);
@@ -469,8 +481,9 @@ public class TestLogicalPlanner {
Expr expr = sqlAnalyzer.parse(
FileUtil.readTextFile(new File
("src/test/resources/queries/TestJoinQuery/testJoinWithMultipleJoinQual1.sql")));
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
LogicalNode node = plan.getRootBlock().getRoot();
testJsonSerDerObject(node);
@@ -510,8 +523,9 @@ public class TestLogicalPlanner {
Expr expr = sqlAnalyzer.parse(
FileUtil.readTextFile(new File
("src/test/resources/queries/TestJoinQuery/testJoinWithMultipleJoinQual2.sql")));
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),expr);
+ LogicalPlan plan = planner.createPlan(qc,expr);
LogicalNode node = plan.getRootBlock().getRoot();
testJsonSerDerObject(node);
@@ -550,8 +564,9 @@ public class TestLogicalPlanner {
Expr expr = sqlAnalyzer.parse(
FileUtil.readTextFile(new File
("src/test/resources/queries/TestJoinQuery/testJoinWithMultipleJoinQual3.sql")));
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
LogicalNode node = plan.getRootBlock().getRoot();
testJsonSerDerObject(node);
@@ -596,8 +611,9 @@ public class TestLogicalPlanner {
Expr expr = sqlAnalyzer.parse(
FileUtil.readTextFile(new File
("src/test/resources/queries/TestJoinQuery/testJoinWithMultipleJoinQual4.sql")));
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
LogicalNode node = plan.getRootBlock().getRoot();
testJsonSerDerObject(node);
@@ -683,8 +699,11 @@ public class TestLogicalPlanner {
@Test
public final void testStoreTable() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr context = sqlAnalyzer.parse(QUERIES[8]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+
+ LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot();
TestLogicalNode.testCloneLogicalNode(plan);
testJsonSerDerObject(plan);
@@ -698,8 +717,11 @@ public class TestLogicalPlanner {
@Test
public final void testOrderBy() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[4]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
TestLogicalNode.testCloneLogicalNode(plan);
@@ -725,8 +747,11 @@ public class TestLogicalPlanner {
@Test
public final void testLimit() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[12]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
TestLogicalNode.testCloneLogicalNode(plan);
@@ -744,8 +769,10 @@ public class TestLogicalPlanner {
@Test
public final void testSPJPush() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[5]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
TestLogicalNode.testCloneLogicalNode(plan);
@@ -764,16 +791,20 @@ public class TestLogicalPlanner {
@Test
public final void testSPJ() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[6]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
TestLogicalNode.testCloneLogicalNode(plan);
}
@Test
public final void testJson() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[9]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
String json = plan.toJson();
@@ -792,9 +823,11 @@ public class TestLogicalPlanner {
@Test
public final void testVisitor() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
// two relations
Expr expr = sqlAnalyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
TestVisitor vis = new TestVisitor();
plan.postOrder(vis);
@@ -817,8 +850,10 @@ public class TestLogicalPlanner {
@Test
public final void testExprNode() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[10]);
- LogicalPlan rootNode = planner.createPlan(session, expr);
+ LogicalPlan rootNode = planner.createPlan(qc, expr);
LogicalNode plan = rootNode.getRootBlock().getRoot();
testJsonSerDerObject(plan);
assertEquals(NodeType.ROOT, plan.getType());
@@ -837,8 +872,10 @@ public class TestLogicalPlanner {
@Test
public final void testAsterisk() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[13]);
- LogicalPlan planNode = planner.createPlan(session, expr);
+ LogicalPlan planNode = planner.createPlan(qc, expr);
LogicalNode plan = planNode.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
TestLogicalNode.testCloneLogicalNode(plan);
@@ -865,8 +902,10 @@ public class TestLogicalPlanner {
@Test
public final void testAlias1() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(ALIAS[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
LogicalRootNode root = (LogicalRootNode) plan;
testJsonSerDerObject(root);
@@ -878,7 +917,7 @@ public class TestLogicalPlanner {
assertEquals("total", col.getSimpleName());
expr = sqlAnalyzer.parse(ALIAS[1]);
- plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
root = (LogicalRootNode) plan;
finalSchema = root.getOutSchema();
@@ -891,8 +930,10 @@ public class TestLogicalPlanner {
@Test
public final void testAlias2() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(ALIAS[1]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
LogicalRootNode root = (LogicalRootNode) plan;
testJsonSerDerObject(root);
@@ -910,8 +951,10 @@ public class TestLogicalPlanner {
@Test
public final void testCreateTableDef() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(CREATE_TABLE[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
LogicalRootNode root = (LogicalRootNode) plan;
testJsonSerDerObject(root);
assertEquals(NodeType.CREATE_TABLE, root.getChild().getType());
@@ -994,8 +1037,10 @@ public class TestLogicalPlanner {
@Test
public final void testSetPlan() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(setStatements[0]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot();
testJsonSerDerObject(plan);
assertEquals(NodeType.ROOT, plan.getType());
LogicalRootNode root = (LogicalRootNode) plan;
@@ -1013,8 +1058,10 @@ public class TestLogicalPlanner {
@Test
public void testSetQualifier() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr context = sqlAnalyzer.parse(setQualifiers[0]);
- LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot();
testJsonSerDerObject(plan);
assertEquals(NodeType.ROOT, plan.getType());
LogicalRootNode root = (LogicalRootNode) plan;
@@ -1023,7 +1070,7 @@ public class TestLogicalPlanner {
assertEquals(NodeType.SCAN, projectionNode.getChild().getType());
context = sqlAnalyzer.parse(setQualifiers[1]);
- plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ plan = planner.createPlan(qc, context).getRootBlock().getRoot();
testJsonSerDerObject(plan);
assertEquals(NodeType.ROOT, plan.getType());
root = (LogicalRootNode) plan;
@@ -1032,7 +1079,7 @@ public class TestLogicalPlanner {
assertEquals(NodeType.GROUP_BY, projectionNode.getChild().getType());
context = sqlAnalyzer.parse(setQualifiers[2]);
- plan = planner.createPlan(session, context).getRootBlock().getRoot();
+ plan = planner.createPlan(qc, context).getRootBlock().getRoot();
testJsonSerDerObject(plan);
root = (LogicalRootNode) plan;
assertEquals(NodeType.PROJECTION, root.getChild().getType());
@@ -1063,8 +1110,10 @@ public class TestLogicalPlanner {
@Test
public final void testInsertInto0() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(insertStatements[0]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
assertEquals(1, plan.getQueryBlocks().size());
InsertNode insertNode = getInsertNode(plan);
assertFalse(insertNode.isOverwrite());
@@ -1074,8 +1123,10 @@ public class TestLogicalPlanner {
@Test
public final void testInsertInto1() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(insertStatements[1]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
assertEquals(1, plan.getQueryBlocks().size());
InsertNode insertNode = getInsertNode(plan);
assertFalse(insertNode.isOverwrite());
@@ -1084,8 +1135,10 @@ public class TestLogicalPlanner {
@Test
public final void testInsertInto2() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(insertStatements[2]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
assertEquals(1, plan.getQueryBlocks().size());
InsertNode insertNode = getInsertNode(plan);
assertFalse(insertNode.isOverwrite());
@@ -1097,8 +1150,10 @@ public class TestLogicalPlanner {
@Test
public final void testInsertInto3() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(insertStatements[3]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
assertEquals(1, plan.getQueryBlocks().size());
InsertNode insertNode = getInsertNode(plan);
assertFalse(insertNode.isOverwrite());
@@ -1107,8 +1162,10 @@ public class TestLogicalPlanner {
@Test
public final void testInsertInto4() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(insertStatements[4]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
assertEquals(1, plan.getQueryBlocks().size());
InsertNode insertNode = getInsertNode(plan);
assertTrue(insertNode.isOverwrite());
@@ -1121,8 +1178,10 @@ public class TestLogicalPlanner {
@Test
public final void testInsertInto5() throws PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(insertStatements[5]);
- LogicalPlan plan = planner.createPlan(session, expr);
+ LogicalPlan plan = planner.createPlan(qc, expr);
assertEquals(1, plan.getQueryBlocks().size());
InsertNode insertNode = getInsertNode(plan);
assertTrue(insertNode.isOverwrite());
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
index 82e7818..b370be7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
@@ -31,7 +31,6 @@ import org.apache.tajo.engine.eval.*;
import org.apache.tajo.engine.function.builtin.SumInt;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.planner.logical.*;
-import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleComparator;
import org.apache.tajo.storage.VTuple;
@@ -52,7 +51,6 @@ public class TestPlannerUtil {
private static CatalogService catalog;
private static SQLAnalyzer analyzer;
private static LogicalPlanner planner;
- private static Session session = LocalTajoTestingUtility.createDummySession();
@BeforeClass
public static void setUp() throws Exception {
@@ -111,7 +109,8 @@ public class TestPlannerUtil {
public final void testFindTopNode() throws CloneNotSupportedException, PlanningException {
// two relations
Expr expr = analyzer.parse(TestLogicalPlanner.QUERIES[1]);
- LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(util.getConfiguration()),
+ expr).getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
LogicalRootNode root = (LogicalRootNode) plan;
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
index ec39609..4beb5c5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
@@ -66,6 +66,7 @@ public class TestBroadcastJoinPlan {
private TajoTestingCluster util;
private CatalogService catalog;
private SQLAnalyzer analyzer;
+ private QueryContext defaultContext;
private Path testDir;
private TableDesc smallTable1;
@@ -79,8 +80,8 @@ public class TestBroadcastJoinPlan {
public void setUp() throws Exception {
util = new TajoTestingCluster();
conf = util.getConfiguration();
- conf.setLongVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD, 500 * 1024);
- conf.setBoolVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO, true);
+ conf.setLongVar(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, 500 * 1024);
+ conf.setBoolVar(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED, true);
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
catalog = util.startCatalogCluster().getCatalog();
@@ -126,6 +127,7 @@ public class TestBroadcastJoinPlan {
catalog.createTable(largeTable3);
analyzer = new SQLAnalyzer();
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
private TableDesc makeTestData(String tableName, Schema schema, int dataSize) throws Exception {
@@ -183,12 +185,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -244,12 +246,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -308,12 +310,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -336,12 +338,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -386,12 +388,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -427,12 +429,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -483,12 +485,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -537,12 +539,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -620,12 +622,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -703,12 +705,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -762,12 +764,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -815,12 +817,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -907,12 +909,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
@@ -972,12 +974,12 @@ public class TestBroadcastJoinPlan {
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
Expr expr = analyzer.parse(query);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog);
globalPlanner.build(masterPlan);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
index d84796a..3fecabd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
@@ -141,7 +141,8 @@ public class TestBNLJoinExec {
@Test
public final void testBNLCrossJoin() throws IOException, PlanningException {
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr).getRootBlock().getRoot();
+ LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf),
+ expr).getRootBlock().getRoot();
JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN);
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN);
@@ -152,7 +153,7 @@ public class TestBNLJoinExec {
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLCrossJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
@@ -174,7 +175,7 @@ public class TestBNLJoinExec {
@Test
public final void testBNLInnerJoin() throws IOException, PlanningException {
Expr context = analyzer.parse(QUERIES[1]);
- LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),
+ LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf),
context).getRootBlock().getRoot();
FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(),
@@ -189,7 +190,7 @@ public class TestBNLJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLInnerJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(),
merged, workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
index bfc3522..f817776 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
@@ -169,10 +169,10 @@ public class TestBSTIndexExec {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual");
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
Expr expr = analyzer.parse(QUERY);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
LogicalNode rootNode = optimizer.optimize(plan);
TmpPlanner phyPlanner = new TmpPlanner(conf, sm);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index 1ce5b5b..e7aac3c 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -121,11 +121,11 @@ public class TestExternalSortExec {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
LogicalNode rootNode = plan.getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
[16/17] git commit: TAJO-928: Session variables should override query
configs in TajoConf.
Posted by ji...@apache.org.
TAJO-928: Session variables should override query configs in TajoConf.
Closes #98
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ddfc3f33
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ddfc3f33
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ddfc3f33
Branch: refs/heads/index_support
Commit: ddfc3f33039ee15ab0a2d3fe5890b9acb40aec3d
Parents: 0603b49
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 11 12:52:52 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 11 12:52:52 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../java/org/apache/tajo/catalog/TableMeta.java | 14 +-
.../apache/tajo/catalog/TestKeyValueSet.java | 83 ++++++-
.../tajo/catalog/store/HCatalogStore.java | 28 +--
.../tajo/catalog/store/TestHCatalogStore.java | 12 +-
.../org/apache/tajo/catalog/TestCatalog.java | 10 +-
.../tajo/cli/DefaultTajoCliOutputFormatter.java | 16 +-
.../java/org/apache/tajo/cli/HelpCommand.java | 100 ++++----
.../java/org/apache/tajo/cli/SetCommand.java | 78 +++++-
.../main/java/org/apache/tajo/cli/TajoCli.java | 149 ++++++++----
.../apache/tajo/cli/TajoCliOutputFormatter.java | 5 +-
.../org/apache/tajo/cli/TajoShellCommand.java | 10 +-
.../main/java/org/apache/tajo/ConfigKey.java | 52 ++++
.../java/org/apache/tajo/OverridableConf.java | 235 +++++++++++++++++++
.../main/java/org/apache/tajo/SessionVars.java | 211 +++++++++++++++++
.../java/org/apache/tajo/conf/TajoConf.java | 199 ++++++++--------
.../main/java/org/apache/tajo/datum/Datum.java | 2 +-
.../java/org/apache/tajo/datum/NullDatum.java | 1 +
.../java/org/apache/tajo/util/KeyValueSet.java | 149 ++++++++++--
.../tajo/datum/TestArithmeticOperator.java | 4 +-
.../tajo/engine/planner/LogicalOptimizer.java | 9 +-
.../apache/tajo/engine/planner/LogicalPlan.java | 4 +-
.../engine/planner/LogicalPlanPreprocessor.java | 18 +-
.../engine/planner/LogicalPlanVerifier.java | 12 +-
.../tajo/engine/planner/LogicalPlanner.java | 35 ++-
.../engine/planner/PhysicalPlannerImpl.java | 44 +++-
.../engine/planner/PreLogicalPlanVerifier.java | 21 +-
.../engine/planner/global/GlobalPlanner.java | 18 +-
.../planner/physical/ExternalSortExec.java | 4 +-
.../engine/planner/physical/StoreTableExec.java | 6 +-
.../apache/tajo/engine/query/QueryContext.java | 144 ++++++------
.../tajo/engine/query/QueryUnitRequestImpl.java | 3 +-
.../org/apache/tajo/master/GlobalEngine.java | 94 ++++----
.../tajo/master/TajoMasterClientService.java | 31 ++-
.../apache/tajo/master/querymaster/Query.java | 4 +-
.../tajo/master/querymaster/QueryMaster.java | 11 +-
.../querymaster/QueryMasterManagerService.java | 4 +-
.../master/querymaster/QueryMasterTask.java | 4 +-
.../tajo/master/querymaster/Repartitioner.java | 16 +-
.../tajo/master/querymaster/SubQuery.java | 15 +-
.../org/apache/tajo/master/session/Session.java | 50 ++--
.../session/SessionLivelinessMonitor.java | 2 +-
.../main/java/org/apache/tajo/worker/Task.java | 4 +-
.../apache/tajo/worker/TaskAttemptContext.java | 12 +-
.../resources/webapps/worker/querydetail.jsp | 10 +
.../apache/tajo/LocalTajoTestingUtility.java | 5 +
.../java/org/apache/tajo/QueryTestCaseBase.java | 2 +-
.../test/java/org/apache/tajo/TpchTestBase.java | 2 +-
.../tajo/cli/TestDefaultCliOutputFormatter.java | 47 +++-
.../java/org/apache/tajo/cli/TestTajoCli.java | 172 ++++++++------
.../org/apache/tajo/client/TestTajoClient.java | 8 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 9 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 16 +-
.../engine/planner/TestLogicalOptimizer.java | 20 +-
.../tajo/engine/planner/TestLogicalPlan.java | 3 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 135 ++++++++---
.../tajo/engine/planner/TestPlannerUtil.java | 5 +-
.../planner/global/TestBroadcastJoinPlan.java | 62 ++---
.../planner/physical/TestBNLJoinExec.java | 9 +-
.../planner/physical/TestBSTIndexExec.java | 4 +-
.../planner/physical/TestExternalSortExec.java | 4 +-
.../physical/TestFullOuterHashJoinExec.java | 24 +-
.../physical/TestFullOuterMergeJoinExec.java | 29 +--
.../planner/physical/TestHashAntiJoinExec.java | 4 +-
.../planner/physical/TestHashJoinExec.java | 19 +-
.../planner/physical/TestHashSemiJoinExec.java | 4 +-
.../physical/TestLeftOuterHashJoinExec.java | 28 +--
.../physical/TestLeftOuterNLJoinExec.java | 25 +-
.../planner/physical/TestMergeJoinExec.java | 4 +-
.../engine/planner/physical/TestNLJoinExec.java | 17 +-
.../planner/physical/TestPhysicalPlanner.java | 100 ++++----
.../physical/TestProgressExternalSortExec.java | 4 +-
.../physical/TestRightOuterHashJoinExec.java | 16 +-
.../physical/TestRightOuterMergeJoinExec.java | 32 +--
.../engine/planner/physical/TestSortExec.java | 4 +-
.../tajo/engine/query/TestGroupByQuery.java | 22 +-
.../tajo/engine/query/TestJoinBroadcast.java | 8 +-
.../apache/tajo/engine/query/TestJoinQuery.java | 90 +++----
.../tajo/engine/query/TestNullValues.java | 14 +-
.../tajo/engine/query/TestSelectQuery.java | 10 +-
.../apache/tajo/engine/query/TestSortQuery.java | 8 +-
.../org/apache/tajo/jdbc/TestResultSet.java | 2 +-
.../tajo/master/TestExecutionBlockCursor.java | 6 +-
.../apache/tajo/master/TestGlobalPlanner.java | 4 +-
.../querymaster/TestQueryUnitStatusUpdate.java | 2 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 8 +-
.../TestTajoCli/testHelpSessionVars.result | 36 +++
...estSelectResultWithNullTrueDeprecated.result | 9 +
.../testStopWhenErrorDeprecated.result | 6 +
.../java/org/apache/tajo/storage/Appender.java | 2 +-
.../java/org/apache/tajo/storage/CSVFile.java | 11 +-
.../org/apache/tajo/storage/StorageUtil.java | 18 +-
.../org/apache/tajo/storage/avro/AvroUtil.java | 17 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileAppender.java | 10 +-
.../sequencefile/SequenceFileScanner.java | 5 +-
.../apache/tajo/storage/v2/RCFileScanner.java | 2 +-
97 files changed, 2057 insertions(+), 992 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index deb41a0..182d07d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -29,6 +29,9 @@ Release 0.9.0 - unreleased
IMPROVEMENT
+ TAJO-928: Session variables should override query configs in TajoConf.
+ (hyunsik)
+
TAJO-989: Cleanup of child blocks after parent execution block is complete
(jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
index 6015fc9..2d95e6b 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
@@ -78,7 +78,19 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj
public void putOption(String key, String val) {
maybeInitBuilder();
- options.put(key, val);
+ options.set(key, val);
+ }
+
+ public boolean containsOption(String key) {
+ TableProtoOrBuilder p = viaProto ? proto : builder;
+ if (options != null) {
+ return this.options.containsKey(key);
+ }
+ if (!p.hasParams()) {
+ return false;
+ }
+ this.options = new KeyValueSet(p.getParams());
+ return options.containsKey(key);
}
public String getOption(String key) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java
index 39d5fbe..b317ba4 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java
@@ -24,23 +24,70 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
public class TestKeyValueSet {
+
@Test
- public final void testPutAndGet() {
+ public final void testPutAndGetString() {
KeyValueSet opts = new KeyValueSet();
- opts.put("name", "abc");
- opts.put("delimiter", ",");
+ opts.set("k1", "v1");
+ opts.set("k2", "v2");
- assertEquals(",", opts.get("delimiter"));
- assertEquals("abc", opts.get("name"));
+ assertEquals("v1", opts.get("k1"));
+ assertEquals("v2", opts.get("k2"));
+ assertEquals("default", opts.get("k3", "default"));
}
+ @Test
+ public final void testPutAndGetBool() {
+ KeyValueSet opts = new KeyValueSet();
+ opts.setBool("k1", true);
+ opts.setBool("k2", false);
+
+ assertEquals(true, opts.getBool("k1"));
+ assertEquals(false, opts.getBool("k2"));
+ assertEquals(true, opts.getBool("k3", true));
+ }
+
+ @Test
+ public final void testPutAndGetInt() {
+ KeyValueSet opts = new KeyValueSet();
+ opts.setInt("k1", 1980);
+ opts.setInt("k2", 401);
+
+ assertEquals(1980, opts.getInt("k1"));
+ assertEquals(401, opts.getInt("k2"));
+ assertEquals(2020, opts.getInt("k3", 2020));
+ }
+
+ @Test
+ public final void testPutAndGetLong() {
+ KeyValueSet opts = new KeyValueSet();
+ opts.setLong("k1", 1980);
+ opts.setLong("k2", 401);
+
+ assertEquals(1980, opts.getLong("k1"));
+ assertEquals(401, opts.getLong("k2"));
+ assertEquals(2020, opts.getLong("k3", 2020l));
+ }
+
+ @Test
+ public final void testPutAndGetFloat() {
+ KeyValueSet opts = new KeyValueSet();
+ opts.setFloat("k1", 1980.4f);
+ opts.setFloat("k2", 401.150f);
+
+ assertTrue(1980.4f == opts.getFloat("k1"));
+ assertTrue(401.150f == opts.getFloat("k2"));
+ assertTrue(3.14f == opts.getFloat("k3", 3.14f));
+ }
+
@Test
public final void testGetProto() {
KeyValueSet opts = new KeyValueSet();
- opts.put("name", "abc");
- opts.put("delimiter", ",");
+ opts.set("name", "abc");
+ opts.set("delimiter", ",");
PrimitiveProtos.KeyValueSetProto proto = opts.getProto();
KeyValueSet opts2 = new KeyValueSet(proto);
@@ -49,16 +96,26 @@ public class TestKeyValueSet {
}
@Test
- public final void testDelete() {
+ public final void testRemove() {
KeyValueSet opts = new KeyValueSet();
- opts.put("name", "abc");
- opts.put("delimiter", ",");
+ opts.set("name", "abc");
+ opts.set("delimiter", ",");
assertEquals("abc", opts.get("name"));
- assertEquals("abc", opts.delete("name"));
- assertNull(opts.get("name"));
+ assertEquals("abc", opts.remove("name"));
+ try {
+ opts.get("name");
+ assertTrue(false);
+ } catch (IllegalArgumentException iae) {
+ assertTrue(true);
+ }
KeyValueSet opts2 = new KeyValueSet(opts.getProto());
- assertNull(opts2.get("name"));
+ try {
+ opts2.get("name");
+ assertTrue(false);
+ } catch (IllegalArgumentException iae) {
+ assertTrue(true);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 6f48348..61bdccd 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -166,7 +166,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
stats = new TableStats();
options = new KeyValueSet();
options.putAll(table.getParameters());
- options.delete("EXTERNAL");
+ options.remove("EXTERNAL");
Properties properties = table.getMetadata();
if (properties != null) {
@@ -186,31 +186,31 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
} else {
nullFormat = "\\N";
}
- options.delete(serdeConstants.SERIALIZATION_NULL_FORMAT);
+ options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT);
// set file output format
String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT);
storeType = CatalogUtil.getStoreType(HCatalogUtil.getStoreType(fileOutputformat));
if (storeType.equals(CatalogProtos.StoreType.CSV)) {
- options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.set(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.set(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
} else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) {
- options.put(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) {
- options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+ options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (ColumnarSerDe.class.getName().equals(serde)) {
- options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
} else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) {
- options.put(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.put(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinarySerDe.class.getName().equals(serde)) {
- options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+ options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (LazySimpleSerDe.class.getName().equals(serde)) {
- options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
}
@@ -481,7 +481,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName());
}
- if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) {
+ if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) {
table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
@@ -504,7 +504,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
StringEscapeUtils.unescapeJava(fieldDelimiter));
table.getParameters().remove(StorageConstants.CSVFILE_DELIMITER);
- if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) {
+ if (tableDesc.getMeta().containsOption(StorageConstants.CSVFILE_NULL)) {
table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL)));
table.getParameters().remove(StorageConstants.CSVFILE_NULL);
@@ -534,7 +534,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
}
- if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) {
+ if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) {
table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index a507b08..f225424 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -117,7 +117,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingRCFileWithBinarySerde() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+ options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -146,7 +146,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingRCFileWithTextSerde() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -174,8 +174,8 @@ public class TestHCatalogStore {
@Test
public void testTableWithNullValue() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
- options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003"));
+ options.set(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
+ options.set(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003"));
TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -315,7 +315,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingSequenceFileWithBinarySerde() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
+ options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -344,7 +344,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingSequenceFileWithTextSerde() throws Exception {
KeyValueSet options = new KeyValueSet();
- options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
+ options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 4039ff2..654736d 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -565,7 +565,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
@@ -605,7 +605,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
Schema partSchema = new Schema();
@@ -643,7 +643,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
Schema partSchema = new Schema();
@@ -680,7 +680,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
Schema partSchema = new Schema();
@@ -717,7 +717,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
- opts.put("file.delimiter", ",");
+ opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
Schema partSchema = new Schema();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
index 62736b5..543ac8c 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
@@ -20,11 +20,10 @@ package org.apache.tajo.cli;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.tajo.QueryId;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.QueryStatus;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.util.FileUtil;
import java.io.InputStream;
@@ -33,20 +32,17 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
- private TajoConf tajoConf;
private int printPauseRecords;
private boolean printPause;
private boolean printErrorTrace;
private String nullChar;
@Override
- public void init(TajoConf tajoConf) {
- this.tajoConf = tajoConf;
-
- this.printPause = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_PAUSE);
- this.printPauseRecords = tajoConf.getIntVar(TajoConf.ConfVars.CLI_PRINT_PAUSE_NUM_RECORDS);
- this.printErrorTrace = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_ERROR_TRACE);
- this.nullChar = tajoConf.getVar(ConfVars.CLI_NULL_CHAR);
+ public void init(TajoCli.TajoCliContext context) {
+ this.printPause = context.getBool(SessionVars.CLI_PAGING_ENABLED);
+ this.printPauseRecords = context.getInt(SessionVars.CLI_PAGE_ROWS);
+ this.printErrorTrace = context.getBool(SessionVars.CLI_DISPLAY_ERROR_TRACE);
+ this.nullChar = context.get(SessionVars.CLI_NULL_CHAR);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java
index 8d908ec..52f13b0 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java
@@ -40,53 +40,69 @@ public class HelpCommand extends TajoShellCommand {
}
@Override
+ public String [] getAliases() {
+ return new String [] {"\\help"};
+ }
+
+ @Override
public void invoke(String[] cmd) throws Exception {
if(targetDocVersion.equalsIgnoreCase("")) {
targetDocVersion = getDocumentationVersion();
}
- PrintWriter sout = context.getOutput();
- sout.println();
-
- sout.println("General");
- sout.println(" \\copyright show Apache License 2.0");
- sout.println(" \\version show Tajo version");
- sout.println(" \\? show help");
- sout.println(" \\q quit tsql");
- sout.println();
- sout.println();
-
- sout.println("Informational");
- sout.println(" \\l list databases");
- sout.println(" \\c show current database");
- sout.println(" \\c [DBNAME] connect to new database");
- sout.println(" \\d list tables");
- sout.println(" \\d [TBNAME] describe table");
- sout.println(" \\df list functions");
- sout.println(" \\df NAME describe function");
- sout.println();
- sout.println();
-
- sout.println("Tool");
- sout.println(" \\! execute a linux shell command");
- sout.println(" \\dfs execute a dfs command");
- sout.println(" \\admin execute tajo admin command");
- sout.println();
- sout.println();
-
- sout.println("Variables");
- sout.println(" \\set [[NAME] [VALUE] set session variable or list session variables");
- sout.println(" \\unset NAME unset session variable");
- sout.println();
- sout.println();
-
- sout.println("Documentations");
- sout.println(" tsql guide http://tajo.apache.org/docs/"+ targetDocVersion +"/cli.html");
- sout.println(" Query language http://tajo.apache.org/docs/"+ targetDocVersion +"/sql_language.html");
- sout.println(" Functions http://tajo.apache.org/docs/"+ targetDocVersion +"/functions.html");
- sout.println(" Backup & restore http://tajo.apache.org/docs/"+ targetDocVersion +"/backup_and_restore.html");
- sout.println(" Configuration http://tajo.apache.org/docs/"+ targetDocVersion +"/configuration.html");
- sout.println();
+ if (cmd.length == 1) {
+ PrintWriter sout = context.getOutput();
+ sout.println();
+
+ sout.println("General");
+ sout.println(" \\copyright show Apache License 2.0");
+ sout.println(" \\version show Tajo version");
+ sout.println(" \\? show help");
+ sout.println(" \\? [COMMAND] show help of a given command");
+ sout.println(" \\help alias of \\?");
+ sout.println(" \\q quit tsql");
+ sout.println();
+ sout.println();
+
+ sout.println("Informational");
+ sout.println(" \\l list databases");
+ sout.println(" \\c show current database");
+ sout.println(" \\c [DBNAME] connect to new database");
+ sout.println(" \\d list tables");
+ sout.println(" \\d [TBNAME] describe table");
+ sout.println(" \\df list functions");
+ sout.println(" \\df NAME describe function");
+ sout.println();
+ sout.println();
+
+ sout.println("Tool");
+ sout.println(" \\! execute a linux shell command");
+ sout.println(" \\dfs execute a dfs command");
+ sout.println(" \\admin execute tajo admin command");
+ sout.println();
+ sout.println();
+
+ sout.println("Variables");
+ sout.println(" \\set [[NAME] [VALUE] set session variable or list session variables");
+ sout.println(" \\unset NAME unset session variable");
+ sout.println();
+ sout.println();
+
+ sout.println("Documentations");
+ sout.println(" tsql guide http://tajo.apache.org/docs/" + targetDocVersion + "/cli.html");
+ sout.println(" Query language http://tajo.apache.org/docs/" + targetDocVersion + "/sql_language.html");
+ sout.println(" Functions http://tajo.apache.org/docs/" + targetDocVersion + "/functions.html");
+ sout.println(" Backup & restore http://tajo.apache.org/docs/" + targetDocVersion + "/backup_and_restore.html");
+ sout.println(" Configuration http://tajo.apache.org/docs/" + targetDocVersion + "/configuration.html");
+ sout.println();
+ } else if (cmd.length == 2) {
+ String slashCommand = "\\" + cmd[1];
+ if (context.getCommands().containsKey(slashCommand)) {
+ context.getCommands().get(slashCommand).printHelp();
+ } else {
+ context.getOutput().println("Command not found: " + cmd[1]);
+ }
+ }
}
private String getDocumentationVersion() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java
index c694507..c957a19 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java
@@ -18,11 +18,15 @@
package org.apache.tajo.cli;
+import com.google.protobuf.ServiceException;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.util.StringUtils;
import java.util.HashMap;
import java.util.Map;
+import static org.apache.tajo.SessionVars.VariableMode;
+
public class SetCommand extends TajoShellCommand {
public SetCommand(TajoCli.TajoCliContext context) {
@@ -34,17 +38,46 @@ public class SetCommand extends TajoShellCommand {
return "\\set";
}
+ private void showAllSessionVars() throws ServiceException {
+ for (Map.Entry<String, String> entry: client.getAllSessionVariables().entrySet()) {
+ context.getOutput().println(StringUtils.quote(entry.getKey()) + "=" + StringUtils.quote(entry.getValue()));
+ }
+ }
+
+ private void updateSessionVariable(String key, String val) throws ServiceException {
+ Map<String, String> variables = new HashMap<String, String>();
+ variables.put(key, val);
+ client.updateSessionVariables(variables);
+ }
+
+ void set(String key, String val) throws ServiceException {
+ SessionVars sessionVar = null;
+
+ if (SessionVars.exists(key)) { // if the variable is one of the session variables
+ sessionVar = SessionVars.get(key);
+
+ // is it cli-side variable?
+ if (sessionVar.getMode() == VariableMode.CLI_SIDE_VAR) {
+ context.setCliSideVar(key, val);
+ } else {
+ updateSessionVariable(key, val);
+ }
+
+ if (SessionVars.isDeprecated(key)) {
+ context.getOutput().println("Warning: deprecated to directly use config key in TajoConf.ConfVars. " +
+ "Please execute '\\help set'.");
+ }
+ } else {
+ updateSessionVariable(key, val);
+ }
+ }
+
@Override
public void invoke(String[] cmd) throws Exception {
if (cmd.length == 1) {
- for (Map.Entry<String, String> entry: client.getAllSessionVariables().entrySet()) {
- context.getOutput().println(StringUtils.quote(entry.getKey()) + "=" + StringUtils.quote(entry.getValue()));
- }
+ showAllSessionVars();
} else if (cmd.length == 3) {
- Map<String, String> variables = new HashMap<String, String>();
- variables.put(cmd[1], cmd[2]);
- client.updateSessionVariables(variables);
- context.setVariable(cmd[1], cmd[2]);
+ set(cmd[1], cmd[2]);
} else {
context.getOutput().println("usage: \\set [[NAME] VALUE]");
}
@@ -59,4 +92,35 @@ public class SetCommand extends TajoShellCommand {
public String getDescription() {
return "set session variable or shows all session variables";
}
+
+ @Override
+ public void printHelp() {
+ context.getOutput().println("\nAvailable Session Variables:\n");
+ for (SessionVars var : SessionVars.values()) {
+
+ if (var.getMode() == VariableMode.DEFAULT ||
+ var.getMode() == VariableMode.CLI_SIDE_VAR ||
+ var.getMode() == VariableMode.FROM_SHELL_ENV) {
+
+ context.getOutput().println("\\set " + var.keyname() + " " + getDisplayType(var.getVarType()) + " - " + var
+ .getDescription());
+ }
+ }
+ }
+
+ public static String getDisplayType(Class<?> clazz) {
+ if (clazz == String.class) {
+ return "[text value]";
+ } else if (clazz == Integer.class) {
+ return "[int value]";
+ } else if (clazz == Long.class) {
+ return "[long value]";
+ } else if (clazz == Float.class) {
+ return "[real value]";
+ } else if (clazz == Boolean.class) {
+ return "[true or false]";
+ } else {
+ return clazz.getSimpleName();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
index c20e44b..81427ab 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
@@ -18,11 +18,12 @@
package org.apache.tajo.cli;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.protobuf.ServiceException;
import jline.console.ConsoleReader;
import org.apache.commons.cli.*;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.QueryIdFactory;
+import org.apache.tajo.*;
import org.apache.tajo.TajoProtos.QueryState;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.client.QueryStatus;
@@ -62,7 +63,7 @@ public class TajoCli {
// Current States
private String currentDatabase;
- private TajoCliOutputFormatter outputFormatter;
+ private TajoCliOutputFormatter displayFormatter;
private boolean wasError = false;
@@ -99,7 +100,10 @@ public class TajoCli {
options.addOption("help", "help", false, "help");
}
- public class TajoCliContext {
+ public class TajoCliContext extends OverridableConf {
+ public TajoCliContext(TajoConf conf) {
+ super(conf, ConfigKey.ConfigType.SESSION);
+ }
public TajoClient getTajoClient() {
return client;
@@ -121,18 +125,50 @@ public class TajoCli {
return conf;
}
- public void setVariable(String key, String value) {
- conf.set(key, value);
- try {
- initFormatter();
- } catch (Exception e) {
- System.err.println(ERROR_PREFIX + e.getMessage());
+ @VisibleForTesting
+ public String getCliSideVar(String key) {
+ if (SessionVars.exists(key)) {
+ ConfigKey configKey = SessionVars.get(key);
+ return get(configKey);
+ } else {
+ return get(key);
+ }
+ }
+
+ public void setCliSideVar(String key, String value) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(value);
+
+ boolean shouldReloadFormatter = false;
+
+ if (SessionVars.exists(key)) {
+ SessionVars configKey = SessionVars.get(key);
+ put(configKey, value);
+ shouldReloadFormatter = configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR;
+ } else {
+ set(key, value);
+
+ // It is hard to recognize it is a client side variable. So, we always reload formatter.
+ shouldReloadFormatter = true;
}
+
+ if (shouldReloadFormatter) {
+ try {
+ initFormatter();
+ } catch (Exception e) {
+ System.err.println(ERROR_PREFIX + e.getMessage());
+ }
+ }
+ }
+
+ public Map<String, TajoShellCommand> getCommands() {
+ return commands;
}
}
public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception {
this.conf = new TajoConf(c);
+ context = new TajoCliContext(conf);
this.sin = in;
this.reader = new ConsoleReader(sin, out);
this.reader.setExpandEvents(false);
@@ -161,13 +197,7 @@ public class TajoCli {
}
if (cmd.getOptionValues("conf") != null) {
- for (String eachParam: cmd.getOptionValues("conf")) {
- String[] tokens = eachParam.split("=");
- if (tokens.length != 2) {
- continue;
- }
- conf.set(tokens[0], tokens[1]);
- }
+ processConfVarCommand(cmd.getOptionValues("conf"));
}
// if there is no "-h" option,
@@ -196,19 +226,22 @@ public class TajoCli {
client = new TajoClient(conf, baseDatabase);
}
- context = new TajoCliContext();
context.setCurrentDatabase(client.getCurrentDatabase());
initHistory();
initCommands();
+ if (cmd.getOptionValues("conf") != null) {
+ processSessionVarCommand(cmd.getOptionValues("conf"));
+ }
+
if (cmd.hasOption("c")) {
- outputFormatter.setScirptMode();
+ displayFormatter.setScirptMode();
int exitCode = executeScript(cmd.getOptionValue("c"));
sout.flush();
System.exit(exitCode);
}
if (cmd.hasOption("f")) {
- outputFormatter.setScirptMode();
+ displayFormatter.setScirptMode();
cmd.getOptionValues("");
File sqlFile = new File(cmd.getOptionValue("f"));
if (sqlFile.exists()) {
@@ -226,13 +259,38 @@ public class TajoCli {
addShutdownHook();
}
+ private void processConfVarCommand(String[] confCommands) throws ServiceException {
+ for (String eachParam: confCommands) {
+ String[] tokens = eachParam.split("=");
+ if (tokens.length != 2) {
+ continue;
+ }
+
+ if (!SessionVars.exists(tokens[0])) {
+ conf.set(tokens[0], tokens[1]);
+ }
+ }
+ }
+
+ private void processSessionVarCommand(String[] confCommands) throws ServiceException {
+ for (String eachParam: confCommands) {
+ String[] tokens = eachParam.split("=");
+ if (tokens.length != 2) {
+ continue;
+ }
+
+ if (SessionVars.exists(tokens[0])) {
+ ((SetCommand)commands.get("\\set")).set(tokens[0], tokens[1]);
+ }
+ }
+ }
+
private void initFormatter() throws Exception {
- Class formatterClass = conf.getClass(ConfVars.CLI_OUTPUT_FORMATTER_CLASS.varname,
- DefaultTajoCliOutputFormatter.class);
- if (outputFormatter == null || !outputFormatter.getClass().equals(formatterClass)) {
- outputFormatter = (TajoCliOutputFormatter)formatterClass.newInstance();
+ Class formatterClass = context.getClass(SessionVars.CLI_FORMATTER_CLASS);
+ if (displayFormatter == null || !displayFormatter.getClass().equals(formatterClass)) {
+ displayFormatter = (TajoCliOutputFormatter)formatterClass.newInstance();
}
- outputFormatter.init(conf);
+ displayFormatter.init(context);
}
public TajoCliContext getContext() {
@@ -280,6 +338,9 @@ public class TajoCli {
throw new RuntimeException(e.getMessage());
}
commands.put(cmd.getCommand(), cmd);
+ for (String alias : cmd.getAliases()) {
+ commands.put(alias, cmd);
+ }
}
}
@@ -332,7 +393,7 @@ public class TajoCli {
exitCode = executeParsedResults(parsedResults);
currentPrompt = updatePrompt(parser.getState());
- if (exitCode != 0 && context.getConf().getBoolVar(ConfVars.CLI_ERROR_STOP)) {
+ if (exitCode != 0 && context.getBool(SessionVars.ON_ERROR_STOP)) {
return exitCode;
}
}
@@ -372,18 +433,18 @@ public class TajoCli {
try {
invoked.invoke(arguments);
} catch (IllegalArgumentException ige) {
- outputFormatter.printErrorMessage(sout, ige);
+ displayFormatter.printErrorMessage(sout, ige);
wasError = true;
return -1;
} catch (Exception e) {
- outputFormatter.printErrorMessage(sout, e);
+ displayFormatter.printErrorMessage(sout, e);
wasError = true;
return -1;
} finally {
context.getOutput().flush();
}
- if (wasError && context.getConf().getBoolVar(ConfVars.CLI_ERROR_STOP)) {
+ if (wasError && context.getBool(SessionVars.ON_ERROR_STOP)) {
break;
}
}
@@ -395,7 +456,7 @@ public class TajoCli {
long startTime = System.currentTimeMillis();
ClientProtos.SubmitQueryResponse response = client.executeQueryWithJson(json);
if (response == null) {
- outputFormatter.printErrorMessage(sout, "response is null");
+ displayFormatter.printErrorMessage(sout, "response is null");
wasError = true;
} else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
if (response.getIsForwarded()) {
@@ -403,7 +464,7 @@ public class TajoCli {
waitForQueryCompleted(queryId);
} else {
if (!response.hasTableDesc() && !response.hasResultSet()) {
- outputFormatter.printMessage(sout, "OK");
+ displayFormatter.printMessage(sout, "OK");
wasError = true;
} else {
localQueryCompleted(response, startTime);
@@ -411,7 +472,7 @@ public class TajoCli {
}
} else {
if (response.hasErrorMessage()) {
- outputFormatter.printErrorMessage(sout, response.getErrorMessage());
+ displayFormatter.printErrorMessage(sout, response.getErrorMessage());
wasError = true;
}
}
@@ -421,7 +482,7 @@ public class TajoCli {
long startTime = System.currentTimeMillis();
ClientProtos.SubmitQueryResponse response = client.executeQuery(statement);
if (response == null) {
- outputFormatter.printErrorMessage(sout, "response is null");
+ displayFormatter.printErrorMessage(sout, "response is null");
wasError = true;
} else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
if (response.getIsForwarded()) {
@@ -429,14 +490,14 @@ public class TajoCli {
waitForQueryCompleted(queryId);
} else {
if (!response.hasTableDesc() && !response.hasResultSet()) {
- outputFormatter.printMessage(sout, "OK");
+ displayFormatter.printMessage(sout, "OK");
} else {
localQueryCompleted(response, startTime);
}
}
} else {
if (response.hasErrorMessage()) {
- outputFormatter.printErrorMessage(sout, response.getErrorMessage());
+ displayFormatter.printErrorMessage(sout, response.getErrorMessage());
wasError = true;
}
}
@@ -454,13 +515,13 @@ public class TajoCli {
// non-forwarded INSERT INTO query does not have any query id.
// In this case, it just returns succeeded query information without printing the query results.
if (response.getMaxRowNum() < 0 && queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
- outputFormatter.printResult(sout, sin, desc, responseTime, res);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
} else {
res = TajoClient.createResultSet(client, response);
- outputFormatter.printResult(sout, sin, desc, responseTime, res);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
}
} catch (Throwable t) {
- outputFormatter.printErrorMessage(sout, t);
+ displayFormatter.printErrorMessage(sout, t);
wasError = true;
} finally {
if (res != null) {
@@ -495,7 +556,7 @@ public class TajoCli {
}
if (status.getState() == QueryState.QUERY_RUNNING || status.getState() == QueryState.QUERY_SUCCEEDED) {
- outputFormatter.printProgress(sout, status);
+ displayFormatter.printProgress(sout, status);
}
if (status.getState() != QueryState.QUERY_RUNNING &&
@@ -509,10 +570,10 @@ public class TajoCli {
}
if (status.getState() == QueryState.QUERY_ERROR || status.getState() == QueryState.QUERY_FAILED) {
- outputFormatter.printErrorMessage(sout, status);
+ displayFormatter.printErrorMessage(sout, status);
wasError = true;
} else if (status.getState() == QueryState.QUERY_KILLED) {
- outputFormatter.printKilledMessage(sout, queryId);
+ displayFormatter.printKilledMessage(sout, queryId);
wasError = true;
} else {
if (status.getState() == QueryState.QUERY_SUCCEEDED) {
@@ -521,15 +582,15 @@ public class TajoCli {
if (status.hasResult()) {
res = TajoClient.createResultSet(client, queryId, response);
TableDesc desc = new TableDesc(response.getTableDesc());
- outputFormatter.printResult(sout, sin, desc, responseTime, res);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
} else {
TableDesc desc = new TableDesc(response.getTableDesc());
- outputFormatter.printResult(sout, sin, desc, responseTime, res);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
}
}
}
} catch (Throwable t) {
- outputFormatter.printErrorMessage(sout, t);
+ displayFormatter.printErrorMessage(sout, t);
wasError = true;
} finally {
if (res != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java
index 0e91669..0ad89f2 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java
@@ -21,7 +21,6 @@ package org.apache.tajo.cli;
import org.apache.tajo.QueryId;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.client.QueryStatus;
-import org.apache.tajo.conf.TajoConf;
import java.io.InputStream;
import java.io.PrintWriter;
@@ -30,9 +29,9 @@ import java.sql.ResultSet;
public interface TajoCliOutputFormatter {
/**
* Initialize formatter
- * @param tajoConf
+ * @param context
*/
- public void init(TajoConf tajoConf);
+ public void init(TajoCli.TajoCliContext context);
/**
* print query result to console
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java
index 1ea2893..39f5377 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java
@@ -23,16 +23,24 @@ import org.apache.tajo.conf.TajoConf;
public abstract class TajoShellCommand {
public abstract String getCommand();
+ public String [] getAliases() {
+ return new String[] {};
+ }
public abstract void invoke(String [] command) throws Exception;
public abstract String getUsage();
public abstract String getDescription();
+ public void printHelp() {
+ context.getOutput().print(getCommand());
+ context.getOutput().print(" - ");
+ context.getOutput().println(getDescription());
+ }
protected TajoCli.TajoCliContext context;
protected TajoClient client;
protected int maxColumn;
public TajoShellCommand(TajoCli.TajoCliContext context) {
- maxColumn = context.getTajoClient().getConf().getIntVar(TajoConf.ConfVars.CLI_MAX_COLUMN);
+ maxColumn = context.getTajoClient().getConf().getIntVar(TajoConf.ConfVars.$CLI_MAX_COLUMN);
this.context = context;
client = context.getTajoClient();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java
new file mode 100644
index 0000000..b9d51ec
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+public interface ConfigKey {
+
+ // Client can set or change variables of this mode.
+ public static final int DEFAULT_MODE = 0;
+ // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables.
+ public static final int FROM_SHELL_ENV_MODE = 1;
+ // only TajoMaster is able to set and change variables of this mode.
+ public static final int SERVER_SIDE_VAR_MODE = 2;
+ // This type variable will be used only in cli side.
+ public static final int CLI_SIDE_VAR_MODE = 3;
+
+ public static enum ConfigType {
+ SYSTEM(""),
+ SESSION("$"),
+ QUERY("@"),
+ CLI("+");
+
+ private String prefix;
+
+ ConfigType(String prefix) {
+ this.prefix = prefix;
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+ }
+
+ public String keyname();
+
+ public ConfigType type();
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
new file mode 100644
index 0000000..220bd43
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.util.KeyValueSet;
+
+import static org.apache.tajo.ConfigKey.ConfigType;
+import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto;
+
+/**
+ * OverridableConf provides a consolidated config system. Tajo basically uses TajoConf, which is a extended class of
+ * Hadoop's Configuration system, However, TajoConf is only used for sharing static system configs, such as binding
+ * address of master and workers, system directories, other system parameters.
+ *
+ * For modifiable or instant configs, we use OverridableConf, which is a set of key-value pairs.
+ * OverridableConf provides more strong-typed way to set configs and its behavior is more clear than Configuration
+ * system.
+ *
+ * By default, OverridableConf recognizes following config types.
+ *
+ * <ul>
+ * <li>System Config - it comes from Hadoop's Configuration class. by tajo-site, catalog-site,
+ * catalog-default and TajoConf.</li>
+ * <li>Session variables - they are instantly configured by users.
+ * Each client session has it own set of session variables.</li>
+ * </ul>
+ *
+ * System configs and session variables can set the same config in the same time. System configs are usually used to set
+ * default configs, and session variables is user-specified configs. So, session variables can override system configs.
+ */
+public class OverridableConf extends KeyValueSet {
+ private static final Log LOG = LogFactory.getLog(OverridableConf.class);
+ private ConfigType [] configTypes;
+ private TajoConf conf;
+
+ public OverridableConf(final TajoConf conf, ConfigType...configTypes) {
+ this.conf = conf;
+ this.configTypes = configTypes;
+ }
+
+ public OverridableConf(final TajoConf conf, KeyValueSetProto proto, ConfigType...configTypes) {
+ super(proto);
+ this.conf = conf;
+ this.configTypes = configTypes;
+ }
+
+ public void setConf(TajoConf conf) {
+ this.conf = conf;
+ }
+
+ public TajoConf getConf() {
+ return conf;
+ }
+
+ public void setBool(ConfigKey key, boolean val) {
+ setBool(key.keyname(), val);
+ }
+
+ public boolean getBool(ConfigKey key, Boolean defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return getBool(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case QUERY:
+ return getBool(key.keyname());
+ case SESSION:
+ return getBool(key.keyname(), conf.getBoolVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getBoolVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public boolean getBool(ConfigKey key) {
+ return getBool(key, null);
+ }
+
+ public int getInt(ConfigKey key, Integer defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return getInt(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case SESSION:
+ return getInt(key.keyname(), conf.getIntVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getIntVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public int getInt(ConfigKey key) {
+ return getInt(key, null);
+ }
+
+ public long getLong(ConfigKey key, Long defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return getLong(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case SESSION:
+ return getLong(key.keyname(), conf.getLongVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getLongVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public long getLong(ConfigKey key) {
+ return getLong(key, null);
+ }
+
+ public float getFloat(ConfigKey key, Float defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return getFloat(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case SESSION:
+ return getFloat(key.keyname(), conf.getFloatVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getFloatVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public float getFloat(ConfigKey key) {
+ return getLong(key, null);
+ }
+
+ public void put(ConfigKey key, String val) {
+ set(key.keyname(), val);
+ }
+
+ private void assertRegisteredEnum(ConfigKey key) {
+ boolean registered = false;
+
+ for (ConfigType c : configTypes) {
+ registered = key.type() == c;
+ }
+
+ registered |= key.type() == ConfigType.SESSION || key.type() != ConfigType.SYSTEM;
+
+ Preconditions.checkArgument(registered, key.keyname() + " (" + key.type() + ") is not allowed in " +
+ getClass().getSimpleName());
+ }
+
+ public String get(ConfigKey key, String defaultVal) {
+ assertRegisteredEnum(key);
+
+ if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) {
+ return get(key.keyname(), defaultVal);
+ } else {
+ switch (key.type()) {
+ case SESSION:
+ return get(key.keyname(), conf.getVar(((SessionVars) key).getConfVars()));
+ case SYSTEM:
+ return conf.getVar((TajoConf.ConfVars) key);
+ default:
+ throw new IllegalStateException("key does not belong to Session and System config sets");
+ }
+ }
+ }
+
+ public String get(ConfigKey key) {
+ return get(key, null);
+ }
+
+ public Class<?> getClass(ConfigKey key) {
+ assertRegisteredEnum(key);
+
+ String className = getTrimmed(key);
+ try {
+ return Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getTrimmed(ConfigKey key) {
+ String value = get(key);
+
+ if (null == value) {
+ return null;
+ } else {
+ return value.trim();
+ }
+ }
+
+ public boolean containsKey(ConfigKey key) {
+ return containsKey(key.keyname());
+ }
+
+ public boolean equalKey(ConfigKey key, String another) {
+ if (containsKey(key)) {
+ return get(key).equals(another);
+ } else {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
new file mode 100644
index 0000000..5503aaa
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+import static org.apache.tajo.SessionVars.VariableMode.*;
+import static org.apache.tajo.conf.TajoConf.ConfVars;
+
+public enum SessionVars implements ConfigKey {
+
+ // Common Suffix Naming Rules:
+ //
+ // * LIMIT - We use the suffix 'LIMIT' if the variable is threshold. So, if some value is greater or less than
+ // the variable with suffix 'LIMIT', some action will be different from before.
+ // * SIZE - The suffix 'SIZE' means a data volume like bytes or mega bytes.
+ // It should be used for user's desired volume.
+ // * ENABLED - The suffix 'ENABLED' means a true or false value. If it is true, it will enable some feature.
+ // Otherwise, the feature will be turned off.
+
+
+ //-------------------------------------------------------------------------------
+ // Server Side Only Variables
+ //-------------------------------------------------------------------------------
+ SESSION_ID(ConfVars.$EMPTY, "session variable", SERVER_SIDE_VAR),
+ SESSION_LAST_ACCESS_TIME(ConfVars.$EMPTY, "last access time", SERVER_SIDE_VAR),
+
+ USERNAME(ConfVars.USERNAME, "username", SERVER_SIDE_VAR),
+ CLIENT_HOST(ConfVars.$EMPTY, "client hostname", SERVER_SIDE_VAR),
+
+ CURRENT_DATABASE(ConfVars.$EMPTY, "current database", SERVER_SIDE_VAR),
+
+ //-------------------------------------------------------------------------------
+ // Client Side Variables
+ //-------------------------------------------------------------------------------
+
+ // Client --------------------------------------------------------
+ SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "session expiry time (secs)", DEFAULT),
+
+ // Command line interface and its behavior --------------------------------
+ CLI_COLUMNS(ConfVars.$CLI_MAX_COLUMN, "Sets the width for the wrapped format", CLI_SIDE_VAR),
+ CLI_FORMATTER_CLASS(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, "Sets the output format class to display results",
+ CLI_SIDE_VAR),
+ CLI_NULL_CHAR(ConfVars.$CLI_NULL_CHAR, "Sets the string to be printed in place of a null value.", CLI_SIDE_VAR),
+
+ CLI_PAGE_ROWS(ConfVars.$CLI_PRINT_PAUSE_NUM_RECORDS, "Sets the number of rows for paging", CLI_SIDE_VAR),
+ CLI_PAGING_ENABLED(ConfVars.$CLI_PRINT_PAUSE, "Enable paging of result display", CLI_SIDE_VAR),
+ CLI_DISPLAY_ERROR_TRACE(ConfVars.$CLI_PRINT_ERROR_TRACE, "Enable display of error trace", CLI_SIDE_VAR),
+
+ ON_ERROR_STOP(ConfVars.$CLI_ERROR_STOP, "tsql will exist if an error occurs.", CLI_SIDE_VAR),
+
+ // Timezone & Date ----------------------------------------------------------
+ TZ(ConfVars.$TIMEZONE, "Sets timezone", FROM_SHELL_ENV),
+ DATE_ORDER(ConfVars.$DATE_ORDER, "date order (default is YMD)", FROM_SHELL_ENV),
+
+ // Locales and Character set ------------------------------------------------
+ // TODO - they are reserved variables, and we should support them.
+ LANG(ConfVars.$EMPTY, "Language", FROM_SHELL_ENV),
+ LC_ALL(ConfVars.$EMPTY, "String sort order", FROM_SHELL_ENV),
+ LC_COLLATE(ConfVars.$EMPTY, "String sort order", FROM_SHELL_ENV),
+ LC_CTYPE(ConfVars.$EMPTY, "Character classification (What is a letter? Its upper-case equivalent?)", FROM_SHELL_ENV),
+ LC_MESSAGES(ConfVars.$EMPTY, "Language of messages", FROM_SHELL_ENV),
+ LC_MONETARY(ConfVars.$EMPTY, "Formatting of currency amounts", FROM_SHELL_ENV),
+ LC_NUMERIC(ConfVars.$EMPTY, "Formatting of numbers", FROM_SHELL_ENV),
+ LC_TIME(ConfVars.$EMPTY, "Formatting of dates and times", FROM_SHELL_ENV),
+
+
+ // Query and Optimization ---------------------------------------------------
+
+ // for distributed query strategies
+ BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "limited size (bytes) of broadcast table",
+ DEFAULT),
+
+ JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT),
+ SORT_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_SORT_TASK_VOLUME, "sort task input size (mb)", DEFAULT),
+ GROUPBY_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_GROUPBY_TASK_VOLUME, "group by task input size (mb)", DEFAULT),
+
+ JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join (mb)", DEFAULT),
+ GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort (mb)", DEFAULT),
+ TABLE_PARTITION_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME,
+ "shuffle output size for partition table write (mb)", DEFAULT),
+
+ // for physical Executors
+ EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT),
+ HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT),
+ INNER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD,
+ "limited size for hash inner join (mb)", DEFAULT),
+ OUTER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash outer join (mb)",
+ DEFAULT),
+ HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "limited size for hash groupby (mb)",
+ DEFAULT),
+ MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT),
+
+ NULL_CHAR(ConfVars.$CSVFILE_NULL, "null char of text file output", DEFAULT),
+
+ // Behavior Control ---------------------------------------------------------
+ ARITHABORT(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT,
+ "If true, a running query will be terminated when an overflow or divide-by-zero occurs.", DEFAULT),
+
+ //-------------------------------------------------------------------------------
+ // Only for Unit Testing
+ //-------------------------------------------------------------------------------
+ DEBUG_ENABLED(ConfVars.$DEBUG_ENABLED, "(debug only) debug mode enabled", DEFAULT),
+ TEST_BROADCAST_JOIN_ENABLED(ConfVars.$TEST_BROADCAST_JOIN_ENABLED, "(test only) broadcast enabled", TEST_VAR),
+ TEST_JOIN_OPT_ENABLED(ConfVars.$TEST_JOIN_OPT_ENABLED, "(test only) join optimization enabled", TEST_VAR),
+ TEST_FILTER_PUSHDOWN_ENABLED(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED, "filter push down enabled", TEST_VAR),
+ TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", TEST_VAR),
+ ;
+
+ public static Map<String, SessionVars> SESSION_VARS = Maps.newHashMap();
+ public static Map<String, SessionVars> DEPRECATED_SESSION_VARS = Maps.newHashMap();
+
+ static {
+ for (SessionVars var : SessionVars.values()) {
+ SESSION_VARS.put(var.keyname(), var);
+ DEPRECATED_SESSION_VARS.put(var.getConfVars().keyname(), var);
+ }
+ }
+
+ private final ConfVars key;
+ private final String description;
+ private final VariableMode mode;
+
+ public static enum VariableMode {
+ DEFAULT, // Client can set or change variables of this mode..
+ FROM_SHELL_ENV, // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables.
+ SERVER_SIDE_VAR, // only TajoMaster is able to set and change variables of this mode.
+ CLI_SIDE_VAR, // This type variable is used in CLI.
+ TEST_VAR // Only used for unit tests
+ }
+
+ SessionVars(ConfVars key, String description, VariableMode mode) {
+ this.key = key;
+ this.description = description;
+ this.mode = mode;
+ }
+
+ public String keyname() {
+ return name();
+ }
+
+ public ConfigType type() {
+ return ConfigType.SESSION;
+ }
+
+ public ConfVars getConfVars() {
+ return key;
+ }
+
+ public Class<?> getVarType() {
+ return key.valClass;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public VariableMode getMode() {
+ return mode;
+ }
+
+ public static boolean exists(String keyname) {
+ return SESSION_VARS.containsKey(keyname) || DEPRECATED_SESSION_VARS.containsKey(keyname);
+ }
+
+ public static boolean isDeprecated(String keyname) {
+ return DEPRECATED_SESSION_VARS.containsKey(keyname);
+ }
+
+ public static boolean isPublic(SessionVars var) {
+ return var.getMode() != SERVER_SIDE_VAR;
+ }
+
+ public static SessionVars get(String keyname) {
+ if (SESSION_VARS.containsKey(keyname)) {
+ return SESSION_VARS.get(keyname);
+ } else if (DEPRECATED_SESSION_VARS.containsKey(keyname)) {
+ return DEPRECATED_SESSION_VARS.get(keyname);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * rename deprecated name to current name if the name is deprecated.
+ *
+ * @param keyname session variable name
+ * @return The current session variable name
+ */
+ public static String handleDeprecatedName(String keyname) {
+ return SessionVars.exists(keyname) ? SessionVars.get(keyname).keyname() : keyname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index b75530b..2504c23 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.ConfigKey;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.util.TUtil;
@@ -81,7 +82,7 @@ public class TajoConf extends Configuration {
try {
if (CURRENT_TIMEZONE == null) {
TajoConf tajoConf = new TajoConf();
- CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.TAJO_TIMEZONE));
+ CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.$TIMEZONE));
}
return CURRENT_TIMEZONE;
} finally {
@@ -105,7 +106,7 @@ public class TajoConf extends Configuration {
try {
if (DATE_ORDER < 0) {
TajoConf tajoConf = new TajoConf();
- String dateOrder = tajoConf.getVar(ConfVars.TAJO_DATE_ORDER);
+ String dateOrder = tajoConf.getVar(ConfVars.$DATE_ORDER);
if ("YMD".equals(dateOrder)) {
DATE_ORDER = DateTimeConstants.DATEORDER_YMD;
} else if ("DMY".equals(dateOrder)) {
@@ -133,11 +134,13 @@ public class TajoConf extends Configuration {
}
}
- public static enum ConfVars {
+ public static enum ConfVars implements ConfigKey {
- //////////////////////////////////
+ ///////////////////////////////////////////////////////////////////////////////////////
// Tajo System Configuration
- //////////////////////////////////
+ //
+ // They are all static configs which are not changed or not overwritten at all.
+ ///////////////////////////////////////////////////////////////////////////////////////
// a username for a running Tajo cluster
ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/"),
@@ -196,23 +199,19 @@ public class TajoConf extends Configuration {
// Catalog
CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005"),
- //////////////////////////////////
- // for Yarn Resource Manager
- //////////////////////////////////
+
+ // for Yarn Resource Manager ----------------------------------------------
+
/** how many launching TaskRunners in parallel */
YARN_RM_QUERY_MASTER_MEMORY_MB("tajo.querymaster.memory-mb", 512),
YARN_RM_QUERY_MASTER_DISKS("tajo.yarn-rm.querymaster.disks", 1),
YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM("tajo.yarn-rm.parallel-task-runner-launcher-num", 16),
YARN_RM_WORKER_NUMBER_PER_NODE("tajo.yarn-rm.max-worker-num-per-node", 8),
- //////////////////////////////////
// Query Configuration
- //////////////////////////////////
QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60),
- //////////////////////////////////
- // Shuffle Configuration
- //////////////////////////////////
+ // Shuffle Configuration --------------------------------------------------
PULLSERVER_PORT("tajo.pullserver.port", 0),
SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false),
SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"),
@@ -221,61 +220,28 @@ public class TajoConf extends Configuration {
SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 120),
SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 20),
- //////////////////////////////////
- // Storage Configuration
- //////////////////////////////////
+
+ // Storage Configuration --------------------------------------------------
ROWFILE_SYNC_INTERVAL("rowfile.sync.interval", 100),
MINIMUM_SPLIT_SIZE("tajo.min.split.size", (long) 1),
// for RCFile
HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true),
- // for Storage Manager v2
+ // for Storage Manager ----------------------------------------------------
STORAGE_MANAGER_VERSION_2("tajo.storage-manager.v2", false),
STORAGE_MANAGER_DISK_SCHEDULER_MAX_READ_BYTES_PER_SLOT("tajo.storage-manager.max-read-bytes", 8 * 1024 * 1024),
STORAGE_MANAGER_DISK_SCHEDULER_REPORT_INTERVAL("tajo.storage-manager.disk-scheduler.report-interval", 60 * 1000),
STORAGE_MANAGER_CONCURRENCY_PER_DISK("tajo.storage-manager.disk-scheduler.per-disk-concurrency", 2),
- //////////////////////////////////////////
- // Distributed Query Execution Parameters
- //////////////////////////////////////////
- DIST_QUERY_BROADCAST_JOIN_AUTO("tajo.dist-query.join.auto-broadcast", true),
- DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576),
-
- DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128),
- DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128),
- DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128),
-
- DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128),
- DIST_QUERY_SORT_PARTITION_VOLUME("tajo.dist-query.sort.partition-volume-mb", 256),
- DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256),
-
- DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256),
-
- //////////////////////////////////
- // Physical Executors
- //////////////////////////////////
- EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1),
- EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L),
- EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8),
-
- EXECUTOR_INNER_JOIN_INMEMORY_HASH_TABLE_SIZE("tajo.executor.join.inner.in-memory-table-num", (long)1000000),
- EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-bytes",
- (long)256 * 1048576),
- EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-bytes",
- (long)256 * 1048576),
- EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes",
- (long)256 * 1048576),
- //////////////////////////////////
- // RPC
- //////////////////////////////////
+ // RPC --------------------------------------------------------------------
RPC_POOL_MAX_IDLE("tajo.rpc.pool.idle.max", 10),
- //Internal RPC Client
+ // Internal RPC Client
INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM("tajo.internal.rpc.client.worker-thread-num",
Runtime.getRuntime().availableProcessors() * 2),
- //Internal RPC Server
+ // Internal RPC Server
MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num",
Runtime.getRuntime().availableProcessors() * 2),
QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.querymaster.rpc.server.worker-thread-num",
@@ -296,58 +262,92 @@ public class TajoConf extends Configuration {
WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM("tajo.worker.service.rpc.server.worker-thread-num",
Runtime.getRuntime().availableProcessors() * 1),
- //////////////////////////////////
- // The Below is reserved
- //////////////////////////////////
-
- // GeoIP
- GEOIP_DATA("tajo.function.geoip-database-location", ""),
-
- //////////////////////////////////
- // Task Configuration
+ // Task Configuration -----------------------------------------------------
TASK_DEFAULT_MEMORY("tajo.task.memory-slot-mb.default", 512),
TASK_DEFAULT_DISK("tajo.task.disk-slot.default", 0.5f),
TASK_DEFAULT_SIZE("tajo.task.size-mb", 128),
- //////////////////////////////////
- //////////////////////////////////
- // User Session Configuration
- //////////////////////////////////
- CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour.
+ // Query and Optimization -------------------------------------------------
+ EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1),
+ EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8),
+
+ EXECUTOR_INNER_JOIN_INMEMORY_HASH_TABLE_SIZE("tajo.executor.join.inner.in-memory-table-num", (long)1000000),
- // Metrics
+ // Metrics ----------------------------------------------------------------
METRICS_PROPERTY_FILENAME("tajo.metrics.property.file", "tajo-metrics.properties"),
- //CLI
- CLI_MAX_COLUMN("tajo.cli.max_columns", 120),
- CLI_PRINT_PAUSE_NUM_RECORDS("tajo.cli.print.pause.num.records", 100),
- CLI_PRINT_PAUSE("tajo.cli.print.pause", true),
- CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true),
- CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"),
- CLI_NULL_CHAR("tajo.cli.nullchar", ""),
- CLI_ERROR_STOP("tajo.cli.error.stop", false),
+ // Misc -------------------------------------------------------------------
- //TIME & DATE
- TAJO_TIMEZONE("tajo.timezone", System.getProperty("user.timezone")),
- TAJO_DATE_ORDER("tajo.date.order", "YMD"),
+ // Geo IP
+ GEOIP_DATA("tajo.function.geoip-database-location", ""),
- //PLANNER
- PLANNER_USE_FILTER_PUSHDOWN("tajo.planner.use.filter.pushdown", true),
+ /////////////////////////////////////////////////////////////////////////////////
+ // User Session Configuration
+ //
+ // All session variables begin with dollor($) sign. They are default configs
+ // for session variables. Do not directly use the following configs. Instead,
+ // please use QueryContext in order to access session variables.
+ //
+ // Also, users can change the default values of session variables in tajo-site.xml.
+ /////////////////////////////////////////////////////////////////////////////////
+
+
+ $EMPTY("tajo._", ""),
+
+ // Query and Optimization ---------------------------------------------------
+
+ // for distributed query strategies
+ $DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576),
+
+ $DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128),
+ $DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128),
+ $DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128),
+
+ $DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128),
+ $DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256),
+ $DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256),
+
+ // for physical Executors
+ $EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L),
+ $EXECUTOR_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.common.in-memory-hash-threshold-bytes",
+ (long)256 * 1048576),
+ $EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-bytes",
+ (long)256 * 1048576),
+ $EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-bytes",
+ (long)256 * 1048576),
+ $EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes",
+ (long)256 * 1048576),
+ $MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite
- // FILE FORMAT
- CSVFILE_NULL("tajo.csvfile.null", "\\\\N"),
- //OPTIMIZER
- OPTIMIZER_JOIN_ENABLE("tajo.optimizer.join.enable", true),
+ // Client -----------------------------------------------------------------
+ $CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour.
- // DEBUG OPTION
- TAJO_DEBUG("tajo.debug", false),
+ // Command line interface and its behavior --------------------------------
+ $CLI_MAX_COLUMN("tajo.cli.max_columns", 120),
+ $CLI_NULL_CHAR("tajo.cli.nullchar", ""),
+ $CLI_PRINT_PAUSE_NUM_RECORDS("tajo.cli.print.pause.num.records", 100),
+ $CLI_PRINT_PAUSE("tajo.cli.print.pause", true),
+ $CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true),
+ $CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"),
+ $CLI_ERROR_STOP("tajo.cli.error.stop", false),
- // ONLY FOR TESTCASE
- TESTCASE_MIN_TASK_NUM("tajo.testcase.min.task.num", -1),
+ // Timezone & Date ----------------------------------------------------------
+ $TIMEZONE("tajo.timezone", System.getProperty("user.timezone")),
+ $DATE_ORDER("tajo.date.order", "YMD"),
- // behavior control
- BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false);
+ // FILE FORMAT
+ $CSVFILE_NULL("tajo.csvfile.null", "\\\\N"),
+
+ // Only for Debug and Testing
+ $DEBUG_ENABLED("tajo.debug.enabled", false),
+ $TEST_BROADCAST_JOIN_ENABLED("tajo.dist-query.join.auto-broadcast", true),
+ $TEST_JOIN_OPT_ENABLED("tajo.test.plan.join-optimization.enabled", true),
+ $TEST_FILTER_PUSHDOWN_ENABLED("tajo.test.plan.filter-pushdown.enabled", true),
+ $TEST_MIN_TASK_NUM("tajo.test.min-task-num", -1),
+
+ // Behavior Control ---------------------------------------------------------
+ $BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false);
;
public final String varname;
@@ -429,6 +429,16 @@ public class TajoConf extends Configuration {
String typeString() { return name().toUpperCase();}
abstract void checkType(String value) throws Exception;
}
+
+ @Override
+ public String keyname() {
+ return varname;
+ }
+
+ @Override
+ public ConfigType type() {
+ return ConfigType.SYSTEM;
+ }
}
public static int getIntVar(Configuration conf, ConfVars var) {
@@ -450,8 +460,12 @@ public class TajoConf extends Configuration {
}
public static long getLongVar(Configuration conf, ConfVars var) {
- assert (var.valClass == Long.class);
- return conf.getLong(var.varname, var.defaultLongVal);
+ assert (var.valClass == Long.class || var.valClass == Integer.class);
+ if (var.valClass == Integer.class) {
+ return conf.getInt(var.varname, var.defaultIntVal);
+ } else {
+ return conf.getLong(var.varname, var.defaultLongVal);
+ }
}
public static long getLongVar(Configuration conf, ConfVars var, long defaultVal) {
@@ -516,7 +530,6 @@ public class TajoConf extends Configuration {
}
public static String getVar(Configuration conf, ConfVars var) {
- assert (var.valClass == String.class);
return conf.get(var.varname, var.defaultVal);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
index 19acafc..874004b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
@@ -114,7 +114,7 @@ public abstract class Datum implements Comparable<Datum>, GsonObject {
}
protected static void initAbortWhenDivideByZero(TajoConf tajoConf) {
- abortWhenDivideByZero = tajoConf.getBoolVar(ConfVars.BEHAVIOR_ARITHMETIC_ABORT);
+ abortWhenDivideByZero = tajoConf.getBoolVar(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT);
}
public abstract int size();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
index a4f79d7..532e7cd 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java
@@ -25,6 +25,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type;
public class NullDatum extends Datum {
private static NullDatum instance;
+ public static final String DEFAULT_TEXT = "";
private static final byte [] EMPTY_BYTES = new byte[0];
private static final DataType NULL_DATA_TYPE;
[10/17] git commit: TAJO-994: 'count(distinct x)' function counts
first null value.
Posted by ji...@apache.org.
TAJO-994: 'count(distinct x)' function counts first null value.
Closes #106
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fcc5da03
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fcc5da03
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fcc5da03
Branch: refs/heads/index_support
Commit: fcc5da03ad1e9a692efad20aaf90a4005f9084d0
Parents: 189cf3f
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 11 11:22:59 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 11 11:22:59 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../function/builtin/CountValueDistinct.java | 4 +-
.../tajo/engine/query/TestGroupByQuery.java | 46 ++++++++++++++------
.../testDistinctAggregationCaseByCase3.sql | 8 ++++
.../testDistinctAggregationCaseByCase4.sql | 7 +++
.../testDistinctAggregationCaseByCase3.result | 3 ++
.../testDistinctAggregationCaseByCase4.result | 3 ++
7 files changed, 57 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc5da03/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 7487983..864eaef 100644
--- a/CHANGES
+++ b/CHANGES
@@ -106,6 +106,8 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-994: 'count(distinct x)' function counts first null value. (hyunsik)
+
TAJO-996: Sometimes, scheduleFetchesByEvenDistributedVolumes loses
some FetchImpls. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc5da03/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java
index c0f5ec3..7a7f3d8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java
@@ -24,7 +24,6 @@ import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.Int8Datum;
-import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.function.FunctionContext;
import org.apache.tajo.engine.function.annotation.Description;
import org.apache.tajo.engine.function.annotation.ParamTypes;
@@ -57,7 +56,8 @@ public final class CountValueDistinct extends CountRows {
public void merge(FunctionContext context, Tuple part) {
CountDistinctValueContext distinctContext = (CountDistinctValueContext) context;
Datum value = part.get(0);
- if ((distinctContext.latest == null || (!distinctContext.latest.equals(value)) && !(value instanceof NullDatum))) {
+
+ if (!value.isNull() && (distinctContext.latest == null || (!distinctContext.latest.equals(value)))) {
distinctContext.latest = value;
distinctContext.count++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc5da03/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
index 79efd92..fe9f990 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
@@ -342,7 +342,7 @@ public class TestGroupByQuery extends QueryTestCaseBase {
}
@Test
- public final void testDistinctAggregationCasebyCase2() throws Exception {
+ public final void testDistinctAggregationCaseByCase3() throws Exception {
// first distinct is smaller than second distinct.
KeyValueSet tableOptions = new KeyValueSet();
tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
@@ -364,22 +364,40 @@ public class TestGroupByQuery extends QueryTestCaseBase {
TajoTestingCluster.createTable("table10", schema, tableOptions, data);
- ResultSet res = executeString(
- "select col1 \n" +
- ",count(distinct col2) as cnt1\n" +
- ",count(distinct case when col3 is not null then col2 else null end) as cnt2\n" +
- "from table10 \n" +
- "group by col1"
- );
- String result = resultSetToString(res);
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
- String expected = "col1,cnt1,cnt2\n" +
- "-------------------------------\n" +
- "a,3,1\n";
+ executeString("DROP TABLE table10 PURGE").close();
+ }
- assertEquals(expected, result);
+ @Test
+ public final void testDistinctAggregationCaseByCase4() throws Exception {
+ // Reproduction case for TAJO-994
+ KeyValueSet tableOptions = new KeyValueSet();
+ tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
- executeString("DROP TABLE table10 PURGE").close();
+ Schema schema = new Schema();
+ schema.addColumn("col1", Type.TEXT);
+ schema.addColumn("col2", Type.TEXT);
+
+ String[] data = new String[]{
+ "a|\\N",
+ "a|\\N|",
+ "a|\\N|",
+ "a|\\N|",
+ "a|\\N|",
+ "a|\\N|"
+ };
+
+ TajoTestingCluster.createTable("table11", schema, tableOptions, data);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+
+ executeString("DROP TABLE table11 PURGE").close();
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc5da03/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregationCaseByCase3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregationCaseByCase3.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregationCaseByCase3.sql
new file mode 100644
index 0000000..925cadd
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregationCaseByCase3.sql
@@ -0,0 +1,8 @@
+select
+ col1,
+ count(distinct col2) as cnt1,
+ count(distinct case when col3 is not null then col2 else null end) as cnt2
+from
+ table10
+group by
+ col1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc5da03/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregationCaseByCase4.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregationCaseByCase4.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregationCaseByCase4.sql
new file mode 100644
index 0000000..352f21e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testDistinctAggregationCaseByCase4.sql
@@ -0,0 +1,7 @@
+select
+ col1,
+ count(distinct col2) as cnt
+from
+ table11
+group by
+ col1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc5da03/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregationCaseByCase3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregationCaseByCase3.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregationCaseByCase3.result
new file mode 100644
index 0000000..320ab29
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregationCaseByCase3.result
@@ -0,0 +1,3 @@
+col1,cnt1,cnt2
+-------------------------------
+a,3,0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/fcc5da03/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregationCaseByCase4.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregationCaseByCase4.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregationCaseByCase4.result
new file mode 100644
index 0000000..a65e71f
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testDistinctAggregationCaseByCase4.result
@@ -0,0 +1,3 @@
+col1,cnt
+-------------------------------
+a,0
\ No newline at end of file
[12/17] TAJO-928: Session variables should override query configs in
TajoConf.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
index b59c495..9501556 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
@@ -49,39 +49,39 @@ public class TestJoinQuery extends QueryTestCaseBase {
public TestJoinQuery(String joinOption) {
super(TajoConstants.DEFAULT_DATABASE_NAME);
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname,
- ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.defaultVal);
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname,
- ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname,
+ ConfVars.$TEST_BROADCAST_JOIN_ENABLED.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname,
+ ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal);
testingCluster.setAllTajoDaemonConfValue(
- ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.varname,
- ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.defaultVal);
+ ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname,
+ ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal);
- testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.varname,
- ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.defaultVal);
- testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
- ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname,
+ ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
+ ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal);
if (joinOption.indexOf("NoBroadcast") >= 0) {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "false");
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "-1");
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "-1");
}
if (joinOption.indexOf("Hash") >= 0) {
testingCluster.setAllTajoDaemonConfValue(
- ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.varname, String.valueOf(256 * 1048576));
- testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.varname,
+ ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, String.valueOf(256 * 1048576));
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname,
String.valueOf(256 * 1048576));
- testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
String.valueOf(256 * 1048576));
}
if (joinOption.indexOf("Sort") >= 0) {
testingCluster.setAllTajoDaemonConfValue(
- ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.varname, String.valueOf(1));
- testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.varname,
+ ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, String.valueOf(1));
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname,
String.valueOf(1));
- testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
String.valueOf(1));
}
}
@@ -98,19 +98,19 @@ public class TestJoinQuery extends QueryTestCaseBase {
@AfterClass
public static void classTearDown() {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname,
- ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.defaultVal);
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname,
- ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname,
+ ConfVars.$TEST_BROADCAST_JOIN_ENABLED.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname,
+ ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal);
testingCluster.setAllTajoDaemonConfValue(
- ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.varname,
- ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.defaultVal);
+ ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname,
+ ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal);
- testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.varname,
- ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.defaultVal);
- testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
- ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname,
+ ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
+ ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal);
}
@Test
@@ -392,8 +392,8 @@ public class TestJoinQuery extends QueryTestCaseBase {
public final void testLeftOuterJoinWithEmptySubquery1() throws Exception {
// Empty Null Supplying table
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
@@ -405,7 +405,7 @@ public class TestJoinQuery extends QueryTestCaseBase {
TajoTestingCluster.createTable("table12", schema, tableOptions, data, 2);
try {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.TESTCASE_MIN_TASK_NUM.varname, "2");
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2");
ResultSet res = executeString("select a.id, b.id from table11 a " +
"left outer join (" +
@@ -423,8 +423,8 @@ public class TestJoinQuery extends QueryTestCaseBase {
assertEquals(expected, resultSetToString(res));
cleanupQuery(res);
} finally {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.TESTCASE_MIN_TASK_NUM.varname,
- ConfVars.TESTCASE_MIN_TASK_NUM.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname,
+ ConfVars.$TEST_MIN_TASK_NUM.defaultVal);
executeString("DROP TABLE table11 PURGE").close();
executeString("DROP TABLE table12 PURGE").close();
}
@@ -434,8 +434,8 @@ public class TestJoinQuery extends QueryTestCaseBase {
public final void testLeftOuterJoinWithEmptySubquery2() throws Exception {
//Empty Preserved Row table
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
@@ -447,7 +447,7 @@ public class TestJoinQuery extends QueryTestCaseBase {
TajoTestingCluster.createTable("table12", schema, tableOptions, data, 2);
try {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.TESTCASE_MIN_TASK_NUM.varname, "2");
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2");
ResultSet res = executeString("select a.id, b.id from " +
"(select table12.id, table12.name, lineitem.l_shipdate " +
@@ -461,8 +461,8 @@ public class TestJoinQuery extends QueryTestCaseBase {
assertEquals(expected, resultSetToString(res));
cleanupQuery(res);
} finally {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.TESTCASE_MIN_TASK_NUM.varname,
- ConfVars.TESTCASE_MIN_TASK_NUM.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname,
+ ConfVars.$TEST_MIN_TASK_NUM.defaultVal);
executeString("DROP TABLE table11 PURGE");
executeString("DROP TABLE table12 PURGE");
}
@@ -923,8 +923,8 @@ public class TestJoinQuery extends QueryTestCaseBase {
private void createOuterJoinTestTable() throws Exception {
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
@@ -1057,8 +1057,8 @@ public class TestJoinQuery extends QueryTestCaseBase {
@Test
public void testJoinWithDifferentShuffleKey() throws Exception {
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
@@ -1077,8 +1077,8 @@ public class TestJoinQuery extends QueryTestCaseBase {
}
TajoTestingCluster.createTable("large_table", schema, tableOptions, data.toArray(new String[]{}));
- int originConfValue = conf.getIntVar(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME);
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME.varname, "1");
+ int originConfValue = conf.getIntVar(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME.varname, "1");
ResultSet res = executeString(
"select count(b.id) " +
"from (select id, count(*) as cnt from large_table group by id) a " +
@@ -1094,7 +1094,7 @@ public class TestJoinQuery extends QueryTestCaseBase {
assertEquals(expected, resultSetToString(res));
} finally {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME.varname, "" + originConfValue);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME.varname, "" + originConfValue);
cleanupQuery(res);
executeString("DROP TABLE large_table PURGE").close();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
index f41b0ab..de75ca7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
@@ -53,7 +53,7 @@ public class TestNullValues {
"3|filled|0.2"
};
KeyValueSet opts = new KeyValueSet();
- opts.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ opts.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable1 where col3 is null");
@@ -80,7 +80,7 @@ public class TestNullValues {
"3|filled|"
};
KeyValueSet opts = new KeyValueSet();
- opts.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ opts.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable2 where col1 is not null");
@@ -115,7 +115,7 @@ public class TestNullValues {
",,,43578,19,13,6,3581,2557,1024"
};
KeyValueSet opts = new KeyValueSet();
- opts.put(StorageConstants.CSVFILE_DELIMITER, ",");
+ opts.set(StorageConstants.CSVFILE_DELIMITER, ",");
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable3 where col1 is null and col2 is null and col3 is null and col4 = 43578");
@@ -148,8 +148,8 @@ public class TestNullValues {
",\\N,,43578"
};
KeyValueSet opts = new KeyValueSet();
- opts.put(StorageConstants.CSVFILE_DELIMITER, ",");
- opts.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ opts.set(StorageConstants.CSVFILE_DELIMITER, ",");
+ opts.set(StorageConstants.CSVFILE_NULL, "\\\\N");
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable4 where col1 is null and col2 is null and col3 is null and col5 is null and col4 = 43578");
@@ -253,8 +253,8 @@ public class TestNullValues {
"4|d|4.0|\\N"
};
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
ResultSet res = TajoTestingCluster
.run(table, schemas, tableOptions, new String[][]{data}, query);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index 50d274d..5528b21 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -432,8 +432,8 @@ public class TestSelectQuery extends QueryTestCaseBase {
@Test
public final void testNowInMultipleTasks() throws Exception {
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
@@ -442,7 +442,7 @@ public class TestSelectQuery extends QueryTestCaseBase {
TajoTestingCluster.createTable("table11", schema, tableOptions, data, 2);
try {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.TESTCASE_MIN_TASK_NUM.varname, "2");
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2");
ResultSet res = executeString("select concat(substr(to_char(now(),'yyyymmddhh24miss'), 1, 14), 'aaa'), sleep(1) from table11");
@@ -474,8 +474,8 @@ public class TestSelectQuery extends QueryTestCaseBase {
}
assertEquals(5, numRecords);
} finally {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.TESTCASE_MIN_TASK_NUM.varname,
- ConfVars.TESTCASE_MIN_TASK_NUM.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname,
+ ConfVars.$TEST_MIN_TASK_NUM.defaultVal);
executeString("DROP TABLE table11 PURGE");
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
index a520e56..c7f6406 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
@@ -181,10 +181,10 @@ public class TestSortQuery extends QueryTestCaseBase {
@Test
public final void testSortNullColumn() throws Exception {
try {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.TESTCASE_MIN_TASK_NUM.varname, "2");
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2");
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- tableOptions.put(StorageConstants.CSVFILE_NULL, "\\\\N");
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N");
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
@@ -214,7 +214,7 @@ public class TestSortQuery extends QueryTestCaseBase {
cleanupQuery(res);
} finally {
- testingCluster.setAllTajoDaemonConfValue(ConfVars.TESTCASE_MIN_TASK_NUM.varname, "0");
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "0");
executeString("DROP TABLE nullsort PURGE;").close();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
index 7ca94c9..38908db 100644
--- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
+++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
@@ -153,7 +153,7 @@ public class TestResultSet {
"2014-01-01|01:00:00|2014-01-01 01:00:00"
};
KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
res = TajoTestingCluster
.run(table, schemas, tableOptions, new String[][]{data}, query);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
index 4b76cb8..6bbf63b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
@@ -61,7 +61,7 @@ public class TestExecutionBlockCursor {
util.startCatalogCluster();
conf = util.getConfiguration();
- conf.set(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "false");
+ conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
catalog = util.getMiniCatalogCluster().getCatalog();
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:!234/warehouse");
@@ -107,9 +107,9 @@ public class TestExecutionBlockCursor {
"join supplier on s_nationkey = n_nationkey " +
"join partsupp on s_suppkey = ps_suppkey " +
"join part on p_partkey = ps_partkey and p_type like '%BRASS' and p_size = 15");
- LogicalPlan logicalPlan = logicalPlanner.createPlan(LocalTajoTestingUtility.createDummySession(), context);
+ LogicalPlan logicalPlan = logicalPlanner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context);
optimizer.optimize(logicalPlan);
- QueryContext queryContext = new QueryContext();
+ QueryContext queryContext = new QueryContext(conf);
MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), queryContext, logicalPlan);
planner.build(plan);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
index 0ce7746..b6ac551 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
@@ -112,9 +112,9 @@ public class TestGlobalPlanner {
private MasterPlan buildPlan(String sql) throws PlanningException, IOException {
Expr expr = sqlAnalyzer.parse(sql);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(util.getConfiguration()), expr);
optimizer.optimize(plan);
- QueryContext context = new QueryContext();
+ QueryContext context = new QueryContext(util.getConfiguration());
MasterPlan masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), context, plan);
globalPlanner.build(masterPlan);
return masterPlan;
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
index e340953..b4be00b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
@@ -44,7 +44,7 @@ public class TestQueryUnitStatusUpdate extends QueryTestCaseBase {
@BeforeClass
public static void setUp() throws Exception {
- conf.set(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "false");
+ conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
index 5375749..5f8efe7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
@@ -143,12 +143,12 @@ public class TestRangeRetrieverHandler {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tableDir, Integer.MAX_VALUE);
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(),
new FileFragment[] {frags[0]}, testDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(SORT_QUERY[0]);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
@@ -267,12 +267,12 @@ public class TestRangeRetrieverHandler {
FileFragment[] frags = sm.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE);
TaskAttemptContext
- ctx = new TaskAttemptContext(conf, new QueryContext(),
+ ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(),
new FileFragment[] {frags[0]}, testDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(SORT_QUERY[1]);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr);
+ LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
new file mode 100644
index 0000000..c294407
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
@@ -0,0 +1,36 @@
+Available Session Variables:
+
+\set SESSION_EXPIRY_TIME [int value] - session expiry time (secs)
+\set CLI_COLUMNS [int value] - Sets the width for the wrapped format
+\set CLI_FORMATTER_CLASS [text value] - Sets the output format class to display results
+\set CLI_NULL_CHAR [text value] - Sets the string to be printed in place of a null value.
+\set CLI_PAGE_ROWS [int value] - Sets the number of rows for paging
+\set CLI_PAGING_ENABLED [true or false] - Enable paging of result display
+\set CLI_DISPLAY_ERROR_TRACE [true or false] - Enable display of error trace
+\set ON_ERROR_STOP [true or false] - tsql will exist if an error occurs.
+\set TZ [text value] - Sets timezone
+\set DATE_ORDER [text value] - date order (default is YMD)
+\set LANG [text value] - Language
+\set LC_ALL [text value] - String sort order
+\set LC_COLLATE [text value] - String sort order
+\set LC_CTYPE [text value] - Character classification (What is a letter? Its upper-case equivalent?)
+\set LC_MESSAGES [text value] - Language of messages
+\set LC_MONETARY [text value] - Formatting of currency amounts
+\set LC_NUMERIC [text value] - Formatting of numbers
+\set LC_TIME [text value] - Formatting of dates and times
+\set BROADCAST_TABLE_SIZE_LIMIT [long value] - limited size (bytes) of broadcast table
+\set JOIN_TASK_INPUT_SIZE [int value] - join task input size (mb)
+\set SORT_TASK_INPUT_SIZE [int value] - sort task input size (mb)
+\set GROUPBY_TASK_INPUT_SIZE [int value] - group by task input size (mb)
+\set JOIN_PER_SHUFFLE_SIZE [int value] - shuffle output size for join (mb)
+\set GROUPBY_PER_SHUFFLE_SIZE [int value] - shuffle output size for sort (mb)
+\set TABLE_PARTITION_PER_SHUFFLE_SIZE [int value] - shuffle output size for partition table write (mb)
+\set EXTSORT_BUFFER_SIZE [long value] - sort buffer size for external sort (mb)
+\set HASH_JOIN_SIZE_LIMIT [long value] - limited size for hash join (mb)
+\set INNER_HASH_JOIN_SIZE_LIMIT [long value] - limited size for hash inner join (mb)
+\set OUTER_HASH_JOIN_SIZE_LIMIT [long value] - limited size for hash outer join (mb)
+\set HASH_GROUPBY_SIZE_LIMIT [long value] - limited size for hash groupby (mb)
+\set MAX_OUTPUT_FILE_SIZE [int value] - Maximum per-output file size (mb). 0 means infinite.
+\set NULL_CHAR [text value] - null char of text file output
+\set ARITHABORT [true or false] - If true, a running query will be terminated when an overflow or divide-by-zero occurs.
+\set DEBUG_ENABLED [true or false] - (debug only) debug mode enabled
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrueDeprecated.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrueDeprecated.result b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrueDeprecated.result
new file mode 100644
index 0000000..36ea548
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrueDeprecated.result
@@ -0,0 +1,9 @@
+Warning: deprecated to directly use config key in TajoConf.ConfVars. Please execute '\help set'.
+c_custkey, o_orderkey, o_orderstatus
+-------------------------------
+1, 1, O
+2, 2, O
+3, 3, F
+4, testnull, testnull
+5, testnull, testnull
+(5 rows, , 30 B selected)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-core/src/test/resources/results/TestTajoCli/testStopWhenErrorDeprecated.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testStopWhenErrorDeprecated.result b/tajo-core/src/test/resources/results/TestTajoCli/testStopWhenErrorDeprecated.result
new file mode 100644
index 0000000..5c49139
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testStopWhenErrorDeprecated.result
@@ -0,0 +1,6 @@
+Warning: deprecated to directly use config key in TajoConf.ConfVars. Please execute '\help set'.
+?count
+-------------------------------
+5
+(1 rows, , 2 B selected)
+ERROR: relation "default.lineitem2" does not exist
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java
index ed6ea34..5b42cbd 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java
@@ -30,7 +30,7 @@ public interface Appender extends Closeable {
void addTuple(Tuple t) throws IOException;
void flush() throws IOException;
-
+
void close() throws IOException;
void enableStats();
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index 8e26ec6..2113794 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -84,7 +84,9 @@ public class CSVFile {
this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_DELIMITER,
StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
this.columnNum = schema.size();
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_NULL));
+
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_NULL,
+ NullDatum.DEFAULT_TEXT));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
@@ -107,8 +109,8 @@ public class CSVFile {
isShuffle = false;
}
- String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
- if(!StringUtils.isEmpty(codecName)){
+ if(this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) {
+ String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
codecFactory = new CompressionCodecFactory(conf);
codec = codecFactory.getCodecByClassName(codecName);
compressor = CodecPool.getCompressor(codec);
@@ -262,7 +264,8 @@ public class CSVFile {
String delim = meta.getOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0);
- String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.CSVFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.CSVFILE_NULL,
+ NullDatum.DEFAULT_TEXT));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 5b2d711..07fa16b 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -114,18 +114,18 @@ public class StorageUtil extends StorageConstants {
public static KeyValueSet newPhysicalProperties(CatalogProtos.StoreType type) {
KeyValueSet options = new KeyValueSet();
if (CatalogProtos.StoreType.CSV == type) {
- options.put(CSVFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
+ options.set(CSVFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
} else if (CatalogProtos.StoreType.RCFILE == type) {
- options.put(RCFILE_SERDE, DEFAULT_BINARY_SERDE);
+ options.set(RCFILE_SERDE, DEFAULT_BINARY_SERDE);
} else if (CatalogProtos.StoreType.SEQUENCEFILE == type) {
- options.put(SEQUENCEFILE_SERDE, DEFAULT_TEXT_SERDE);
- options.put(SEQUENCEFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
+ options.set(SEQUENCEFILE_SERDE, DEFAULT_TEXT_SERDE);
+ options.set(SEQUENCEFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
} else if (type == CatalogProtos.StoreType.PARQUET) {
- options.put(ParquetOutputFormat.BLOCK_SIZE, PARQUET_DEFAULT_BLOCK_SIZE);
- options.put(ParquetOutputFormat.PAGE_SIZE, PARQUET_DEFAULT_PAGE_SIZE);
- options.put(ParquetOutputFormat.COMPRESSION, PARQUET_DEFAULT_COMPRESSION_CODEC_NAME);
- options.put(ParquetOutputFormat.ENABLE_DICTIONARY, PARQUET_DEFAULT_IS_DICTIONARY_ENABLED);
- options.put(ParquetOutputFormat.VALIDATION, PARQUET_DEFAULT_IS_VALIDATION_ENABLED);
+ options.set(ParquetOutputFormat.BLOCK_SIZE, PARQUET_DEFAULT_BLOCK_SIZE);
+ options.set(ParquetOutputFormat.PAGE_SIZE, PARQUET_DEFAULT_PAGE_SIZE);
+ options.set(ParquetOutputFormat.COMPRESSION, PARQUET_DEFAULT_COMPRESSION_CODEC_NAME);
+ options.set(ParquetOutputFormat.ENABLE_DICTIONARY, PARQUET_DEFAULT_IS_DICTIONARY_ENABLED);
+ options.set(ParquetOutputFormat.VALIDATION, PARQUET_DEFAULT_IS_VALIDATION_ENABLED);
}
return options;
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java
index 962c63d..c15d20b 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java
@@ -31,15 +31,20 @@ import org.apache.tajo.catalog.TableMeta;
public class AvroUtil {
public static Schema getAvroSchema(TableMeta meta, Configuration conf)
throws IOException {
- String schemaLiteral = meta.getOption(StorageConstants.AVRO_SCHEMA_LITERAL);
- String schemaUrl = meta.getOption(StorageConstants.AVRO_SCHEMA_URL);
- if (schemaLiteral == null && schemaUrl == null) {
+
+
+ boolean isSchemaLiteral = meta.containsOption(StorageConstants.AVRO_SCHEMA_LITERAL);
+ boolean isSchemaUrl = meta.containsOption(StorageConstants.AVRO_SCHEMA_URL);
+ if (!isSchemaLiteral && !isSchemaUrl) {
throw new RuntimeException("No Avro schema for table.");
}
- if (schemaLiteral != null) {
- return new Schema.Parser().parse(schemaLiteral);
+ if (isSchemaLiteral) {
+ String schema = meta.getOption(StorageConstants.AVRO_SCHEMA_LITERAL);
+ return new Schema.Parser().parse(schema);
}
- Path schemaPath = new Path(schemaUrl);
+
+ String schemaURL = meta.getOption(StorageConstants.AVRO_SCHEMA_URL);
+ Path schemaPath = new Path(schemaURL);
FileSystem fs = schemaPath.getFileSystem(conf);
FSDataInputStream inputStream = fs.open(schemaPath);
return new Schema.Parser().parse(inputStream);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
index 78498c7..e5507ad 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
@@ -733,8 +733,8 @@ public class RCFile {
isShuffle = false;
}
- String codecClassname = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
- if (!StringUtils.isEmpty(codecClassname)) {
+ if (this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) {
+ String codecClassname = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
try {
Class<? extends CompressionCodec> codecClass = conf.getClassByName(
codecClassname).asSubclass(CompressionCodec.class);
@@ -745,7 +745,8 @@ public class RCFile {
}
}
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.RCFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.RCFILE_NULL,
+ NullDatum.DEFAULT_TEXT));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
@@ -1193,7 +1194,8 @@ public class RCFile {
rowId = new LongWritable();
readBytes = 0;
- String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.RCFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.RCFILE_NULL,
+ NullDatum.DEFAULT_TEXT));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
index 86d902a..f5cef62 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
@@ -97,7 +97,8 @@ public class SequenceFileAppender extends FileAppender {
this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_DELIMITER,
StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
this.columnNum = schema.size();
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL,
+ NullDatum.DEFAULT_TEXT));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
@@ -108,8 +109,8 @@ public class SequenceFileAppender extends FileAppender {
throw new FileNotFoundException(path.toString());
}
- String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
- if(!StringUtils.isEmpty(codecName)){
+ if(this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) {
+ String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
codecFactory = new CompressionCodecFactory(conf);
codec = codecFactory.getCodecByClassName(codecName);
} else {
@@ -119,7 +120,8 @@ public class SequenceFileAppender extends FileAppender {
}
try {
- String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
+ String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE,
+ TextSerializerDeserializer.class.getName());
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
index 3c39841..b0ef67d 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
@@ -25,7 +25,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.*;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -35,7 +34,6 @@ import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.BytesUtils;
-import org.apache.tajo.util.ReflectionUtil;
import java.io.IOException;
@@ -86,7 +84,8 @@ public class SequenceFileScanner extends FileScanner {
reader = new SequenceFile.Reader(fs, fragment.getPath(), conf);
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL,
+ NullDatum.DEFAULT_TEXT));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
index 5d2f861..c8e19dd 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
@@ -70,7 +70,7 @@ public class RCFileScanner extends FileScannerV2 {
key = new LongWritable();
column = new BytesRefArrayWritable();
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(NULL, NullDatum.DEFAULT_TEXT));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
[05/17] git commit: TAJO-975: alias name which is the same to
existing column name may cause NPE during PPD.
Posted by ji...@apache.org.
TAJO-975: alias name which is the same to existing column name may cause NPE during PPD.
Closes #92
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ae384685
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ae384685
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ae384685
Branch: refs/heads/index_support
Commit: ae384685f5c16853d47cc4a9a611729dfe2b14fc
Parents: 6b16264
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 4 15:33:43 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 4 15:33:43 2014 +0900
----------------------------------------------------------------------
CHANGES | 5 ++++-
.../planner/rewrite/ProjectionPushDownRule.java | 10 ++++++++++
.../tajo/engine/query/TestSelectQuery.java | 9 +++++++++
.../testSelectColumnAliasExistingInRelation3.sql | 19 +++++++++++++++++++
...stSelectColumnAliasExistingInRelation3.result | 11 +++++++++++
5 files changed, 53 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/ae384685/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index f98da28..61593a2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -103,7 +103,10 @@ Release 0.9.0 - unreleased
BUG FIXES
- TAJO-979: Dividing float value by zero should throw "Divide by zero
+ TAJO-975: alias name which is the same to existing column name may cause
+ NPE during PPD. (hyunsik)
+
+ TAJO-979: Dividing float value by zero should throw "Divide by zero
Exception" (Hyoungjun Kim via hyunsik)
TAJO-978: RoundFloat8 should return Float8Datum type. (Hyoungjun Kim via
http://git-wip-us.apache.org/repos/asf/tajo/blob/ae384685/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
index e5a329d..4a12d99 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
@@ -191,6 +191,7 @@ public class ProjectionPushDownRule extends
// if a name already exists, it only just keeps an actual
// expression instead of a column reference.
if (nameToIdBiMap.containsKey(specifiedName)) {
+
int refId = nameToIdBiMap.get(specifiedName);
EvalNode found = idToEvalBiMap.get(refId);
if (found != null) {
@@ -204,7 +205,16 @@ public class ProjectionPushDownRule extends
}
if (found.getType() == EvalType.FIELD) {
+ Integer daggling = idToEvalBiMap.inverse().get(evalNode);
idToEvalBiMap.forcePut(refId, evalNode);
+ if (daggling != null) {
+ String name = getPrimaryName(daggling);
+ idToNamesMap.remove(daggling);
+ nameToIdBiMap.put(name, refId);
+ if (!idToNamesMap.get(refId).contains(name)) {
+ TUtil.putToNestedList(idToNamesMap, refId, name);
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ae384685/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index 2fd7b14..50d274d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -138,6 +138,15 @@ public class TestSelectQuery extends QueryTestCaseBase {
cleanupQuery(res);
}
+ @Test
+ public final void testSelectColumnAliasExistingInRelation3() throws Exception {
+ // This is a reproduction code and validator of TAJO-975 Bug
+ // Please see TAJO-975 in order to know this test in detail.
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
@Test
public final void testSelectSameConstantsWithDifferentAliases() throws Exception {
http://git-wip-us.apache.org/repos/asf/tajo/blob/ae384685/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation3.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation3.sql
new file mode 100644
index 0000000..98336b7
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation3.sql
@@ -0,0 +1,19 @@
+SELECT l_orderkey FROM (
+
+-- actual test query
+ SELECT
+ T1.l_orderkey
+ FROM
+ LINEITEM
+ INNER JOIN (
+ SELECT
+ T1.l_orderkey
+ FROM (
+ SELECT
+ LINEITEM.l_orderkey AS l_orderkey
+ FROM
+ LINEITEM
+ ) T1
+ ) T1 ON LINEITEM.l_orderkey=T1.l_orderkey
+
+) A ORDER BY l_orderkey; -- for determinant query result
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/ae384685/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation3.result b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation3.result
new file mode 100644
index 0000000..3b40488
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation3.result
@@ -0,0 +1,11 @@
+l_orderkey
+-------------------------------
+1
+1
+1
+1
+2
+3
+3
+3
+3
\ No newline at end of file
[08/17] git commit: TAJO-990: Implement a tool to find tajo
configurations. (jaehwa)
Posted by ji...@apache.org.
TAJO-990: Implement a tool to find tajo configurations. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/912df6f0
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/912df6f0
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/912df6f0
Branch: refs/heads/index_support
Commit: 912df6f081b91785a56cb4b312e94b71228ecfd2
Parents: 242d6ad
Author: Jaehwa Jung <bl...@apache.org>
Authored: Sun Aug 10 15:03:07 2014 +0900
Committer: Jaehwa Jung <bl...@apache.org>
Committed: Sun Aug 10 15:03:07 2014 +0900
----------------------------------------------------------------------
.../main/java/org/apache/tajo/cli/TajoCli.java | 3 +-
.../org/apache/tajo/cli/TajoGetConfCommand.java | 57 +++++++
.../java/org/apache/tajo/client/TajoAdmin.java | 16 ++
.../org/apache/tajo/client/TajoGetConf.java | 171 +++++++++++++++++++
.../java/org/apache/tajo/cli/TestTajoCli.java | 31 ++++
tajo-dist/src/main/bin/tajo | 5 +
6 files changed, 282 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
index 98a5bf4..c20e44b 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
@@ -79,7 +79,8 @@ public class TajoCli {
UnsetCommand.class,
ExecExternalShellCommand.class,
HdfsCommand.class,
- TajoAdminCommand.class
+ TajoAdminCommand.class,
+ TajoGetConfCommand.class
};
private final Map<String, TajoShellCommand> commands = new TreeMap<String, TajoShellCommand>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-client/src/main/java/org/apache/tajo/cli/TajoGetConfCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoGetConfCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoGetConfCommand.java
new file mode 100644
index 0000000..83ba4dd
--- /dev/null
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoGetConfCommand.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.cli;
+
+import org.apache.tajo.client.TajoGetConf;
+
+public class TajoGetConfCommand extends TajoShellCommand {
+ private TajoGetConf getconf;
+
+ public TajoGetConfCommand(TajoCli.TajoCliContext context) {
+ super(context);
+ getconf = new TajoGetConf(context.getConf(), context.getOutput(), context.getTajoClient());
+ }
+
+ @Override
+ public String getCommand() {
+ return "\\getconf";
+ }
+
+ @Override
+ public void invoke(String[] command) throws Exception {
+ try {
+ String[] getConfCommands = new String[command.length - 1];
+ System.arraycopy(command, 1, getConfCommands, 0, getConfCommands.length);
+
+ getconf.runCommand(getConfCommands);
+ } catch (Exception e) {
+ context.getOutput().println("ERROR: " + e.getMessage());
+ }
+ }
+
+ @Override
+ public String getUsage() {
+ return "<command> [options]";
+ }
+
+ @Override
+ public String getDescription() {
+ return "execute a tajo getconf command.";
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java
index ad42675..4f38858 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoAdmin.java
@@ -26,11 +26,13 @@ import org.apache.tajo.TajoProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.ipc.ClientProtos.BriefQueryInfo;
import org.apache.tajo.ipc.ClientProtos.WorkerResourceInfo;
+import org.apache.tajo.util.NetUtils;
import org.apache.tajo.util.TajoIdUtils;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
+import java.net.InetSocketAddress;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
@@ -58,6 +60,7 @@ public class TajoAdmin {
options.addOption("p", "port", true, "Tajo server port");
options.addOption("list", null, false, "Show Tajo query list");
options.addOption("cluster", null, false, "Show Cluster Info");
+ options.addOption("showmasters", null, false, "gets list of tajomasters in the cluster");
options.addOption("desc", null, false, "Show Query Description");
options.addOption("kill", null, true, "Kill a running query");
}
@@ -108,6 +111,8 @@ public class TajoAdmin {
} else if (cmd.hasOption("kill")) {
cmdType = 4;
queryId = cmd.getOptionValue("kill");
+ } else if (cmd.hasOption("showmasters")) {
+ cmdType = 5;
}
// if there is no "-h" option,
@@ -155,6 +160,9 @@ public class TajoAdmin {
case 4:
processKill(writer, queryId);
break;
+ case 5:
+ processMasters(writer);
+ break;
default:
printUsage();
break;
@@ -406,6 +414,14 @@ public class TajoAdmin {
}
}
+ private void processMasters(Writer writer) throws ParseException, IOException,
+ ServiceException, SQLException {
+ String confMasterServiceAddr = tajoClient.getConf().getVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS);
+ InetSocketAddress masterAddress = NetUtils.createSocketAddr(confMasterServiceAddr);
+ writer.write(masterAddress.getHostName());
+ writer.write("\n");
+ }
+
public static void main(String [] args) throws Exception {
TajoConf conf = new TajoConf();
http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-client/src/main/java/org/apache/tajo/client/TajoGetConf.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoGetConf.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoGetConf.java
new file mode 100644
index 0000000..32e6382
--- /dev/null
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoGetConf.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.client;
+
+import com.google.protobuf.ServiceException;
+import org.apache.commons.cli.*;
+import org.apache.commons.lang.StringUtils;
+import org.apache.tajo.QueryId;
+import org.apache.tajo.TajoProtos;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ipc.ClientProtos.BriefQueryInfo;
+import org.apache.tajo.ipc.ClientProtos.WorkerResourceInfo;
+import org.apache.tajo.util.NetUtils;
+import org.apache.tajo.util.TajoIdUtils;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.net.InetSocketAddress;
+import java.sql.SQLException;
+import java.text.DecimalFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TajoGetConf {
+ private static final org.apache.commons.cli.Options options;
+
+ static {
+ options = new Options();
+ options.addOption("h", "host", true, "Tajo server host");
+ options.addOption("p", "port", true, "Tajo server port");
+ }
+
+ private TajoConf tajoConf;
+ private TajoClient tajoClient;
+ private Writer writer;
+
+ public final static String defaultLeftPad = " ";
+ public final static String defaultDescPad = " ";
+
+ public TajoGetConf(TajoConf tajoConf, Writer writer) {
+ this(tajoConf, writer, null);
+ }
+
+ public TajoGetConf(TajoConf tajoConf, Writer writer, TajoClient tajoClient) {
+ this.tajoConf = tajoConf;
+ this.writer = writer;
+ this.tajoClient = tajoClient;
+ }
+
+ private void printUsage(boolean tsqlMode) {
+ if (!tsqlMode) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp( "getconf <key> [options]", options );
+ }
+ System.out.println(defaultLeftPad + "key" + defaultDescPad + "gets a specific key from the configuration");
+ }
+
+ public void runCommand(String[] args) throws Exception {
+ runCommand(args, true);
+ }
+
+ public void runCommand(String[] args, boolean tsqlMode) throws Exception {
+ CommandLineParser parser = new PosixParser();
+
+ if (args.length == 0) {
+ printUsage(tsqlMode);
+ return;
+ }
+
+ CommandLine cmd = parser.parse(options, args);
+
+ String hostName = null;
+ Integer port = null;
+ if (cmd.hasOption("h")) {
+ hostName = cmd.getOptionValue("h");
+ }
+ if (cmd.hasOption("p")) {
+ port = Integer.parseInt(cmd.getOptionValue("p"));
+ }
+
+ String param;
+ if (cmd.getArgs().length > 1) {
+ printUsage(tsqlMode);
+ return;
+ } else {
+ param = cmd.getArgs()[0];
+ }
+
+ // if there is no "-h" option,
+ if(hostName == null) {
+ if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) {
+ // it checks if the client service address is given in configuration and distributed mode.
+ // if so, it sets entryAddr.
+ hostName = tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0];
+ }
+ }
+ if (port == null) {
+ if (tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) {
+ // it checks if the client service address is given in configuration and distributed mode.
+ // if so, it sets entryAddr.
+ port = Integer.parseInt(tajoConf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]);
+ }
+ }
+
+ if ((hostName == null) ^ (port == null)) {
+ return;
+ } else if (hostName != null && port != null) {
+ tajoConf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName + ":" + port);
+ tajoClient = new TajoClient(tajoConf);
+ } else if (hostName == null && port == null) {
+ tajoClient = new TajoClient(tajoConf);
+ }
+
+ processConfKey(writer, param);
+ writer.flush();
+ }
+
+ private void processConfKey(Writer writer, String param) throws ParseException, IOException,
+ ServiceException, SQLException {
+ String value = tajoClient.getConf().getTrimmed(param);
+
+ // If there is no value in the configuration file, we need to find all ConfVars.
+ if (value == null) {
+ for(TajoConf.ConfVars vars : TajoConf.ConfVars.values()) {
+ if (vars.varname.equalsIgnoreCase(param)) {
+ value = tajoClient.getConf().getVar(vars);
+ break;
+ }
+ }
+ }
+
+ if (value != null) {
+ writer.write(value);
+ } else {
+ writer.write("Configuration " + param + " is missing.");
+ }
+
+ writer.write("\n");
+ }
+
+ public static void main(String [] args) throws Exception {
+ TajoConf conf = new TajoConf();
+
+ Writer writer = new PrintWriter(System.out); try {
+ System.out.println("### 1000 ###");
+ TajoGetConf admin = new TajoGetConf(conf, writer);
+ admin.runCommand(args, false);
+ } finally {
+ writer.close();
+ System.exit(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java
index f862cb1..0631b6e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java
@@ -30,6 +30,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.NetUtils;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
@@ -292,6 +293,36 @@ public class TestTajoCli {
assertOutputResult(consoleResult);
}
+ @Test
+ public void testGetConf() throws Exception {
+ TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
+ tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
+ tajoCli.executeMetaCommand("\\getconf tajo.rootdir");
+
+ String consoleResult = new String(out.toByteArray());
+ assertEquals(consoleResult, tajoCli.getContext().getConf().getVar(ConfVars.ROOT_DIR) + "\n");
+ }
+
+ @Test
+ public void testShowMasters() throws Exception {
+ TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration();
+ tajoConf.setVar(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out);
+ tajoCli.executeMetaCommand("\\admin -showmasters");
+
+ String consoleResult = new String(out.toByteArray());
+
+ String masterAddress = tajoCli.getContext().getConf().getVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS);
+ String host = masterAddress.split(":")[0];
+
+ assertEquals(consoleResult, host + "\n");
+ }
+
public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputFormatter {
@Override
protected String getResponseTimeReadable(float responseTime) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/912df6f0/tajo-dist/src/main/bin/tajo
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo
index a05c65c..b6acdd8 100755
--- a/tajo-dist/src/main/bin/tajo
+++ b/tajo-dist/src/main/bin/tajo
@@ -68,6 +68,7 @@ if [ $# = 0 ]; then
echo " catutil catalog utility"
echo " cli run the tajo cli"
echo " admin run the tajo admin util"
+ echo " getconf print tajo configuration"
echo " jar <jar> run a jar file"
echo " benchmark run the benchmark driver"
echo " or"
@@ -348,6 +349,10 @@ elif [ "$COMMAND" = "admin" ] ; then
CLASS='org.apache.tajo.client.TajoAdmin'
TAJO_ROOT_LOGGER_APPENDER="${TAJO_ROOT_LOGGER_APPENDER:-NullAppender}"
TAJO_OPTS="$TAJO_OPTS $TAJO_CLI_OPTS"
+elif [ "$COMMAND" = "getconf" ] ; then
+ CLASS='org.apache.tajo.client.TajoGetConf'
+ TAJO_ROOT_LOGGER_APPENDER="${TAJO_ROOT_LOGGER_APPENDER:-NullAppender}"
+ TAJO_OPTS="$TAJO_OPTS $TAJO_CLI_OPTS"
elif [ "$COMMAND" = "dump" ] ; then
CLASS='org.apache.tajo.client.TajoDump'
TAJO_ROOT_LOGGER_APPENDER="${TAJO_ROOT_LOGGER_APPENDER:-NullAppender}"
[17/17] git commit: Merge branch 'master' of
http://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/tajo into index_support
Conflicts:
tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fa8f037b
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fa8f037b
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fa8f037b
Branch: refs/heads/index_support
Commit: fa8f037bb5676e9499c23a0f489cf2a448885ddd
Parents: 5c0277f ddfc3f3
Author: Jihoon Son <ji...@apache.org>
Authored: Mon Aug 11 17:11:04 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Mon Aug 11 17:11:04 2014 +0900
----------------------------------------------------------------------
CHANGES | 33 +-
.../java/org/apache/tajo/catalog/TableMeta.java | 14 +-
.../apache/tajo/catalog/TestKeyValueSet.java | 83 +++-
.../tajo/catalog/store/HCatalogStore.java | 28 +-
.../tajo/catalog/store/TestHCatalogStore.java | 12 +-
.../org/apache/tajo/catalog/TestCatalog.java | 10 +-
.../tajo/cli/DefaultTajoCliOutputFormatter.java | 16 +-
.../java/org/apache/tajo/cli/HelpCommand.java | 100 ++---
.../java/org/apache/tajo/cli/SetCommand.java | 78 +++-
.../main/java/org/apache/tajo/cli/TajoCli.java | 152 +++++---
.../apache/tajo/cli/TajoCliOutputFormatter.java | 5 +-
.../org/apache/tajo/cli/TajoGetConfCommand.java | 57 +++
.../org/apache/tajo/cli/TajoShellCommand.java | 10 +-
.../java/org/apache/tajo/client/TajoAdmin.java | 16 +
.../org/apache/tajo/client/TajoGetConf.java | 171 +++++++++
.../main/java/org/apache/tajo/ConfigKey.java | 52 +++
.../java/org/apache/tajo/OverridableConf.java | 235 ++++++++++++
.../main/java/org/apache/tajo/SessionVars.java | 211 +++++++++++
.../java/org/apache/tajo/conf/TajoConf.java | 204 +++++-----
.../main/java/org/apache/tajo/datum/Datum.java | 72 ++++
.../java/org/apache/tajo/datum/Float4Datum.java | 60 ++-
.../java/org/apache/tajo/datum/Float8Datum.java | 78 +++-
.../java/org/apache/tajo/datum/Int2Datum.java | 76 +++-
.../java/org/apache/tajo/datum/Int4Datum.java | 78 +++-
.../java/org/apache/tajo/datum/Int8Datum.java | 78 +++-
.../org/apache/tajo/datum/IntervalDatum.java | 16 +-
.../java/org/apache/tajo/datum/NullDatum.java | 1 +
.../java/org/apache/tajo/util/BytesUtils.java | 35 ++
.../java/org/apache/tajo/util/KeyValueSet.java | 149 ++++++--
.../tajo/datum/TestArithmeticOperator.java | 377 +++++++++++++++++++
.../exception/RangeOverflowException.java | 5 +-
.../function/builtin/CountValueDistinct.java | 4 +-
.../apache/tajo/engine/function/math/Round.java | 2 +-
.../tajo/engine/function/math/RoundFloat8.java | 39 +-
.../tajo/engine/planner/LogicalOptimizer.java | 9 +-
.../apache/tajo/engine/planner/LogicalPlan.java | 4 +-
.../engine/planner/LogicalPlanPreprocessor.java | 18 +-
.../engine/planner/LogicalPlanVerifier.java | 12 +-
.../tajo/engine/planner/LogicalPlanner.java | 35 +-
.../engine/planner/PhysicalPlannerImpl.java | 44 ++-
.../engine/planner/PreLogicalPlanVerifier.java | 21 +-
.../engine/planner/RangePartitionAlgorithm.java | 97 ++---
.../engine/planner/UniformRangePartition.java | 259 ++++++++-----
.../engine/planner/global/GlobalPlanner.java | 18 +-
.../planner/physical/ExternalSortExec.java | 4 +-
.../engine/planner/physical/StoreTableExec.java | 6 +-
.../planner/rewrite/ProjectionPushDownRule.java | 10 +
.../apache/tajo/engine/query/QueryContext.java | 144 +++----
.../tajo/engine/query/QueryUnitRequestImpl.java | 3 +-
.../org/apache/tajo/master/GlobalEngine.java | 112 +++---
.../tajo/master/TajoMasterClientService.java | 34 +-
.../apache/tajo/master/querymaster/Query.java | 4 +-
.../tajo/master/querymaster/QueryMaster.java | 39 +-
.../querymaster/QueryMasterManagerService.java | 7 +-
.../master/querymaster/QueryMasterTask.java | 4 +-
.../tajo/master/querymaster/Repartitioner.java | 138 +++++--
.../tajo/master/querymaster/SubQuery.java | 31 +-
.../org/apache/tajo/master/session/Session.java | 50 +--
.../session/SessionLivelinessMonitor.java | 2 +-
.../java/org/apache/tajo/worker/FetchImpl.java | 23 ++
.../java/org/apache/tajo/worker/Fetcher.java | 31 +-
.../tajo/worker/TajoWorkerManagerService.java | 13 +
.../main/java/org/apache/tajo/worker/Task.java | 12 +-
.../apache/tajo/worker/TaskAttemptContext.java | 12 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 21 +-
.../src/main/proto/TajoWorkerProtocol.proto | 5 +
.../resources/webapps/worker/querydetail.jsp | 10 +
.../apache/tajo/LocalTajoTestingUtility.java | 5 +
.../java/org/apache/tajo/QueryTestCaseBase.java | 2 +-
.../test/java/org/apache/tajo/TpchTestBase.java | 2 +-
.../tajo/cli/TestDefaultCliOutputFormatter.java | 47 ++-
.../java/org/apache/tajo/cli/TestTajoCli.java | 185 +++++----
.../org/apache/tajo/client/TestTajoClient.java | 8 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 9 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 16 +-
.../tajo/engine/function/TestMathFunctions.java | 26 +-
.../engine/planner/TestLogicalOptimizer.java | 20 +-
.../tajo/engine/planner/TestLogicalPlan.java | 3 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 138 +++++--
.../tajo/engine/planner/TestPlannerUtil.java | 5 +-
.../planner/TestUniformRangePartition.java | 159 +++++++-
.../planner/global/TestBroadcastJoinPlan.java | 62 +--
.../planner/physical/TestBNLJoinExec.java | 9 +-
.../planner/physical/TestBSTIndexExec.java | 4 +-
.../planner/physical/TestExternalSortExec.java | 4 +-
.../physical/TestFullOuterHashJoinExec.java | 24 +-
.../physical/TestFullOuterMergeJoinExec.java | 29 +-
.../planner/physical/TestHashAntiJoinExec.java | 4 +-
.../planner/physical/TestHashJoinExec.java | 19 +-
.../planner/physical/TestHashSemiJoinExec.java | 4 +-
.../physical/TestLeftOuterHashJoinExec.java | 28 +-
.../physical/TestLeftOuterNLJoinExec.java | 25 +-
.../planner/physical/TestMergeJoinExec.java | 4 +-
.../engine/planner/physical/TestNLJoinExec.java | 17 +-
.../planner/physical/TestPhysicalPlanner.java | 100 ++---
.../physical/TestProgressExternalSortExec.java | 4 +-
.../physical/TestRightOuterHashJoinExec.java | 16 +-
.../physical/TestRightOuterMergeJoinExec.java | 32 +-
.../engine/planner/physical/TestSortExec.java | 4 +-
.../tajo/engine/query/TestCreateIndex.java | 5 +-
.../tajo/engine/query/TestGroupByQuery.java | 64 ++--
.../tajo/engine/query/TestJoinBroadcast.java | 8 +-
.../apache/tajo/engine/query/TestJoinQuery.java | 90 ++---
.../tajo/engine/query/TestNullValues.java | 14 +-
.../tajo/engine/query/TestSelectQuery.java | 40 +-
.../apache/tajo/engine/query/TestSortQuery.java | 8 +-
.../org/apache/tajo/jdbc/TestResultSet.java | 2 +-
.../tajo/master/TestExecutionBlockCursor.java | 6 +-
.../apache/tajo/master/TestGlobalPlanner.java | 4 +-
.../apache/tajo/master/TestRepartitioner.java | 93 ++++-
.../querymaster/TestQueryUnitStatusUpdate.java | 2 +-
.../apache/tajo/worker/TestDeletionService.java | 61 +++
.../org/apache/tajo/worker/TestFetcher.java | 29 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 8 +-
.../testDistinctAggregationCaseByCase3.sql | 8 +
.../testDistinctAggregationCaseByCase4.sql | 7 +
.../TestSelectQuery/testCaseWhenRound.sql | 8 +
...testSelectColumnAliasExistingInRelation3.sql | 19 +
.../testDistinctAggregationCaseByCase3.result | 3 +
.../testDistinctAggregationCaseByCase4.result | 3 +
.../TestSelectQuery/testCaseWhenRound.result | 22 ++
...tSelectColumnAliasExistingInRelation3.result | 11 +
.../TestTajoCli/testHelpSessionVars.result | 36 ++
...estSelectResultWithNullTrueDeprecated.result | 9 +
.../testStopWhenErrorDeprecated.result | 6 +
tajo-dist/src/main/bin/tajo | 5 +
.../java/org/apache/tajo/storage/Appender.java | 2 +-
.../java/org/apache/tajo/storage/CSVFile.java | 11 +-
.../org/apache/tajo/storage/StorageUtil.java | 18 +-
.../org/apache/tajo/storage/avro/AvroUtil.java | 17 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileAppender.java | 30 +-
.../sequencefile/SequenceFileScanner.java | 21 +-
.../apache/tajo/storage/v2/RCFileScanner.java | 2 +-
.../tajo/storage/TestCompressionStorages.java | 12 +
.../org/apache/tajo/storage/TestStorages.java | 13 +
.../tajo/pullserver/TajoPullServerService.java | 54 ++-
137 files changed, 4202 insertions(+), 1362 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/CHANGES
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 0e2b26c,35df11f..162aed2
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@@ -1850,57 -1844,6 +1849,57 @@@ public class LogicalPlanner extends Bas
}
@Override
+ public LogicalNode visitCreateIndex(PlanContext context, Stack<Expr> stack, CreateIndex createIndex)
+ throws PlanningException {
+ stack.push(createIndex);
+ LogicalNode child = visit(context, stack, createIndex.getChild());
+ stack.pop();
+
+ QueryBlock block = context.queryBlock;
+ CreateIndexNode createIndexNode = block.getNodeFromExpr(createIndex);
+ if (CatalogUtil.isFQTableName(createIndex.getIndexName())) {
+ createIndexNode.setIndexName(createIndex.getIndexName());
+ } else {
+ createIndexNode.setIndexName(
- CatalogUtil.buildFQName(context.session.getCurrentDatabase(), createIndex.getIndexName()));
++ CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), createIndex.getIndexName()));
+ }
+ createIndexNode.setUnique(createIndex.isUnique());
+ Sort.SortSpec[] sortSpecs = createIndex.getSortSpecs();
+ int sortKeyNum = sortSpecs.length;
+ String[] referNames = new String[sortKeyNum];
+
+ ExprNormalizedResult[] normalizedExprList = new ExprNormalizedResult[sortKeyNum];
+ for (int i = 0; i < sortKeyNum; i++) {
+ normalizedExprList[i] = normalizer.normalize(context, sortSpecs[i].getKey());
+ }
+ for (int i = 0; i < sortKeyNum; i++) {
+ referNames[i] = block.namedExprsMgr.addExpr(normalizedExprList[i].baseExpr);
+ block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].aggExprs);
+ block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs);
+ }
+
+ createIndexNode.setSortSpecs(annotateSortSpecs(block, referNames, sortSpecs));
+ createIndexNode.setIndexType(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase()));
+
+ if (createIndex.getParams() != null) {
+ KeyValueSet keyValueSet = new KeyValueSet();
+ keyValueSet.putAll(createIndex.getParams());
+ createIndexNode.setOptions(keyValueSet);
+ }
+
+ createIndexNode.setInSchema(child.getOutSchema());
+ createIndexNode.setChild(child);
+ return createIndexNode;
+ }
+
+ @Override
+ public LogicalNode visitDropIndex(PlanContext context, Stack<Expr> stack, DropIndex dropIndex) {
+ DropIndexNode dropIndexNode = context.queryBlock.getNodeFromExpr(dropIndex);
+ dropIndexNode.init(dropIndex.getIndexName(), dropIndex.isIfExists());
+ return dropIndexNode;
+ }
+
+ @Override
public LogicalNode visitTruncateTable(PlanContext context, Stack<Expr> stack, TruncateTable truncateTable)
throws PlanningException {
TruncateTableNode truncateTableNode = context.queryBlock.getNodeFromExpr(truncateTable);
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index aed2fe6,37a56ba..24637b1
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@@ -190,17 -173,10 +176,23 @@@ public class GlobalEngine extends Abstr
if (PlannerUtil.checkIfDDLPlan(rootNode)) {
context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
- updateQuery(queryContext, rootNode.getChild());
- responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
++//<<<<<<< HEAD
++//=======
++// updateQuery(queryContext, rootNode.getChild());
++// responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
++// responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
++//>>>>>>> ddfc3f33039ee15ab0a2d3fe5890b9acb40aec3d
+
+ if (PlannerUtil.checkIfCreateIndexPlan(rootNode)) {
+ return createIndex(session, (CreateIndexNode)rootNode.getChild(), queryContext,
+ plan, sql, jsonExpr, responseBuilder);
+ } else {
- updateQuery(session, rootNode.getChild());
++ updateQuery(queryContext, rootNode.getChild());
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ return responseBuilder.build();
+ }
} else if (plan.isExplain()) { // explain query
String explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
Schema schema = new Schema();
@@@ -468,13 -432,8 +460,13 @@@
return true;
case TRUNCATE_TABLE:
TruncateTableNode truncateTable = (TruncateTableNode) root;
- truncateTable(session, truncateTable);
+ truncateTable(queryContext, truncateTable);
return true;
+ case DROP_INDEX:
+ DropIndexNode dropIndexNode = (DropIndexNode) root;
- dropIndex(session, dropIndexNode);
++ dropIndex(queryContext, dropIndexNode);
+ return true;
+
default:
throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson());
}
@@@ -663,98 -624,6 +657,98 @@@
}
}
+ /**
+ * Create an index for a given table.
+ * @param session user session
+ * @param createIndexNode the root of logical plan
+ */
+ private SubmitQueryResponse createIndex(final Session session, final CreateIndexNode createIndexNode,
+ QueryContext queryContext, LogicalPlan plan,
+ String sql, String jsonExpr,
+ SubmitQueryResponse.Builder responseBuilder) throws Exception {
+ SubmitQueryResponse response = null;
+ final CatalogService catalog = context.getCatalog();
+ final String dbName = session.getCurrentDatabase();
+ String indexName = createIndexNode.getIndexName();
+ if (CatalogUtil.isFQTableName(indexName)) {
+ indexName = CatalogUtil.splitFQTableName(indexName)[1];
+ }
+
+ boolean exists = catalog.existIndexByName(dbName, indexName);
+ if (exists) {
+ if (createIndexNode.isIfNotExists()) {
+ LOG.info("index \"" + indexName + "\" already exists." );
+ } else {
+ throw new AlreadyExistsIndexException(createIndexNode.getIndexName());
+ }
+ } else {
+ response = executeInCluster(queryContext, plan, session, sql, jsonExpr, responseBuilder);
+
+ // get the table name and predicate from scan
+ try {
+ ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN);
+ String tableName;
+ if (CatalogUtil.isFQTableName(scanNode.getTableName())) {
+ tableName = CatalogUtil.splitFQTableName(scanNode.getTableName())[1];
+ } else {
+ tableName = scanNode.getTableName();
+ }
+ String predicate = scanNode.hasQual() ? scanNode.getQual().toJson() : null;
+ // extract index keys
+ List<IndexKey> indexKeys = TUtil.newList();
+ for (SortSpec eachKey : createIndexNode.getSortSpecs()) {
+ indexKeys.add(new IndexKey(eachKey.getSortKey().toJson(), eachKey.isAscending(), eachKey.isNullFirst()));
+ }
+
+ IndexDesc indexDesc = new IndexDesc(indexName, dbName, tableName, createIndexNode.getIndexType(),
+ indexKeys, createIndexNode.isUnique(), createIndexNode.isClustered(), predicate);
+ catalog.createIndex(indexDesc);
+ } catch (Exception e) {
+ // delete index
+ deleteIndexFiles(dbName, indexName);
+ }
+ }
+ return response;
+ }
+
+ /**
+ * Drop the specified index.
- * @param session user session
++ * @param queryContext query context
+ * @param indexName index name
+ * @param ifExists if exists
+ */
- public void dropIndex(final Session session, String indexName, boolean ifExists) {
++ public void dropIndex(final QueryContext queryContext, String indexName, boolean ifExists) {
+ final CatalogService catalog = context.getCatalog();
- final String dbName = session.getCurrentDatabase();
++ final String dbName = queryContext.getCurrentDatabase();
+
+ boolean exists = catalog.existIndexByName(dbName, indexName);
+ LOG.info("index name exist: " + exists);
+ if (!exists) {
+ if (ifExists) {
+ LOG.info("index \"" + indexName + "\" does not exist." );
+ } else {
+ throw new NoSuchIndexException(indexName);
+ }
+ } else {
+ catalog.dropIndex(dbName, indexName);
+ deleteIndexFiles(dbName, indexName);
+ }
+ }
+
+ private void deleteIndexFiles(String dbName, String indexName) {
+ Path indexPath = new Path(context.getConf().getVar(ConfVars.WAREHOUSE_DIR), dbName + "/" + indexName);
+ try {
+ FileSystem fs = indexPath.getFileSystem(context.getConf());
+ fs.delete(indexPath, true);
+ } catch (IOException e) {
+ throw new InternalError(e.getMessage());
+ }
+ }
+
- private void dropIndex(final Session session, final DropIndexNode dropIndexNode) {
- dropIndex(session, dropIndexNode.getIndexName(), dropIndexNode.isIfExists());
++ private void dropIndex(final QueryContext queryContext, final DropIndexNode dropIndexNode) {
++ dropIndex(queryContext, dropIndexNode.getIndexName(), dropIndexNode.isIfExists());
+ }
+
private boolean existColumnName(String tableName, String columnName) {
final TableDesc tableDesc = catalog.getTableDesc(tableName);
return tableDesc.getSchema().containsByName(columnName) ? true : false;
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 1ac85e3,7d80a88..2f3cbf9
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@@ -754,42 -770,5 +773,43 @@@ public class TajoMasterClientService ex
throw new ServiceException(t);
}
}
+
+ @Override
+ public BoolProto existIndex(RpcController controller, SessionedStringProto request) throws ServiceException {
+ try {
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String databaseName;
+ String indexName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ indexName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ indexName = request.getValue();
+ }
+
+ if (catalog.existIndexByName(databaseName, indexName)) {
+ return BOOL_TRUE;
+ } else {
+ return BOOL_FALSE;
+ }
+ } catch (Throwable e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public BoolProto dropIndex(RpcController controller, SessionedStringProto request) throws ServiceException {
+ try {
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
- context.getGlobalEngine().dropIndex(session, request.getValue(), true);
++ QueryContext queryContext = new QueryContext(conf, session);
++ context.getGlobalEngine().dropIndex(queryContext, request.getValue(), true);
+ return BOOL_TRUE;
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index 6a6d07c,73f72be..81be5c4
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@@ -835,31 -871,11 +870,34 @@@ public class TestLogicalPlanner
}
@Test
+ public final void testCreateIndexNode() throws PlanningException {
++ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+ Expr expr = sqlAnalyzer.parse(QUERIES[11]);
- LogicalPlan rootNode = planner.createPlan(session, expr);
++ LogicalPlan rootNode = planner.createPlan(qc, expr);
+ LogicalNode plan = rootNode.getRootBlock().getRoot();
+ testJsonSerDerObject(plan);
+
+ LogicalRootNode root = (LogicalRootNode) plan;
+ assertEquals(NodeType.CREATE_INDEX, root.getChild().getType());
+ CreateIndexNode createIndexNode = root.getChild();
+
+ assertEquals(NodeType.PROJECTION, createIndexNode.getChild().getType());
+ ProjectionNode projNode = createIndexNode.getChild();
+
+ assertEquals(NodeType.SELECTION, projNode.getChild().getType());
+ SelectionNode selNode = projNode.getChild();
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName());
+ }
+
+ @Test
public final void testAsterisk() throws CloneNotSupportedException, PlanningException {
+ QueryContext qc = new QueryContext(util.getConfiguration(), session);
+
Expr expr = sqlAnalyzer.parse(QUERIES[13]);
- LogicalPlan planNode = planner.createPlan(session, expr);
+ LogicalPlan planNode = planner.createPlan(qc, expr);
LogicalNode plan = planNode.getRootBlock().getRoot();
assertEquals(NodeType.ROOT, plan.getType());
TestLogicalNode.testCloneLogicalNode(plan);
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 6c36040,a184a9a..072cff3
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@@ -797,17 -799,11 +799,17 @@@ public class TestPhysicalPlanner
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex");
+ Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee");
+ if (sm.getFileSystem().exists(indexPath)) {
+ sm.getFileSystem().delete(indexPath, true);
+ }
+
- TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(),
+ TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
+ ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(createIndexStmt[0]);
- LogicalPlan plan = planner.createPlan(session, context);
+ LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
http://git-wip-us.apache.org/repos/asf/tajo/blob/fa8f037b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
index 4714bd6,0000000..f03ed61
mode 100644,000000..100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
@@@ -1,90 -1,0 +1,91 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.query;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.IntegrationTest;
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.TajoConstants;
++import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+@Category(IntegrationTest.class)
+public class TestCreateIndex extends QueryTestCaseBase {
+
+ public TestCreateIndex() {
+ super(TajoConstants.DEFAULT_DATABASE_NAME);
+ }
+
+ private void checkIndexExist(String indexName) throws IOException, ServiceException {
- Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName);
++ Path indexPath = new Path(TajoConf.getWarehouseDir(conf), "default/" + indexName);
+ FileSystem fs = indexPath.getFileSystem(conf);
+ assertTrue(fs.exists(indexPath));
+ assertEquals(2, fs.listStatus(indexPath).length);
+ assertIndexExists(indexName);
+ }
+
+ private void checkIndexNotExist(String indexName) throws IOException, ServiceException {
- Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName);
++ Path indexPath = new Path(TajoConf.getWarehouseDir(conf), "default/" + indexName);
+ FileSystem fs = indexPath.getFileSystem(conf);
+ assertFalse(fs.exists(indexPath));
+ assertIndexNotExists(indexName);
+ }
+
+ @Test
+ public final void testCreateIndex() throws Exception {
+ executeQuery();
+ checkIndexExist("l_orderkey_idx");
+ executeString("drop index l_orderkey_idx");
+ checkIndexNotExist("l_orderkey_idx");
+ }
+
+ @Test
+ public final void testCreateIndexOnMultiAttrs() throws Exception {
+ executeQuery();
+ checkIndexExist("l_orderkey_partkey_idx");
+ executeString("drop index l_orderkey_partkey_idx");
+ checkIndexNotExist("l_orderkey_partkey_idx");
+ }
+
+ @Test
+ public final void testCreateIndexWithCondition() throws Exception {
+ executeQuery();
+ checkIndexExist("l_orderkey_partkey_lt10_idx");
+ executeString("drop index l_orderkey_partkey_lt10_idx");
+ checkIndexNotExist("l_orderkey_partkey_lt10_idx");
+ }
+
+ @Test
+ public final void testCreateIndexOnExpression() throws Exception {
+ executeQuery();
+ checkIndexExist("l_orderkey_100_lt10_idx");
+ executeString("drop index l_orderkey_100_lt10_idx");
+ checkIndexNotExist("l_orderkey_100_lt10_idx");
+ }
+}
[04/17] git commit: TAJO-966: Range partition should support split of
multiple characters.
Posted by ji...@apache.org.
TAJO-966: Range partition should support split of multiple characters.
Closes #91
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/6b16264c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/6b16264c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/6b16264c
Branch: refs/heads/index_support
Commit: 6b16264c1860f7e8156466fd806ba9251147a702
Parents: eeaf379
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 4 14:52:11 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 4 14:52:26 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../java/org/apache/tajo/util/BytesUtils.java | 35 +++
.../exception/RangeOverflowException.java | 5 +-
.../engine/planner/RangePartitionAlgorithm.java | 97 +++----
.../engine/planner/UniformRangePartition.java | 259 ++++++++++++-------
.../tajo/master/querymaster/Repartitioner.java | 10 +-
.../planner/TestUniformRangePartition.java | 159 ++++++++++--
7 files changed, 408 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6b16264c/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index d6ac279..f98da28 100644
--- a/CHANGES
+++ b/CHANGES
@@ -29,6 +29,9 @@ Release 0.9.0 - unreleased
IMPROVEMENT
+ TAJO-966: Range partition should support split of multiple characters.
+ (hyunsik)
+
TAJO-987: Hash shuffle should be balanced according to intermediate
volumes. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/6b16264c/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
index 5f309c2..59ed4fb 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
@@ -179,4 +179,39 @@ public class BytesUtils {
}
return (byte[][]) list.toArray(new byte[list.size()][]);
}
+
+ /**
+ * It gets the maximum length among all given the array of bytes.
+ * Then, it adds padding (i.e., \0) to byte arrays which are shorter
+ * than the maximum length.
+ *
+ * @param bytes Byte arrays to be padded
+ * @return The array of padded bytes
+ */
+ public static byte[][] padBytes(byte []...bytes) {
+ byte [][] padded = new byte[bytes.length][];
+
+ int maxLen = Integer.MIN_VALUE;
+
+ for (int i = 0; i < bytes.length; i++) {
+ maxLen = Math.max(maxLen, bytes[i].length);
+ }
+
+ for (int i = 0; i < bytes.length; i++) {
+ int padLen = maxLen - bytes[i].length;
+ if (padLen == 0) {
+ padded[i] = bytes[i];
+ } else if (padLen > 0) {
+ padded[i] = Bytes.padTail(bytes[i], padLen);
+ } else {
+ throw new RuntimeException("maximum length: " + maxLen + ", bytes[" + i + "].length:" + bytes[i].length);
+ }
+ }
+
+ return padded;
+ }
+
+ public static byte [] trimBytes(byte [] bytes) {
+ return new String(bytes).trim().getBytes();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/6b16264c/tajo-core/src/main/java/org/apache/tajo/engine/exception/RangeOverflowException.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/exception/RangeOverflowException.java b/tajo-core/src/main/java/org/apache/tajo/engine/exception/RangeOverflowException.java
index 409d6ed..013c9c2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/exception/RangeOverflowException.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/exception/RangeOverflowException.java
@@ -22,7 +22,8 @@ import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleRange;
public class RangeOverflowException extends RuntimeException {
- public RangeOverflowException(TupleRange range, Tuple overflowValue, long inc) {
- super("Overflow Error: tried to increase " + inc + " to " + overflowValue + ", but the range " + range);
+ public RangeOverflowException(TupleRange range, Tuple overflowValue, long inc, boolean ascending) {
+ super("Overflow Error: tried to " + (ascending ? "increase " : "decrease ") + inc + " to " + overflowValue
+ + ", but the range " + range);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/6b16264c/tajo-core/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
index 0aa6f97..db53cd7 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java
@@ -18,20 +18,19 @@
package org.apache.tajo.engine.planner;
-import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleRange;
+import org.apache.tajo.util.Bytes;
-import java.math.BigDecimal;
+import java.math.BigInteger;
public abstract class RangePartitionAlgorithm {
protected SortSpec [] sortSpecs;
- protected TupleRange range;
- protected final BigDecimal totalCard;
+ protected TupleRange mergedRange;
+ protected final BigInteger totalCard;
/** true if the end of the range is inclusive. Otherwise, it should be false. */
protected final boolean inclusive;
@@ -43,7 +42,7 @@ public abstract class RangePartitionAlgorithm {
*/
public RangePartitionAlgorithm(SortSpec [] sortSpecs, TupleRange totalRange, boolean inclusive) {
this.sortSpecs = sortSpecs;
- this.range = totalRange;
+ this.mergedRange = totalRange;
this.inclusive = inclusive;
this.totalCard = computeCardinalityForAllColumns(sortSpecs, totalRange, inclusive);
}
@@ -56,117 +55,119 @@ public abstract class RangePartitionAlgorithm {
* @param end
* @return
*/
- public static BigDecimal computeCardinality(DataType dataType, Datum start, Datum end,
+ public static BigInteger computeCardinality(DataType dataType, Datum start, Datum end,
boolean inclusive, boolean isAscending) {
- BigDecimal columnCard;
+ BigInteger columnCard;
switch (dataType.getType()) {
case BOOLEAN:
- columnCard = new BigDecimal(2);
+ columnCard = BigInteger.valueOf(2);
break;
case CHAR:
if (isAscending) {
- columnCard = new BigDecimal(end.asChar() - start.asChar());
+ columnCard = BigInteger.valueOf((int)end.asChar() - (int)start.asChar());
} else {
- columnCard = new BigDecimal(start.asChar() - end.asChar());
+ columnCard = BigInteger.valueOf(start.asChar() - end.asChar());
}
break;
case BIT:
if (isAscending) {
- columnCard = new BigDecimal(end.asByte() - start.asByte());
+ columnCard = BigInteger.valueOf(end.asByte() - start.asByte());
} else {
- columnCard = new BigDecimal(start.asByte() - end.asByte());
+ columnCard = BigInteger.valueOf(start.asByte() - end.asByte());
}
break;
case INT2:
if (isAscending) {
- columnCard = new BigDecimal(end.asInt2() - start.asInt2());
+ columnCard = BigInteger.valueOf(end.asInt2() - start.asInt2());
} else {
- columnCard = new BigDecimal(start.asInt2() - end.asInt2());
+ columnCard = BigInteger.valueOf(start.asInt2() - end.asInt2());
}
break;
case INT4:
if (isAscending) {
- columnCard = new BigDecimal(end.asInt4() - start.asInt4());
+ columnCard = BigInteger.valueOf(end.asInt4() - start.asInt4());
} else {
- columnCard = new BigDecimal(start.asInt4() - end.asInt4());
+ columnCard = BigInteger.valueOf(start.asInt4() - end.asInt4());
}
break;
- case INT8:
+ case INT8:
+ case TIME:
+ case TIMESTAMP:
if (isAscending) {
- columnCard = new BigDecimal(end.asInt8() - start.asInt8());
+ columnCard = BigInteger.valueOf(end.asInt8() - start.asInt8());
} else {
- columnCard = new BigDecimal(start.asInt8() - end.asInt8());
+ columnCard = BigInteger.valueOf(start.asInt8() - end.asInt8());
}
break;
case FLOAT4:
if (isAscending) {
- columnCard = new BigDecimal(end.asInt4() - start.asInt4());
+ columnCard = BigInteger.valueOf(end.asInt4() - start.asInt4());
} else {
- columnCard = new BigDecimal(start.asInt4() - end.asInt4());
+ columnCard = BigInteger.valueOf(start.asInt4() - end.asInt4());
}
break;
case FLOAT8:
if (isAscending) {
- columnCard = new BigDecimal(end.asInt8() - start.asInt8());
+ columnCard = BigInteger.valueOf(end.asInt8() - start.asInt8());
} else {
- columnCard = new BigDecimal(start.asInt8() - end.asInt8());
+ columnCard = BigInteger.valueOf(start.asInt8() - end.asInt8());
}
break;
- case TEXT:
- final char textStart = (start instanceof NullDatum || start.size() == 0) ? '0' : start.asChars().charAt(0);
- final char textEnd = (end instanceof NullDatum || end.size() == 0) ? '0' : end.asChars().charAt(0);
+ case TEXT: {
+ byte [] a;
+ byte [] b;
if (isAscending) {
- columnCard = new BigDecimal(textEnd - textStart);
+ a = start.asByteArray();
+ b = end.asByteArray();
} else {
- columnCard = new BigDecimal(textStart - textEnd);
+ b = start.asByteArray();
+ a = end.asByteArray();
}
+
+ byte [] prependHeader = {1, 0};
+ final BigInteger startBI = new BigInteger(Bytes.add(prependHeader, a));
+ final BigInteger stopBI = new BigInteger(Bytes.add(prependHeader, b));
+ BigInteger diffBI = stopBI.subtract(startBI);
+ columnCard = diffBI;
break;
+ }
case DATE:
if (isAscending) {
- columnCard = new BigDecimal(end.asInt4() - start.asInt4());
- } else {
- columnCard = new BigDecimal(start.asInt4() - end.asInt4());
- }
- break;
- case TIME:
- case TIMESTAMP:
- if (isAscending) {
- columnCard = new BigDecimal(end.asInt8() - start.asInt8());
+ columnCard = BigInteger.valueOf(end.asInt4() - start.asInt4());
} else {
- columnCard = new BigDecimal(start.asInt8() - end.asInt8());
+ columnCard = BigInteger.valueOf(start.asInt4() - end.asInt4());
}
break;
case INET4:
if (isAscending) {
- columnCard = new BigDecimal(end.asInt4() - start.asInt4());
+ columnCard = BigInteger.valueOf(end.asInt4() - start.asInt4());
} else {
- columnCard = new BigDecimal(start.asInt4() - end.asInt4());
+ columnCard = BigInteger.valueOf(start.asInt4() - end.asInt4());
}
break;
default:
throw new UnsupportedOperationException(dataType + " is not supported yet");
}
- return inclusive ? columnCard.add(new BigDecimal(1)).abs() : columnCard.abs();
+ return inclusive ? columnCard.add(BigInteger.valueOf(1)).abs() : columnCard.abs();
}
/**
* It computes the value cardinality of a tuple range.
* @return
*/
- public static BigDecimal computeCardinalityForAllColumns(SortSpec[] sortSpecs, TupleRange range, boolean inclusive) {
+ public static BigInteger computeCardinalityForAllColumns(SortSpec[] sortSpecs, TupleRange range, boolean inclusive) {
Tuple start = range.getStart();
Tuple end = range.getEnd();
- Column col;
- BigDecimal cardinality = new BigDecimal(1);
- BigDecimal columnCard;
+ BigInteger cardinality = BigInteger.ONE;
+ BigInteger columnCard;
for (int i = 0; i < sortSpecs.length; i++) {
columnCard = computeCardinality(sortSpecs[i].getSortKey().getDataType(), start.get(i), end.get(i), inclusive,
sortSpecs[i].isAscending());
- if (new BigDecimal(0).compareTo(columnCard) < 0) {
+ if (BigInteger.ZERO.compareTo(columnCard) < 0) {
cardinality = cardinality.multiply(columnCard);
}
}
@@ -174,7 +175,7 @@ public abstract class RangePartitionAlgorithm {
return cardinality;
}
- public BigDecimal getTotalCardinality() {
+ public BigInteger getTotalCardinality() {
return totalCard;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/6b16264c/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
index 88cb061..0a1389a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/UniformRangePartition.java
@@ -20,26 +20,29 @@ package org.apache.tajo.engine.planner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.primitives.UnsignedLong;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.SortSpec;
+import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.exception.RangeOverflowException;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleRange;
import org.apache.tajo.storage.VTuple;
import org.apache.tajo.util.Bytes;
+import org.apache.tajo.util.BytesUtils;
import java.math.BigDecimal;
+import java.math.BigInteger;
import java.math.RoundingMode;
import java.util.List;
public class UniformRangePartition extends RangePartitionAlgorithm {
private int variableId;
- private BigDecimal[] cardForEachDigit;
- private BigDecimal[] colCards;
+ private BigInteger[] cardForEachDigit;
+ private BigInteger[] colCards;
/**
*
@@ -47,15 +50,18 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
* @param sortSpecs The description of sort keys
* @param inclusive true if the end of the range is inclusive
*/
- public UniformRangePartition(TupleRange totalRange, SortSpec[] sortSpecs, boolean inclusive) {
+ public UniformRangePartition(final TupleRange totalRange, final SortSpec[] sortSpecs, boolean inclusive) {
super(sortSpecs, totalRange, inclusive);
- colCards = new BigDecimal[sortSpecs.length];
+ colCards = new BigInteger[sortSpecs.length];
+
+ normalize(sortSpecs, this.mergedRange);
+
for (int i = 0; i < sortSpecs.length; i++) {
colCards[i] = computeCardinality(sortSpecs[i].getSortKey().getDataType(), totalRange.getStart().get(i),
totalRange.getEnd().get(i), inclusive, sortSpecs[i].isAscending());
}
- cardForEachDigit = new BigDecimal[colCards.length];
+ cardForEachDigit = new BigInteger[colCards.length];
for (int i = 0; i < colCards.length ; i++) {
if (i == 0) {
cardForEachDigit[i] = colCards[i];
@@ -74,17 +80,17 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
Preconditions.checkArgument(partNum > 0,
"The number of partitions must be positive, but the given number: "
+ partNum);
- Preconditions.checkArgument(totalCard.compareTo(new BigDecimal(partNum)) >= 0,
+ Preconditions.checkArgument(totalCard.compareTo(BigInteger.valueOf(partNum)) >= 0,
"the number of partition cannot exceed total cardinality (" + totalCard + ")");
int varId;
for (varId = 0; varId < cardForEachDigit.length; varId++) {
- if (cardForEachDigit[varId].compareTo(new BigDecimal(partNum)) >= 0)
+ if (cardForEachDigit[varId].compareTo(BigInteger.valueOf(partNum)) >= 0)
break;
}
this.variableId = varId;
- BigDecimal [] reverseCardsForDigit = new BigDecimal[variableId+1];
+ BigInteger [] reverseCardsForDigit = new BigInteger[variableId+1];
for (int i = variableId; i >= 0; i--) {
if (i == variableId) {
reverseCardsForDigit[i] = colCards[i];
@@ -94,25 +100,81 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
}
List<TupleRange> ranges = Lists.newArrayList();
- BigDecimal term = reverseCardsForDigit[0].divide(
- new BigDecimal(partNum), RoundingMode.CEILING);
- BigDecimal reminder = reverseCardsForDigit[0];
- Tuple last = range.getStart();
- while(reminder.compareTo(new BigDecimal(0)) > 0) {
+
+ BigDecimal x = new BigDecimal(reverseCardsForDigit[0]);
+
+ BigInteger term = x.divide(BigDecimal.valueOf(partNum), RoundingMode.CEILING).toBigInteger();
+ BigInteger reminder = reverseCardsForDigit[0];
+ Tuple last = mergedRange.getStart();
+ TupleRange tupleRange;
+ while(reminder.compareTo(BigInteger.ZERO) > 0) {
if (reminder.compareTo(term) <= 0) { // final one is inclusive
- ranges.add(new TupleRange(sortSpecs, last, range.getEnd()));
+ tupleRange = new TupleRange(sortSpecs, last, mergedRange.getEnd());
} else {
- Tuple next = increment(last, term.longValue(), variableId);
- ranges.add(new TupleRange(sortSpecs, last, next));
+ Tuple next = increment(last, term, variableId);
+ tupleRange = new TupleRange(sortSpecs, last, next);
}
+
+ ranges.add(tupleRange);
last = ranges.get(ranges.size() - 1).getEnd();
reminder = reminder.subtract(term);
}
+ for (TupleRange r : ranges) {
+ denormalize(sortSpecs, r);
+ }
+
return ranges.toArray(new TupleRange[ranges.size()]);
}
/**
+ * It normalizes the start and end keys to have the same length bytes if they are texts or bytes.
+ *
+ * @param sortSpecs The sort specs
+ * @param range Tuple range to be normalize
+ */
+ public static void normalize(final SortSpec [] sortSpecs, TupleRange range) {
+ // normalize text fields to have same bytes length
+ for (int i = 0; i < sortSpecs.length; i++) {
+ if (sortSpecs[i].getSortKey().getDataType().getType() == TajoDataTypes.Type.TEXT) {
+ byte [] startBytes;
+ byte [] endBytes;
+ if (range.getStart().isNull(i)) {
+ startBytes = BigInteger.ZERO.toByteArray();
+ } else {
+ startBytes = range.getStart().getBytes(i);
+ }
+
+ if (range.getEnd().isNull(i)) {
+ endBytes = BigInteger.ZERO.toByteArray();
+ } else {
+ endBytes = range.getEnd().getBytes(i);
+ }
+
+ byte [][] padded = BytesUtils.padBytes(startBytes, endBytes);
+ range.getStart().put(i, DatumFactory.createText(padded[0]));
+ range.getEnd().put(i, DatumFactory.createText(padded[1]));
+ }
+ }
+ }
+
+ /**
+ * Normalized keys have padding values, but it will cause the key mismatch in pull server.
+ * So, it denormalize the normalized keys again.
+ *
+ * @param sortSpecs The sort specs
+ * @param range Tuple range to be denormalized
+ */
+ public static void denormalize(SortSpec [] sortSpecs, TupleRange range) {
+ for (int i = 0; i < sortSpecs.length; i++) {
+ if (sortSpecs[i].getSortKey().getDataType().getType() == TajoDataTypes.Type.TEXT) {
+ range.getStart().put(i,DatumFactory.createText(BytesUtils.trimBytes(range.getStart().getBytes(i))));
+ range.getEnd().put(i,DatumFactory.createText(BytesUtils.trimBytes(range.getEnd().getBytes(i))));
+ }
+ }
+ }
+
+ /**
* Check whether an overflow occurs or not.
*
* @param colId The column id to be checked
@@ -121,105 +183,111 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
* @param sortSpecs
* @return
*/
- public boolean isOverflow(int colId, Datum last, BigDecimal inc, SortSpec [] sortSpecs) {
+ public boolean isOverflow(int colId, Datum last, BigInteger inc, SortSpec [] sortSpecs) {
Column column = sortSpecs[colId].getSortKey();
+ BigDecimal incDecimal = new BigDecimal(inc);
BigDecimal candidate;
boolean overflow = false;
switch (column.getDataType().getType()) {
case BIT: {
if (sortSpecs[colId].isAscending()) {
- candidate = inc.add(new BigDecimal(last.asByte()));
- return new BigDecimal(range.getEnd().get(colId).asByte()).compareTo(candidate) < 0;
+ candidate = incDecimal.add(new BigDecimal(last.asByte()));
+ return new BigDecimal(mergedRange.getEnd().get(colId).asByte()).compareTo(candidate) < 0;
} else {
- candidate = new BigDecimal(last.asByte()).subtract(inc);
- return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asByte())) < 0;
+ candidate = new BigDecimal(last.asByte()).subtract(incDecimal);
+ return candidate.compareTo(new BigDecimal(mergedRange.getEnd().get(colId).asByte())) < 0;
}
}
case CHAR: {
if (sortSpecs[colId].isAscending()) {
- candidate = inc.add(new BigDecimal((int)last.asChar()));
- return new BigDecimal((int)range.getEnd().get(colId).asChar()).compareTo(candidate) < 0;
+ candidate = incDecimal.add(new BigDecimal((int)last.asChar()));
+ return new BigDecimal((int) mergedRange.getEnd().get(colId).asChar()).compareTo(candidate) < 0;
} else {
- candidate = new BigDecimal((int)last.asChar()).subtract(inc);
- return candidate.compareTo(new BigDecimal((int)range.getEnd().get(colId).asChar())) < 0;
+ candidate = new BigDecimal((int)last.asChar()).subtract(incDecimal);
+ return candidate.compareTo(new BigDecimal((int) mergedRange.getEnd().get(colId).asChar())) < 0;
}
}
case INT2: {
if (sortSpecs[colId].isAscending()) {
- candidate = inc.add(new BigDecimal(last.asInt2()));
- return new BigDecimal(range.getEnd().get(colId).asInt2()).compareTo(candidate) < 0;
+ candidate = incDecimal.add(new BigDecimal(last.asInt2()));
+ return new BigDecimal(mergedRange.getEnd().get(colId).asInt2()).compareTo(candidate) < 0;
} else {
- candidate = new BigDecimal(last.asInt2()).subtract(inc);
- return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asInt2())) < 0;
+ candidate = new BigDecimal(last.asInt2()).subtract(incDecimal);
+ return candidate.compareTo(new BigDecimal(mergedRange.getEnd().get(colId).asInt2())) < 0;
}
}
case DATE:
case INT4: {
if (sortSpecs[colId].isAscending()) {
- candidate = inc.add(new BigDecimal(last.asInt4()));
- return new BigDecimal(range.getEnd().get(colId).asInt4()).compareTo(candidate) < 0;
+ candidate = incDecimal.add(new BigDecimal(last.asInt4()));
+ return new BigDecimal(mergedRange.getEnd().get(colId).asInt4()).compareTo(candidate) < 0;
} else {
- candidate = new BigDecimal(last.asInt4()).subtract(inc);
- return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asInt4())) < 0;
+ candidate = new BigDecimal(last.asInt4()).subtract(incDecimal);
+ return candidate.compareTo(new BigDecimal(mergedRange.getEnd().get(colId).asInt4())) < 0;
}
}
case TIME:
case TIMESTAMP:
case INT8: {
if (sortSpecs[colId].isAscending()) {
- candidate = inc.add(new BigDecimal(last.asInt8()));
- return new BigDecimal(range.getEnd().get(colId).asInt8()).compareTo(candidate) < 0;
+ candidate = incDecimal.add(new BigDecimal(last.asInt8()));
+ return new BigDecimal(mergedRange.getEnd().get(colId).asInt8()).compareTo(candidate) < 0;
} else {
- candidate = new BigDecimal(last.asInt8()).subtract(inc);
- return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asInt8())) < 0;
+ candidate = new BigDecimal(last.asInt8()).subtract(incDecimal);
+ return candidate.compareTo(new BigDecimal(mergedRange.getEnd().get(colId).asInt8())) < 0;
}
}
case FLOAT4: {
if (sortSpecs[colId].isAscending()) {
- candidate = inc.add(new BigDecimal(last.asFloat4()));
- return new BigDecimal(range.getEnd().get(colId).asFloat4()).compareTo(candidate) < 0;
+ candidate = incDecimal.add(new BigDecimal(last.asFloat4()));
+ return new BigDecimal(mergedRange.getEnd().get(colId).asFloat4()).compareTo(candidate) < 0;
} else {
- candidate = new BigDecimal(last.asFloat4()).subtract(inc);
- return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asFloat4())) < 0;
+ candidate = new BigDecimal(last.asFloat4()).subtract(incDecimal);
+ return candidate.compareTo(new BigDecimal(mergedRange.getEnd().get(colId).asFloat4())) < 0;
}
}
case FLOAT8: {
if (sortSpecs[colId].isAscending()) {
- candidate = inc.add(new BigDecimal(last.asFloat8()));
- return new BigDecimal(range.getEnd().get(colId).asFloat8()).compareTo(candidate) < 0;
+ candidate = incDecimal.add(new BigDecimal(last.asFloat8()));
+ return new BigDecimal(mergedRange.getEnd().get(colId).asFloat8()).compareTo(candidate) < 0;
} else {
- candidate = new BigDecimal(last.asFloat8()).subtract(inc);
- return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asFloat8())) < 0;
+ candidate = new BigDecimal(last.asFloat8()).subtract(incDecimal);
+ return candidate.compareTo(new BigDecimal(mergedRange.getEnd().get(colId).asFloat8())) < 0;
}
}
case TEXT: {
+ byte [] lastBytes = last.asByteArray();
+ byte [] endBytes = mergedRange.getEnd().getBytes(colId);
+
+ Preconditions.checkState(lastBytes.length == endBytes.length);
+
if (sortSpecs[colId].isAscending()) {
- candidate = inc.add(new BigDecimal((int)(last instanceof NullDatum ? '0' : last.asChars().charAt(0))));
- return new BigDecimal(range.getEnd().get(colId).asChars().charAt(0)).compareTo(candidate) < 0;
+ candidate = incDecimal.add(new BigDecimal(new BigInteger(lastBytes)));
+ return new BigDecimal(new BigInteger(endBytes)).compareTo(candidate) < 0;
} else {
- candidate = new BigDecimal((int)(last.asChars().charAt(0))).subtract(inc);
- return candidate.compareTo(new BigDecimal(range.getEnd().get(colId).asChars().charAt(0))) < 0;
+ candidate = new BigDecimal(new BigInteger(lastBytes)).subtract(incDecimal);
+ return candidate.compareTo(new BigDecimal(new BigInteger(endBytes))) < 0;
}
}
case INET4: {
int candidateIntVal;
byte[] candidateBytesVal = new byte[4];
if (sortSpecs[colId].isAscending()) {
- candidateIntVal = inc.intValue() + last.asInt4();
- if (candidateIntVal - inc.intValue() != last.asInt4()) {
+ candidateIntVal = incDecimal.intValue() + last.asInt4();
+ if (candidateIntVal - incDecimal.intValue() != last.asInt4()) {
return true;
}
Bytes.putInt(candidateBytesVal, 0, candidateIntVal);
- return Bytes.compareTo(range.getEnd().get(colId).asByteArray(), candidateBytesVal) < 0;
+ return Bytes.compareTo(mergedRange.getEnd().get(colId).asByteArray(), candidateBytesVal) < 0;
} else {
- candidateIntVal = last.asInt4() - inc.intValue();
- if (candidateIntVal + inc.intValue() != last.asInt4()) {
+ candidateIntVal = last.asInt4() - incDecimal.intValue();
+ if (candidateIntVal + incDecimal.intValue() != last.asInt4()) {
return true;
}
Bytes.putInt(candidateBytesVal, 0, candidateIntVal);
- return Bytes.compareTo(candidateBytesVal, range.getEnd().get(colId).asByteArray()) < 0;
+ return Bytes.compareTo(candidateBytesVal, mergedRange.getEnd().get(colId).asByteArray()) < 0;
}
}
}
@@ -232,20 +300,20 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
switch (column.getDataType().getType()) {
case BIT: {
long candidate = last.asByte() + inc;
- byte end = range.getEnd().get(colId).asByte();
+ byte end = mergedRange.getEnd().get(colId).asByte();
reminder = candidate - end;
break;
}
case CHAR: {
long candidate = last.asChar() + inc;
- char end = range.getEnd().get(colId).asChar();
+ char end = mergedRange.getEnd().get(colId).asChar();
reminder = candidate - end;
break;
}
case DATE:
case INT4: {
int candidate = (int) (last.asInt4() + inc);
- int end = range.getEnd().get(colId).asInt4();
+ int end = mergedRange.getEnd().get(colId).asInt4();
reminder = candidate - end;
break;
}
@@ -254,26 +322,34 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case INT8:
case INET4: {
long candidate = last.asInt8() + inc;
- long end = range.getEnd().get(colId).asInt8();
+ long end = mergedRange.getEnd().get(colId).asInt8();
reminder = candidate - end;
break;
}
case FLOAT4: {
float candidate = last.asFloat4() + inc;
- float end = range.getEnd().get(colId).asFloat4();
+ float end = mergedRange.getEnd().get(colId).asFloat4();
reminder = (long) (candidate - end);
break;
}
case FLOAT8: {
double candidate = last.asFloat8() + inc;
- double end = range.getEnd().get(colId).asFloat8();
+ double end = mergedRange.getEnd().get(colId).asFloat8();
reminder = (long) Math.ceil(candidate - end);
break;
}
case TEXT: {
- char candidate = ((char)(last.asChars().charAt(0) + inc));
- char end = range.getEnd().get(colId).asChars().charAt(0);
- reminder = (char) (candidate - end);
+ byte [] lastBytes = last.asByteArray();
+ byte [] endBytes = mergedRange.getEnd().get(colId).asByteArray();
+
+ Preconditions.checkState(lastBytes.length == endBytes.length);
+
+ BigInteger lastBInt = new BigInteger(lastBytes);
+ BigInteger endBInt = new BigInteger(endBytes);
+ BigInteger incBInt = BigInteger.valueOf(inc);
+
+ BigInteger candidate = lastBInt.add(incBInt);
+ reminder = candidate.subtract(endBInt).longValue();
break;
}
}
@@ -285,16 +361,16 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
/**
*
* @param last
- * @param inc
+ * @param interval
* @return
*/
- public Tuple increment(final Tuple last, final long inc, final int baseDigit) {
- BigDecimal [] incs = new BigDecimal[last.size()];
+ public Tuple increment(final Tuple last, BigInteger interval, final int baseDigit) {
+ BigInteger [] incs = new BigInteger[last.size()];
boolean [] overflowFlag = new boolean[last.size()];
- BigDecimal [] result;
- BigDecimal value = new BigDecimal(inc);
+ BigInteger [] result;
+ BigInteger value = interval;
- BigDecimal [] reverseCardsForDigit = new BigDecimal[baseDigit + 1];
+ BigInteger [] reverseCardsForDigit = new BigInteger[baseDigit + 1];
for (int i = baseDigit; i >= 0; i--) {
if (i == baseDigit) {
reverseCardsForDigit[i] = colCards[i];
@@ -313,11 +389,11 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
for (int i = finalId; i >= 0; i--) {
if (isOverflow(i, last.get(i), incs[i], sortSpecs)) {
if (i == 0) {
- throw new RangeOverflowException(range, last, incs[i].longValue());
+ throw new RangeOverflowException(mergedRange, last, incs[i].longValue(), sortSpecs[i].isAscending());
}
long rem = incrementAndGetReminder(i, last.get(i), value.longValue());
- incs[i] = new BigDecimal(rem);
- incs[i - 1] = incs[i-1].add(new BigDecimal(1));
+ incs[i] = BigInteger.valueOf(rem);
+ incs[i - 1] = incs[i-1].add(BigInteger.ONE);
overflowFlag[i] = true;
} else {
if (i > 0) {
@@ -329,7 +405,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
for (int i = 0; i < incs.length; i++) {
if (incs[i] == null) {
- incs[i] = new BigDecimal(0);
+ incs[i] = BigInteger.ZERO;
}
}
@@ -340,7 +416,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
switch (column.getDataType().getType()) {
case CHAR:
if (overflowFlag[i]) {
- end.put(i, DatumFactory.createChar((char) (range.getStart().get(i).asChar() + incs[i].longValue())));
+ end.put(i, DatumFactory.createChar((char) (mergedRange.getStart().get(i).asChar() + incs[i].longValue())));
} else {
end.put(i, DatumFactory.createChar((char) (last.get(i).asChar() + incs[i].longValue())));
}
@@ -348,7 +424,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case BIT:
if (overflowFlag[i]) {
end.put(i, DatumFactory.createBit(
- (byte) (range.getStart().get(i).asByte() + incs[i].longValue())));
+ (byte) (mergedRange.getStart().get(i).asByte() + incs[i].longValue())));
} else {
end.put(i, DatumFactory.createBit((byte) (last.get(i).asByte() + incs[i].longValue())));
}
@@ -356,7 +432,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case INT2:
if (overflowFlag[i]) {
end.put(i, DatumFactory.createInt2(
- (short) (range.getStart().get(i).asInt2() + incs[i].longValue())));
+ (short) (mergedRange.getStart().get(i).asInt2() + incs[i].longValue())));
} else {
end.put(i, DatumFactory.createInt2((short) (last.get(i).asInt2() + incs[i].longValue())));
}
@@ -364,7 +440,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case INT4:
if (overflowFlag[i]) {
end.put(i, DatumFactory.createInt4(
- (int) (range.getStart().get(i).asInt4() + incs[i].longValue())));
+ (int) (mergedRange.getStart().get(i).asInt4() + incs[i].longValue())));
} else {
if (sortSpecs[i].isAscending()) {
end.put(i, DatumFactory.createInt4((int) (last.get(i).asInt4() + incs[i].longValue())));
@@ -376,7 +452,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case INT8:
if (overflowFlag[i]) {
end.put(i, DatumFactory.createInt8(
- range.getStart().get(i).asInt8() + incs[i].longValue()));
+ mergedRange.getStart().get(i).asInt8() + incs[i].longValue()));
} else {
end.put(i, DatumFactory.createInt8(last.get(i).asInt8() + incs[i].longValue()));
}
@@ -384,7 +460,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case FLOAT4:
if (overflowFlag[i]) {
end.put(i, DatumFactory.createFloat4(
- range.getStart().get(i).asFloat4() + incs[i].longValue()));
+ mergedRange.getStart().get(i).asFloat4() + incs[i].longValue()));
} else {
end.put(i, DatumFactory.createFloat4(last.get(i).asFloat4() + incs[i].longValue()));
}
@@ -392,30 +468,35 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case FLOAT8:
if (overflowFlag[i]) {
end.put(i, DatumFactory.createFloat8(
- range.getStart().get(i).asFloat8() + incs[i].longValue()));
+ mergedRange.getStart().get(i).asFloat8() + incs[i].longValue()));
} else {
end.put(i, DatumFactory.createFloat8(last.get(i).asFloat8() + incs[i].longValue()));
}
break;
case TEXT:
if (overflowFlag[i]) {
- end.put(i, DatumFactory.createText(((char) (range.getStart().get(i).asChars().charAt(0)
+ end.put(i, DatumFactory.createText(((char) (mergedRange.getStart().get(i).asChars().charAt(0)
+ incs[i].longValue())) + ""));
} else {
- end.put(i, DatumFactory.createText(
- ((char) ((last.get(i) instanceof NullDatum ? '0': last.get(i).asChars().charAt(0)) + incs[i].longValue())) + ""));
+ BigInteger lastBigInt;
+ if (last.isNull(i)) {
+ lastBigInt = BigInteger.valueOf(0);
+ } else {
+ lastBigInt = UnsignedLong.valueOf(new BigInteger(last.get(i).asByteArray())).bigIntegerValue();
+ }
+ end.put(i, DatumFactory.createText(lastBigInt.add(incs[i]).toByteArray()));
}
break;
case DATE:
if (overflowFlag[i]) {
- end.put(i, DatumFactory.createDate((int) (range.getStart().get(i).asInt4() + incs[i].longValue())));
+ end.put(i, DatumFactory.createDate((int) (mergedRange.getStart().get(i).asInt4() + incs[i].longValue())));
} else {
end.put(i, DatumFactory.createDate((int) (last.get(i).asInt4() + incs[i].longValue())));
}
break;
case TIME:
if (overflowFlag[i]) {
- end.put(i, DatumFactory.createTime(range.getStart().get(i).asInt8() + incs[i].longValue()));
+ end.put(i, DatumFactory.createTime(mergedRange.getStart().get(i).asInt8() + incs[i].longValue()));
} else {
end.put(i, DatumFactory.createTime(last.get(i).asInt8() + incs[i].longValue()));
}
@@ -423,7 +504,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case TIMESTAMP:
if (overflowFlag[i]) {
end.put(i, DatumFactory.createTimestmpDatumWithJavaMillis(
- range.getStart().get(i).asInt8() + incs[i].longValue()));
+ mergedRange.getStart().get(i).asInt8() + incs[i].longValue()));
} else {
end.put(i, DatumFactory.createTimestmpDatumWithJavaMillis(last.get(i).asInt8() + incs[i].longValue()));
}
@@ -431,7 +512,7 @@ public class UniformRangePartition extends RangePartitionAlgorithm {
case INET4:
byte[] ipBytes;
if (overflowFlag[i]) {
- ipBytes = range.getStart().get(i).asByteArray();
+ ipBytes = mergedRange.getStart().get(i).asByteArray();
assert ipBytes.length == 4;
end.put(i, DatumFactory.createInet4(ipBytes));
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/6b16264c/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index 1fa3f11..43d6fd2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -56,7 +56,7 @@ import org.apache.tajo.worker.FetchImpl;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
-import java.math.BigDecimal;
+import java.math.BigInteger;
import java.net.URI;
import java.util.*;
import java.util.Map.Entry;
@@ -574,12 +574,12 @@ public class Repartitioner {
}
TupleRange mergedRange = TupleUtil.columnStatToRange(sortSpecs, sortSchema, totalStat.getColumnStats(), false);
RangePartitionAlgorithm partitioner = new UniformRangePartition(mergedRange, sortSpecs);
- BigDecimal card = partitioner.getTotalCardinality();
+ BigInteger card = partitioner.getTotalCardinality();
// if the number of the range cardinality is less than the desired number of tasks,
// we set the the number of tasks to the number of range cardinality.
int determinedTaskNum;
- if (card.compareTo(new BigDecimal(maxNum)) < 0) {
+ if (card.compareTo(BigInteger.valueOf(maxNum)) < 0) {
LOG.info(subQuery.getId() + ", The range cardinality (" + card
+ ") is less then the desired number of tasks (" + maxNum + ")");
determinedTaskNum = card.intValue();
@@ -587,9 +587,7 @@ public class Repartitioner {
determinedTaskNum = maxNum;
}
- // for LOG
- TupleRange mergedRangeForPrint = TupleUtil.columnStatToRange(sortSpecs, sortSchema, totalStat.getColumnStats(), true);
- LOG.info(subQuery.getId() + ", Try to divide " + mergedRangeForPrint + " into " + determinedTaskNum +
+ LOG.info(subQuery.getId() + ", Try to divide " + mergedRange + " into " + determinedTaskNum +
" sub ranges (total units: " + determinedTaskNum + ")");
TupleRange [] ranges = partitioner.partition(determinedTaskNum);
if (ranges == null || ranges.length == 0) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/6b16264c/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
index f4c114f..58653d1 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java
@@ -27,6 +27,8 @@ import org.apache.tajo.storage.TupleRange;
import org.apache.tajo.storage.VTuple;
import org.junit.Test;
+import java.math.BigInteger;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -68,11 +70,11 @@ public class TestUniformRangePartition {
result[10] = "DB";
result[11] = "DC";
- Tuple end = partitioner.increment(s, 1, 1);
+ Tuple end = partitioner.increment(s, BigInteger.valueOf(1), 1);
assertEquals("A", end.get(0).asChars());
assertEquals("B", end.get(1).asChars());
for (int i = 2; i < 11; i++ ) {
- end = partitioner.increment(end, 1, 1);
+ end = partitioner.increment(end, BigInteger.valueOf(1), 1);
assertEquals(result[i].charAt(0), end.get(0).asChars().charAt(0));
assertEquals(result[i].charAt(1), end.get(1).asChars().charAt(0));
}
@@ -115,10 +117,10 @@ public class TestUniformRangePartition {
result[10] = "DB";
result[11] = "DC";
- Tuple end = partitioner.increment(s, 6, 1);
+ Tuple end = partitioner.increment(s, BigInteger.valueOf(6), 1);
assertEquals("C", end.get(0).asChars());
assertEquals("A", end.get(1).asChars());
- end = partitioner.increment(end, 5, 1);
+ end = partitioner.increment(end, BigInteger.valueOf(5), 1);
assertEquals("D", end.get(0).asChars());
assertEquals("C", end.get(1).asChars());
}
@@ -149,11 +151,11 @@ public class TestUniformRangePartition {
UniformRangePartition partitioner = new UniformRangePartition(expected, sortSpecs);
assertEquals(24, partitioner.getTotalCardinality().intValue());
- Tuple overflowBefore = partitioner.increment(s, 5, 2);
+ Tuple overflowBefore = partitioner.increment(s, BigInteger.valueOf(5), 2);
assertEquals("A", overflowBefore.get(0).asChars());
assertEquals("B", overflowBefore.get(1).asChars());
assertEquals("C", overflowBefore.get(2).asChars());
- Tuple overflowed = partitioner.increment(overflowBefore, 1, 2);
+ Tuple overflowed = partitioner.increment(overflowBefore, BigInteger.valueOf(1), 2);
assertEquals("B", overflowed.get(0).asChars());
assertEquals("A", overflowed.get(1).asChars());
assertEquals("A", overflowed.get(2).asChars());
@@ -179,10 +181,10 @@ public class TestUniformRangePartition {
UniformRangePartition partitioner = new UniformRangePartition(expected, sortSpecs);
assertEquals(200, partitioner.getTotalCardinality().longValue());
- Tuple range2 = partitioner.increment(s, 100, 1);
+ Tuple range2 = partitioner.increment(s, BigInteger.valueOf(100), 1);
assertEquals(15, range2.get(0).asInt4());
assertEquals(20, range2.get(1).asInt4());
- Tuple range3 = partitioner.increment(range2, 99, 1);
+ Tuple range3 = partitioner.increment(range2, BigInteger.valueOf(99), 1);
assertEquals(19, range3.get(0).asInt4());
assertEquals(39, range3.get(1).asInt4());
}
@@ -209,11 +211,11 @@ public class TestUniformRangePartition {
UniformRangePartition partitioner = new UniformRangePartition(expected, sortSpecs);
assertEquals(24, partitioner.getTotalCardinality().longValue());
- Tuple beforeOverflow = partitioner.increment(s, 5, 2);
+ Tuple beforeOverflow = partitioner.increment(s, BigInteger.valueOf(5), 2);
assertEquals(1, beforeOverflow.get(0).asInt8());
assertEquals(2, beforeOverflow.get(1).asInt8());
assertEquals(3, beforeOverflow.get(2).asInt8());
- Tuple overflow = partitioner.increment(beforeOverflow, 1, 2);
+ Tuple overflow = partitioner.increment(beforeOverflow, BigInteger.valueOf(1), 2);
assertEquals(2, overflow.get(0).asInt8());
assertEquals(1, overflow.get(1).asInt8());
assertEquals(1, overflow.get(2).asInt8());
@@ -242,11 +244,11 @@ public class TestUniformRangePartition {
UniformRangePartition partitioner = new UniformRangePartition(expected, sortSpecs);
assertEquals(24, partitioner.getTotalCardinality().longValue());
- Tuple beforeOverflow = partitioner.increment(s, 5, 2);
+ Tuple beforeOverflow = partitioner.increment(s, BigInteger.valueOf(5), 2);
assertTrue(1.1d == beforeOverflow.get(0).asFloat8());
assertTrue(2.1d == beforeOverflow.get(1).asFloat8());
assertTrue(3.1d == beforeOverflow.get(2).asFloat8());
- Tuple overflow = partitioner.increment(beforeOverflow, 1, 2);
+ Tuple overflow = partitioner.increment(beforeOverflow, BigInteger.valueOf(1), 2);
assertTrue(2.1d == overflow.get(0).asFloat8());
assertTrue(1.1d == overflow.get(1).asFloat8());
assertTrue(1.1d == overflow.get(2).asFloat8());
@@ -275,11 +277,11 @@ public class TestUniformRangePartition {
UniformRangePartition partitioner = new UniformRangePartition(expected, sortSpecs);
assertEquals(24, partitioner.getTotalCardinality().longValue());
- Tuple beforeOverflow = partitioner.increment(s, 5, 2);
+ Tuple beforeOverflow = partitioner.increment(s, BigInteger.valueOf(5), 2);
assertTrue("127.0.1.1".equals(beforeOverflow.get(0).asChars()));
assertTrue("127.0.0.2".equals(beforeOverflow.get(1).asChars()));
assertTrue("128.0.0.255".equals(beforeOverflow.get(2).asChars()));
- Tuple overflow = partitioner.increment(beforeOverflow, 1, 2);
+ Tuple overflow = partitioner.increment(beforeOverflow, BigInteger.valueOf(1), 2);
assertTrue("127.0.1.2".equals(overflow.get(0).asChars()));
assertTrue("127.0.0.1".equals(overflow.get(1).asChars()));
assertTrue("128.0.0.253".equals(overflow.get(2).asChars()));
@@ -360,6 +362,133 @@ public class TestUniformRangePartition {
}
@Test
+ public void testPartitionForMultipleChars() {
+ Schema schema = new Schema()
+ .addColumn("KEY1", Type.TEXT);
+
+ SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
+
+ Tuple s = new VTuple(1);
+ s.put(0, DatumFactory.createText("AAA"));
+ Tuple e = new VTuple(1);
+ e.put(0, DatumFactory.createText("ZZZ"));
+
+ TupleRange expected = new TupleRange(sortSpecs, s, e);
+ RangePartitionAlgorithm partitioner =
+ new UniformRangePartition(expected, sortSpecs, true);
+ TupleRange [] ranges = partitioner.partition(48);
+
+ TupleRange prev = null;
+ for (TupleRange r : ranges) {
+ if (prev == null) {
+ prev = r;
+ } else {
+ assertTrue(prev.compareTo(r) < 0);
+ }
+ }
+ assertEquals(48, ranges.length);
+ assertTrue(ranges[0].getStart().equals(s));
+ assertTrue(ranges[47].getEnd().equals(e));
+ }
+
+ @Test
+ public void testPartitionForMultipleChars2() {
+ Schema schema = new Schema()
+ .addColumn("KEY1", Type.TEXT);
+
+ SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
+
+ Tuple s = new VTuple(1);
+ s.put(0, DatumFactory.createText("A1"));
+ Tuple e = new VTuple(1);
+ e.put(0, DatumFactory.createText("A999975"));
+
+ final int partNum = 2;
+
+ TupleRange expected = new TupleRange(sortSpecs, s, e);
+ RangePartitionAlgorithm partitioner =
+ new UniformRangePartition(expected, sortSpecs, true);
+ TupleRange [] ranges = partitioner.partition(partNum);
+
+ TupleRange prev = null;
+ for (TupleRange r : ranges) {
+ if (prev == null) {
+ prev = r;
+ } else {
+ assertTrue(prev.compareTo(r) < 0);
+ }
+ }
+ assertEquals(partNum, ranges.length);
+ assertTrue(ranges[0].getStart().equals(s));
+ assertTrue(ranges[partNum - 1].getEnd().equals(e));
+ }
+
+ @Test
+ public void testPartitionForMultipleChars2Desc() {
+ Schema schema = new Schema()
+ .addColumn("KEY1", Type.TEXT);
+
+ SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
+ sortSpecs[0].setDescOrder();
+
+ Tuple s = new VTuple(1);
+ s.put(0, DatumFactory.createText("A999975"));
+ Tuple e = new VTuple(1);
+ e.put(0, DatumFactory.createText("A1"));
+
+ final int partNum = 48;
+
+ TupleRange expected = new TupleRange(sortSpecs, s, e);
+ RangePartitionAlgorithm partitioner =
+ new UniformRangePartition(expected, sortSpecs, true);
+ TupleRange [] ranges = partitioner.partition(partNum);
+
+ TupleRange prev = null;
+ for (TupleRange r : ranges) {
+ if (prev == null) {
+ prev = r;
+ } else {
+ assertTrue(prev.compareTo(r) > 0);
+ }
+ }
+ assertEquals(partNum, ranges.length);
+ assertTrue(ranges[0].getStart().equals(s));
+ assertTrue(ranges[partNum - 1].getEnd().equals(e));
+ }
+
+ @Test
+ public void testPartitionForMultipleCharsWithSameFirstChar() {
+ Schema schema = new Schema()
+ .addColumn("KEY1", Type.TEXT);
+
+ SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema);
+
+ Tuple s = new VTuple(1);
+ s.put(0, DatumFactory.createText("AAA"));
+ Tuple e = new VTuple(1);
+ e.put(0, DatumFactory.createText("AAZ"));
+
+ final int partNum = 4;
+
+ TupleRange expected = new TupleRange(sortSpecs, s, e);
+ RangePartitionAlgorithm partitioner =
+ new UniformRangePartition(expected, sortSpecs, true);
+ TupleRange [] ranges = partitioner.partition(partNum);
+
+ TupleRange prev = null;
+ for (TupleRange r : ranges) {
+ if (prev == null) {
+ prev = r;
+ } else {
+ assertTrue(prev.compareTo(r) < 0);
+ }
+ }
+ assertEquals(partNum, ranges.length);
+ assertTrue(ranges[0].getStart().equals(s));
+ assertTrue(ranges[partNum - 1].getEnd().equals(e));
+ }
+
+ @Test
public void testPartitionForOnePartNumWithBothValueNull() {
Schema schema = new Schema()
.addColumn("l_returnflag", Type.TEXT)
@@ -406,7 +535,7 @@ public class TestUniformRangePartition {
if (prev == null) {
prev = r;
} else {
- assertTrue(prev.compareTo(r) > 0);
+ assertTrue(prev.compareTo(r) < 0);
}
}
}
[07/17] git commit: TAJO-949: PullServer does not release files,
when a channel throws an internal exception. (jinho)
Posted by ji...@apache.org.
TAJO-949: PullServer does not release files, when a channel throws an internal exception. (jinho)
Closes #76
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/242d6ad6
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/242d6ad6
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/242d6ad6
Branch: refs/heads/index_support
Commit: 242d6ad656685b82f2c552aa09abe862107ae366
Parents: 0f3412a
Author: jinossy <ji...@gmail.com>
Authored: Tue Aug 5 17:14:47 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Tue Aug 5 17:14:47 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../java/org/apache/tajo/conf/TajoConf.java | 6 +--
.../apache/tajo/engine/function/math/Round.java | 2 +-
.../querymaster/QueryMasterManagerService.java | 5 +-
.../java/org/apache/tajo/worker/Fetcher.java | 31 +++++++----
.../main/java/org/apache/tajo/worker/Task.java | 2 +-
.../org/apache/tajo/worker/TestFetcher.java | 29 ++++++++++-
.../tajo/pullserver/TajoPullServerService.java | 54 ++++++++++----------
8 files changed, 87 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/242d6ad6/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index be71bf4..e1d6d03 100644
--- a/CHANGES
+++ b/CHANGES
@@ -105,6 +105,9 @@ Release 0.9.0 - unreleased
(Hyoungjun Kim via hyunsik)
BUG FIXES
+
+ TAJO-949: PullServer does not release files, when a channel throws
+ an internal exception. (jinho)
TAJO-975: alias name which is the same to existing column name may cause
NPE during PPD. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/242d6ad6/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index d5e8bc4..b75530b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -217,9 +217,9 @@ public class TajoConf extends Configuration {
SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false),
SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"),
SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM("tajo.shuffle.fetcher.parallel-execution.max-num", 2),
- SHUFFLE_FETCHER_CHUNK_MAX_SIZE("tajo.shuffle.fetcher.chunk.max-size", 8192 * 8),
- SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 5),
- SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 5),
+ SHUFFLE_FETCHER_CHUNK_MAX_SIZE("tajo.shuffle.fetcher.chunk.max-size", 8192),
+ SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 120),
+ SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 20),
//////////////////////////////////
// Storage Configuration
http://git-wip-us.apache.org/repos/asf/tajo/blob/242d6ad6/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java
index e457791..cdcb70a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java
@@ -42,7 +42,7 @@ import org.apache.tajo.storage.Tuple;
paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT4}),
@ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT8}),
@ParamTypes(paramTypes = {TajoDataTypes.Type.INT4}),
- @ParamTypes(paramTypes = {TajoDataTypes.Type.INT8}),
+ @ParamTypes(paramTypes = {TajoDataTypes.Type.INT8})
}
)
public class Round extends GeneralFunction {
http://git-wip-us.apache.org/repos/asf/tajo/blob/242d6ad6/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
index 826052d..f52d143 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
@@ -204,7 +204,10 @@ public class QueryMasterManagerService extends CompositeService
try {
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(
new QueryId(report.getId().getQueryUnitId().getExecutionBlockId().getQueryId()));
- queryMasterTask.getEventHandler().handle(new TaskCompletionEvent(report));
+ // queryMaster terminated by internal error before task has not done
+ if (queryMasterTask != null) {
+ queryMasterTask.getEventHandler().handle(new TaskCompletionEvent(report));
+ }
done.run(TajoWorker.TRUE_PROTO);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/tajo/blob/242d6ad6/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java b/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java
index 2aa2875..1b95238 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/Fetcher.java
@@ -116,9 +116,9 @@ public class Fetcher {
public File get() throws IOException {
this.startTime = System.currentTimeMillis();
this.state = TajoProtos.FetcherState.FETCH_FETCHING;
-
+ ChannelFuture future = null;
try {
- ChannelFuture future = bootstrap.connect(new InetSocketAddress(host, port));
+ future = bootstrap.connect(new InetSocketAddress(host, port));
// Wait until the connection attempt succeeds or fails.
Channel channel = future.awaitUninterruptibly().getChannel();
@@ -145,10 +145,13 @@ public class Fetcher {
channelFuture.addListener(ChannelFutureListener.CLOSE);
- // Close the channel to exit.
- future.getChannel().close();
return file;
} finally {
+ if(future != null){
+ // Close the channel to exit.
+ future.getChannel().close();
+ }
+
this.finishTime = System.currentTimeMillis();
LOG.info("Status: " + getState() + ", URI:" + uri);
if (timer != null) {
@@ -249,7 +252,6 @@ public class Fetcher {
}
if(fileLen == length){
- IOUtils.cleanup(LOG, fc, raf);
finishTime = System.currentTimeMillis();
state = TajoProtos.FetcherState.FETCH_FINISHED;
}
@@ -265,15 +267,26 @@ public class Fetcher {
LOG.error("Fetch failed :", e.getCause());
}
- if(ctx.getChannel().isConnected()){
- ctx.getChannel().close().setFailure(e.getCause());
- }
-
// this fetching will be retry
IOUtils.cleanup(LOG, fc, raf);
+ if(ctx.getChannel().isConnected()){
+ ctx.getChannel().close();
+ }
finishTime = System.currentTimeMillis();
state = TajoProtos.FetcherState.FETCH_FAILED;
}
+
+ @Override
+ public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ super.channelDisconnected(ctx, e);
+
+ if(getState() != TajoProtos.FetcherState.FETCH_FINISHED){
+ //channel is closed, but cannot complete fetcher
+ finishTime = System.currentTimeMillis();
+ state = TajoProtos.FetcherState.FETCH_FAILED;
+ }
+ IOUtils.cleanup(LOG, fc, raf);
+ }
}
class HttpClientPipelineFactory implements
http://git-wip-us.apache.org/repos/asf/tajo/blob/242d6ad6/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
index 3a4536a..195b35e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
@@ -626,7 +626,7 @@ public class Task {
if (fetcher.getState() == TajoProtos.FetcherState.FETCH_FINISHED && fetched != null) {
break;
}
- } catch (IOException e) {
+ } catch (Throwable e) {
LOG.error("Fetch failed: " + fetcher.getURI(), e);
}
retryNum++;
http://git-wip-us.apache.org/repos/asf/tajo/blob/242d6ad6/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java
index 82d662b..c13842b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java
@@ -36,8 +36,7 @@ import java.io.IOException;
import java.net.URI;
import java.util.Random;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.*;
public class TestFetcher {
private String TEST_DATA = "target/test-data/TestFetcher";
@@ -195,4 +194,30 @@ public class TestFetcher {
fetcher.get();
assertEquals(TajoProtos.FetcherState.FETCH_FAILED, fetcher.getState());
}
+
+ @Test
+ public void testServerFailure() throws Exception {
+ QueryId queryId = QueryIdFactory.NULL_QUERY_ID;
+ String sid = "1";
+ String ta = "1_0";
+ String partId = "1";
+
+ String dataPath = INPUT_DIR + queryId.toString() + "/output"+ "/" + sid + "/" +ta + "/output/" + partId;
+ String params = String.format("qid=%s&sid=%s&p=%s&type=%s&ta=%s", queryId, sid, partId, "h", ta);
+
+ URI uri = URI.create("http://127.0.0.1:" + pullServerService.getPort() + "/?" + params);
+ final Fetcher fetcher = new Fetcher(conf, uri, new File(OUTPUT_DIR + "data"), channelFactory);
+ assertEquals(TajoProtos.FetcherState.FETCH_INIT, fetcher.getState());
+
+ pullServerService.stop();
+
+ boolean failure = false;
+ try{
+ fetcher.get();
+ } catch (Throwable e){
+ failure = true;
+ }
+ assertTrue(failure);
+ assertEquals(TajoProtos.FetcherState.FETCH_FAILED, fetcher.getState());
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/242d6ad6/tajo-yarn-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
----------------------------------------------------------------------
diff --git a/tajo-yarn-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java b/tajo-yarn-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
index 12cd1a3..3b0ee1f 100644
--- a/tajo-yarn-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
+++ b/tajo-yarn-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
@@ -57,7 +57,6 @@ import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.*;
import org.jboss.netty.channel.group.ChannelGroup;
import org.jboss.netty.channel.group.DefaultChannelGroup;
-import org.jboss.netty.handler.codec.frame.TooLongFrameException;
import org.jboss.netty.handler.codec.http.*;
import org.jboss.netty.handler.ssl.SslHandler;
import org.jboss.netty.handler.stream.ChunkedWriteHandler;
@@ -491,27 +490,33 @@ public class TajoPullServerService extends AbstractService {
private ChannelFuture sendFile(ChannelHandlerContext ctx,
Channel ch,
FileChunk file) throws IOException {
- RandomAccessFile spill;
+ RandomAccessFile spill = null;
+ ChannelFuture writeFuture;
try {
spill = new RandomAccessFile(file.getFile(), "r");
+ if (ch.getPipeline().get(SslHandler.class) == null) {
+ final FadvisedFileRegionWrapper filePart = new FadvisedFileRegionWrapper(spill,
+ file.startOffset, file.length(), manageOsCache, readaheadLength,
+ readaheadPool, file.getFile().getAbsolutePath());
+ writeFuture = ch.write(filePart);
+ writeFuture.addListener(new FileCloseListener(filePart));
+ } else {
+ // HTTPS cannot be done with zero copy.
+ final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,
+ file.startOffset, file.length, sslFileBufferSize,
+ manageOsCache, readaheadLength, readaheadPool,
+ file.getFile().getAbsolutePath());
+ writeFuture = ch.write(chunk);
+ }
} catch (FileNotFoundException e) {
LOG.info(file.getFile() + " not found");
return null;
- }
- ChannelFuture writeFuture;
- if (ch.getPipeline().get(SslHandler.class) == null) {
- final FadvisedFileRegionWrapper filePart = new FadvisedFileRegionWrapper(spill,
- file.startOffset, file.length(), manageOsCache, readaheadLength,
- readaheadPool, file.getFile().getAbsolutePath());
- writeFuture = ch.write(filePart);
- writeFuture.addListener(new FileCloseListener(filePart));
- } else {
- // HTTPS cannot be done with zero copy.
- final FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill,
- file.startOffset, file.length, sslFileBufferSize,
- manageOsCache, readaheadLength, readaheadPool,
- file.getFile().getAbsolutePath());
- writeFuture = ch.write(chunk);
+ } catch (Throwable e) {
+ if (spill != null) {
+ //should close a opening file
+ spill.close();
+ }
+ return null;
}
metrics.shuffleConnections.incr();
metrics.shuffleOutputBytes.incr(file.length); // optimistic
@@ -537,17 +542,10 @@ public class TajoPullServerService extends AbstractService {
@Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e)
throws Exception {
- Channel ch = e.getChannel();
- Throwable cause = e.getCause();
- if (cause instanceof TooLongFrameException) {
- sendError(ctx, BAD_REQUEST);
- return;
- }
-
- LOG.error("PullServer error: ", cause);
- if (ch.isConnected()) {
- LOG.error("PullServer error " + e);
- sendError(ctx, INTERNAL_SERVER_ERROR);
+ LOG.error(e.getCause().getMessage(), e.getCause());
+ //if channel.close() is not called, never closed files in this request
+ if (ctx.getChannel().isConnected()){
+ ctx.getChannel().close();
}
}
}
[09/17] git commit: TAJO-996: Sometimes,
scheduleFetchesByEvenDistributedVolumes loses some FetchImpls.
Posted by ji...@apache.org.
TAJO-996: Sometimes, scheduleFetchesByEvenDistributedVolumes loses some FetchImpls.
Closes #109
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/189cf3ff
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/189cf3ff
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/189cf3ff
Branch: refs/heads/index_support
Commit: 189cf3ffe9d1b5c625c553f27fc5be37cd7522be
Parents: 912df6f
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 11 10:54:20 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 11 10:54:20 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../tajo/master/querymaster/Repartitioner.java | 1 +
.../java/org/apache/tajo/worker/FetchImpl.java | 23 +++++++++
.../apache/tajo/master/TestRepartitioner.java | 52 +++++++++++++++-----
4 files changed, 68 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/189cf3ff/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index e1d6d03..7487983 100644
--- a/CHANGES
+++ b/CHANGES
@@ -106,6 +106,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-996: Sometimes, scheduleFetchesByEvenDistributedVolumes loses
+ some FetchImpls. (hyunsik)
+
TAJO-949: PullServer does not release files, when a channel throws
an internal exception. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/189cf3ff/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index 43d6fd2..fa1ed4c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -831,6 +831,7 @@ public class Repartitioner {
while (p > 0 && iterator.hasNext()) {
FetchGroupMeta fetchGroupMeta = iterator.next();
assignedVolumes[p] += fetchGroupMeta.getVolume();
+ TUtil.putCollectionToNestedList(fetchesArray[p], tableName, fetchGroupMeta.fetchUrls);
// While the current one is smaller than next one, it adds additional fetches to current one.
while(iterator.hasNext() && assignedVolumes[p - 1] > assignedVolumes[p]) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/189cf3ff/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
index 869c106..7baae64 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
@@ -26,6 +26,7 @@ import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.querymaster.QueryUnit;
import org.apache.tajo.master.querymaster.Repartitioner;
+import org.apache.tajo.util.TUtil;
import java.net.URI;
import java.util.ArrayList;
@@ -220,4 +221,26 @@ public class FetchImpl implements ProtoObject<TajoWorkerProtocol.FetchProto>, Cl
}
return newFetchImpl;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ FetchImpl fetch = (FetchImpl) o;
+
+ return TUtil.checkEquals(hasNext, fetch.hasNext) &&
+ TUtil.checkEquals(partitionId, fetch.partitionId) &&
+ TUtil.checkEquals(attemptIds, fetch.attemptIds) &&
+ TUtil.checkEquals(executionBlockId, fetch.executionBlockId) &&
+ TUtil.checkEquals(host, fetch.host) &&
+ TUtil.checkEquals(name, fetch.name) &&
+ TUtil.checkEquals(rangeParams, fetch.rangeParams) &&
+ TUtil.checkEquals(taskIds, fetch.taskIds) &&
+ TUtil.checkEquals(type, fetch.type);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/189cf3ff/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
index 009c02e..29aeccd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
@@ -19,7 +19,9 @@
package org.apache.tajo.master;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.QueryId;
import org.apache.tajo.TestTajoIds;
import org.apache.tajo.ipc.TajoWorkerProtocol;
@@ -32,12 +34,10 @@ import org.jboss.netty.handler.codec.http.QueryStringDecoder;
import org.junit.Test;
import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import static junit.framework.Assert.assertEquals;
+import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType;
import static org.apache.tajo.master.querymaster.Repartitioner.FetchGroupMeta;
import static org.junit.Assert.assertTrue;
@@ -55,7 +55,7 @@ public class TestRepartitioner {
intermediateEntries.add(new QueryUnit.IntermediateEntry(i, 0, partitionId, new QueryUnit.PullHost(hostName, port)));
}
- FetchImpl fetch = new FetchImpl(new QueryUnit.PullHost(hostName, port), TajoWorkerProtocol.ShuffleType.HASH_SHUFFLE,
+ FetchImpl fetch = new FetchImpl(new QueryUnit.PullHost(hostName, port), ShuffleType.HASH_SHUFFLE,
sid, partitionId, intermediateEntries);
fetch.setName(sid.toString());
@@ -95,38 +95,49 @@ public class TestRepartitioner {
String tableName = "test1";
- fetchGroups.put(0, new FetchGroupMeta(100, new FetchImpl()));
- fetchGroups.put(1, new FetchGroupMeta(80, new FetchImpl()));
- fetchGroups.put(2, new FetchGroupMeta(70, new FetchImpl()));
- fetchGroups.put(3, new FetchGroupMeta(30, new FetchImpl()));
- fetchGroups.put(4, new FetchGroupMeta(10, new FetchImpl()));
- fetchGroups.put(5, new FetchGroupMeta(5, new FetchImpl()));
+ ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
+ FetchImpl [] fetches = new FetchImpl[12];
+ for (int i = 0; i < 12; i++) {
+ fetches[i] = new FetchImpl(new QueryUnit.PullHost("localhost", 10000 + i), ShuffleType.HASH_SHUFFLE, ebId, i / 2);
+ }
+
+ int [] VOLUMES = {100, 80, 70, 30, 10, 5};
+
+ for (int i = 0; i < 12; i += 2) {
+ fetchGroups.put(i, new FetchGroupMeta(VOLUMES[i / 2], fetches[i]).addFetche(fetches[i + 1]));
+ }
Pair<Long [], Map<String, List<FetchImpl>>[]> results;
results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 1);
long expected [] = {100 + 80 + 70 + 30 + 10 + 5};
assertFetchVolumes(expected, results.getFirst());
+ assertFetchImpl(fetches, results.getSecond());
results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 2);
long expected0 [] = {130, 165};
assertFetchVolumes(expected0, results.getFirst());
+ assertFetchImpl(fetches, results.getSecond());
results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 3);
long expected1 [] = {100, 95, 100};
assertFetchVolumes(expected1, results.getFirst());
+ assertFetchImpl(fetches, results.getSecond());
results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 4);
long expected2 [] = {100, 80, 70, 45};
assertFetchVolumes(expected2, results.getFirst());
+ assertFetchImpl(fetches, results.getSecond());
results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 5);
long expected3 [] = {100, 80, 70, 30, 15};
assertFetchVolumes(expected3, results.getFirst());
+ assertFetchImpl(fetches, results.getSecond());
results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 6);
long expected4 [] = {100, 80, 70, 30, 10, 5};
assertFetchVolumes(expected4, results.getFirst());
+ assertFetchImpl(fetches, results.getSecond());
}
private static void assertFetchVolumes(long [] expected, Long [] results) {
@@ -136,4 +147,23 @@ public class TestRepartitioner {
assertTrue(expected[i] + " is expected, but " + results[i], expected[i] == results[i]);
}
}
+
+ private static void assertFetchImpl(FetchImpl [] expected, Map<String, List<FetchImpl>>[] result) {
+ Set<FetchImpl> expectedURLs = Sets.newHashSet();
+
+ for (FetchImpl f : expected) {
+ expectedURLs.add(f);
+ }
+
+ Set<FetchImpl> resultURLs = Sets.newHashSet();
+
+ for (Map<String, List<FetchImpl>> e : result) {
+ for (List<FetchImpl> list : e.values()) {
+ resultURLs.addAll(list);
+ }
+ }
+
+ assertEquals(expectedURLs.size(), resultURLs.size());
+ assertEquals(expectedURLs, resultURLs);
+ }
}
[02/17] git commit: TAJO-979: Dividing float value by zero should
throw "Divide by zero Exception" (Hyoungjun Kim via hyunsik)
Posted by ji...@apache.org.
TAJO-979: Dividing float value by zero should throw "Divide by zero Exception" (Hyoungjun Kim via hyunsik)
Closes #100
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/072b5a3a
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/072b5a3a
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/072b5a3a
Branch: refs/heads/index_support
Commit: 072b5a3ad2f21189ab3bdf159e666086957401d6
Parents: 650157c
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri Aug 1 17:16:20 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri Aug 1 17:18:00 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../java/org/apache/tajo/conf/TajoConf.java | 5 +-
.../main/java/org/apache/tajo/datum/Datum.java | 72 ++++
.../java/org/apache/tajo/datum/Float4Datum.java | 60 ++-
.../java/org/apache/tajo/datum/Float8Datum.java | 78 +++-
.../java/org/apache/tajo/datum/Int2Datum.java | 76 +++-
.../java/org/apache/tajo/datum/Int4Datum.java | 78 +++-
.../java/org/apache/tajo/datum/Int8Datum.java | 78 +++-
.../org/apache/tajo/datum/IntervalDatum.java | 16 +-
.../tajo/datum/TestArithmeticOperator.java | 377 +++++++++++++++++++
.../tajo/engine/function/TestMathFunctions.java | 15 +-
11 files changed, 759 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 76cfff7..705e91e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -97,6 +97,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-979: Dividing float value by zero should throw "Divide by zero
+ Exception" (Hyoungjun Kim via hyunsik)
+
TAJO-978: RoundFloat8 should return Float8Datum type. (Hyoungjun Kim via
hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index a8e6d8d..d5e8bc4 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -344,7 +344,10 @@ public class TajoConf extends Configuration {
TAJO_DEBUG("tajo.debug", false),
// ONLY FOR TESTCASE
- TESTCASE_MIN_TASK_NUM("tajo.testcase.min.task.num", -1)
+ TESTCASE_MIN_TASK_NUM("tajo.testcase.min.task.num", -1),
+
+ // behavior control
+ BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false);
;
public final String varname;
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
index af88ebe..19acafc 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
@@ -19,6 +19,8 @@
package org.apache.tajo.datum;
import com.google.gson.annotations.Expose;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.exception.InvalidCastException;
import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.json.CommonGsonHelper;
@@ -27,6 +29,12 @@ import org.apache.tajo.json.GsonObject;
import static org.apache.tajo.common.TajoDataTypes.Type;
public abstract class Datum implements Comparable<Datum>, GsonObject {
+ static boolean abortWhenDivideByZero;
+
+ static {
+ initAbortWhenDivideByZero(new TajoConf());
+ }
+
@Expose private final Type type;
public Datum(Type type) {
@@ -105,6 +113,10 @@ public abstract class Datum implements Comparable<Datum>, GsonObject {
this.type == Type.FLOAT8;
}
+ protected static void initAbortWhenDivideByZero(TajoConf tajoConf) {
+ abortWhenDivideByZero = tajoConf.getBoolVar(ConfVars.BEHAVIOR_ARITHMETIC_ABORT);
+ }
+
public abstract int size();
public Datum and(Datum datum) {
@@ -190,4 +202,64 @@ public abstract class Datum implements Comparable<Datum>, GsonObject {
public String toString() {
return asChars();
}
+
+ protected boolean validateDivideZero(short value) {
+ if (value == (short)0) {
+ if (abortWhenDivideByZero) {
+ throw new ArithmeticException("/ by zero");
+ } else {
+ return false;
+ }
+ } else {
+ return true;
+ }
+ }
+
+ protected boolean validateDivideZero(int value) {
+ if (value == 0) {
+ if (abortWhenDivideByZero) {
+ throw new ArithmeticException("/ by zero");
+ } else {
+ return false;
+ }
+ } else {
+ return true;
+ }
+ }
+
+ protected boolean validateDivideZero(long value) {
+ if (value == 0L) {
+ if (abortWhenDivideByZero) {
+ throw new ArithmeticException("/ by zero");
+ } else {
+ return false;
+ }
+ } else {
+ return true;
+ }
+ }
+
+ protected boolean validateDivideZero(float value) {
+ if (value == 0.0f) {
+ if (abortWhenDivideByZero) {
+ throw new ArithmeticException("/ by zero");
+ } else {
+ return false;
+ }
+ } else {
+ return true;
+ }
+ }
+
+ protected boolean validateDivideZero(double value) {
+ if (value == 0.0) {
+ if (abortWhenDivideByZero) {
+ throw new ArithmeticException("/ by zero");
+ } else {
+ return false;
+ }
+ } else {
+ return true;
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java
index 610ea95..8b16528 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java
@@ -277,15 +277,35 @@ public class Float4Datum extends NumericDatum {
public Datum divide(Datum datum) {
switch (datum.type()) {
case INT2:
- return DatumFactory.createFloat4(val / datum.asInt2());
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val / paramValueI2);
case INT4:
- return DatumFactory.createFloat4(val / datum.asInt4());
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val / paramValueI4);
case INT8:
- return DatumFactory.createFloat4(val / datum.asInt8());
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val / paramValueI8);
case FLOAT4:
- return DatumFactory.createFloat4(val / datum.asFloat4());
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val / paramValueF4);
case FLOAT8:
- return DatumFactory.createFloat8(val / datum.asFloat8());
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueF8);
case NULL_TYPE:
return datum;
default:
@@ -297,15 +317,35 @@ public class Float4Datum extends NumericDatum {
public Datum modular(Datum datum) {
switch (datum.type()) {
case INT2:
- return DatumFactory.createFloat4(val % datum.asInt2());
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val % paramValueI2);
case INT4:
- return DatumFactory.createFloat4(val % datum.asInt4());
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val % paramValueI4);
case INT8:
- return DatumFactory.createFloat4(val % datum.asInt8());
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val % paramValueI8);
case FLOAT4:
- return DatumFactory.createFloat4(val % datum.asFloat4());
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val % paramValueF4);
case FLOAT8:
- return DatumFactory.createFloat8(val % datum.asFloat8());
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueF8);
case NULL_TYPE:
return datum;
default:
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java
index 90adcc7..d41af2d 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java
@@ -265,20 +265,40 @@ public class Float8Datum extends NumericDatum {
@Override
public Datum divide(Datum datum) {
switch (datum.type()) {
- case INT2:
- return DatumFactory.createFloat8(val / datum.asInt2());
- case INT4:
- return DatumFactory.createFloat8(val / datum.asInt4());
- case INT8:
- return DatumFactory.createFloat8(val / datum.asInt8());
- case FLOAT4:
- return DatumFactory.createFloat8(val / datum.asFloat4());
- case FLOAT8:
- return DatumFactory.createFloat8(val / datum.asFloat8());
- case NULL_TYPE:
- return datum;
- default:
- throw new InvalidOperationException(datum.type());
+ case INT2:
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueI2);
+ case INT4:
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueI4);
+ case INT8:
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueI8);
+ case FLOAT4:
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueF4);
+ case FLOAT8:
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueF8);
+ case NULL_TYPE:
+ return datum;
+ default:
+ throw new InvalidOperationException(datum.type());
}
}
@@ -286,15 +306,35 @@ public class Float8Datum extends NumericDatum {
public Datum modular(Datum datum) {
switch (datum.type()) {
case INT2:
- return DatumFactory.createFloat8(val % datum.asInt2());
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueI2);
case INT4:
- return DatumFactory.createFloat8(val % datum.asInt4());
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueI4);
case INT8:
- return DatumFactory.createFloat8(val % datum.asInt8());
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueI8);
case FLOAT4:
- return DatumFactory.createFloat8(val % datum.asFloat4());
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueF4);
case FLOAT8:
- return DatumFactory.createFloat8(val % datum.asFloat8());
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueF8);
case NULL_TYPE:
return datum;
default:
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java
index ab17bdc..2a6c691 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java
@@ -267,20 +267,40 @@ public class Int2Datum extends NumericDatum {
@Override
public Datum divide(Datum datum) {
switch (datum.type()) {
- case INT2:
- return DatumFactory.createInt2((short) (val / datum.asInt2()));
- case INT4:
- return DatumFactory.createInt4(val / datum.asInt4());
- case INT8:
- return DatumFactory.createInt8(val / datum.asInt8());
- case FLOAT4:
- return DatumFactory.createFloat4(val / datum.asFloat4());
- case FLOAT8:
- return DatumFactory.createFloat8(val / datum.asFloat8());
- case NULL_TYPE:
- return datum;
- default:
- throw new InvalidOperationException(datum.type());
+ case INT2:
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt2((short) (val / paramValueI2));
+ case INT4:
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt4(val / paramValueI4);
+ case INT8:
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val / paramValueI8);
+ case FLOAT4:
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val / paramValueF4);
+ case FLOAT8:
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueF8);
+ case NULL_TYPE:
+ return datum;
+ default:
+ throw new InvalidOperationException(datum.type());
}
}
@@ -288,14 +308,34 @@ public class Int2Datum extends NumericDatum {
public Datum modular(Datum datum) {
switch (datum.type()) {
case INT2:
- return DatumFactory.createInt2((short) (val % datum.asInt2()));
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt2((short) (val % paramValueI2));
case INT4:
- return DatumFactory.createInt4(val % datum.asInt4());
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt4(val % paramValueI4);
case INT8:
- return DatumFactory.createInt8(val % datum.asInt8());
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val % paramValueI8);
case FLOAT4:
- return DatumFactory.createFloat4(val % datum.asFloat4());
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val % paramValueF4);
case FLOAT8:
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
return DatumFactory.createFloat8(val % datum.asFloat8());
case NULL_TYPE:
return datum;
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java
index 9a60863..26201fb 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java
@@ -271,20 +271,40 @@ public class Int4Datum extends NumericDatum {
@Override
public Datum divide(Datum datum) {
switch (datum.type()) {
- case INT2:
- return DatumFactory.createInt4(val / datum.asInt2());
- case INT4:
- return DatumFactory.createInt4(val / datum.asInt4());
- case INT8:
- return DatumFactory.createInt8(val / datum.asInt8());
- case FLOAT4:
- return DatumFactory.createFloat4(val / datum.asFloat4());
- case FLOAT8:
- return DatumFactory.createFloat8(val / datum.asFloat8());
- case NULL_TYPE:
- return datum;
- default:
- throw new InvalidOperationException(datum.type());
+ case INT2:
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt4(val / paramValueI2);
+ case INT4:
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt4(val / paramValueI4);
+ case INT8:
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val / paramValueI8);
+ case FLOAT4:
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val / paramValueF4);
+ case FLOAT8:
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueF8);
+ case NULL_TYPE:
+ return datum;
+ default:
+ throw new InvalidOperationException(datum.type());
}
}
@@ -292,15 +312,35 @@ public class Int4Datum extends NumericDatum {
public Datum modular(Datum datum) {
switch (datum.type()) {
case INT2:
- return DatumFactory.createInt4(val % datum.asInt2());
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt4(val % paramValueI2);
case INT4:
- return DatumFactory.createInt4(val % datum.asInt4());
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt4(val % paramValueI4);
case INT8:
- return DatumFactory.createInt8(val % datum.asInt8());
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val % paramValueI8);
case FLOAT4:
- return DatumFactory.createFloat4(val % datum.asFloat4());
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat4(val % paramValueF4);
case FLOAT8:
- return DatumFactory.createFloat8(val % datum.asFloat8());
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueF8);
case NULL_TYPE:
return datum;
default:
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java
index db8a12b..9fa41d1 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java
@@ -280,20 +280,40 @@ public class Int8Datum extends NumericDatum {
@Override
public Datum divide(Datum datum) {
switch (datum.type()) {
- case INT2:
- return DatumFactory.createInt8(val / datum.asInt2());
- case INT4:
- return DatumFactory.createInt8(val / datum.asInt4());
- case INT8:
- return DatumFactory.createInt8(val / datum.asInt8());
- case FLOAT4:
- return DatumFactory.createFloat8(val / datum.asFloat4());
- case FLOAT8:
- return DatumFactory.createFloat8(val / datum.asFloat8());
- case NULL_TYPE:
- return datum;
- default:
- throw new InvalidOperationException(datum.type());
+ case INT2:
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val / paramValueI2);
+ case INT4:
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val / paramValueI4);
+ case INT8:
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val / paramValueI8);
+ case FLOAT4:
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueF4);
+ case FLOAT8:
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val / paramValueF8);
+ case NULL_TYPE:
+ return datum;
+ default:
+ throw new InvalidOperationException(datum.type());
}
}
@@ -301,15 +321,35 @@ public class Int8Datum extends NumericDatum {
public Datum modular(Datum datum) {
switch (datum.type()) {
case INT2:
- return DatumFactory.createInt8(val % datum.asInt2());
+ short paramValueI2 = datum.asInt2();
+ if (!validateDivideZero(paramValueI2)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val % paramValueI2);
case INT4:
- return DatumFactory.createInt8(val % datum.asInt4());
+ int paramValueI4 = datum.asInt4();
+ if (!validateDivideZero(paramValueI4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val % paramValueI4);
case INT8:
- return DatumFactory.createInt8(val % datum.asInt8());
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createInt8(val % paramValueI8);
case FLOAT4:
- return DatumFactory.createFloat8(val % datum.asFloat4());
+ float paramValueF4 = datum.asFloat4();
+ if (!validateDivideZero(paramValueF4)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueF4);
case FLOAT8:
- return DatumFactory.createFloat8(val % datum.asFloat8());
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return DatumFactory.createFloat8(val % paramValueF8);
case NULL_TYPE:
return datum;
default:
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
index 28a0a47..c6f3922 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
@@ -289,16 +289,22 @@ public class IntervalDatum extends Datum {
@Override
public Datum divide(Datum datum) {
- switch(datum.type()) {
+ switch (datum.type()) {
case INT2:
case INT4:
case INT8:
- long int8Val = datum.asInt8();
- return createIntervalDatum((double)months / int8Val, (double) millieconds / int8Val);
+ long paramValueI8 = datum.asInt8();
+ if (!validateDivideZero(paramValueI8)) {
+ return NullDatum.get();
+ }
+ return createIntervalDatum((double) months / paramValueI8, (double) millieconds / paramValueI8);
case FLOAT4:
case FLOAT8:
- double float8Val = datum.asFloat8();
- return createIntervalDatum((double)months / float8Val, (double) millieconds / float8Val);
+ double paramValueF8 = datum.asFloat8();
+ if (!validateDivideZero(paramValueF8)) {
+ return NullDatum.get();
+ }
+ return createIntervalDatum((double) months / paramValueF8, (double) millieconds / paramValueF8);
default:
throw new InvalidOperationException(datum.type());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java
new file mode 100644
index 0000000..adf80b0
--- /dev/null
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestArithmeticOperator.java
@@ -0,0 +1,377 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.datum;
+
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(Parameterized.class)
+public class TestArithmeticOperator {
+ String option;
+
+ public TestArithmeticOperator(String option) {
+ this.option = option;
+ }
+
+ @Parameters
+ public static Collection<Object[]> generateParameters() {
+ return Arrays.asList(new Object[][]{
+ {"Zero_Null"},
+ {"Zero_Exception"},
+ });
+ }
+
+ @Before
+ public void setUp() {
+ TajoConf tajoConf = new TajoConf();
+ if ("Zero_Exception".equals(option)) {
+ tajoConf.setBoolVar(ConfVars.BEHAVIOR_ARITHMETIC_ABORT, true);
+ } else {
+ tajoConf.setBoolVar(ConfVars.BEHAVIOR_ARITHMETIC_ABORT, false);
+ }
+ Datum.initAbortWhenDivideByZero(tajoConf);
+ }
+
+ @Test
+ public void testInt2Datum() throws Exception {
+ //plus
+ runAndAssert("plus", new Int2Datum((short)10), new Int2Datum((short)5), new Int2Datum((short)15));
+ runAndAssert("plus", new Int2Datum((short)10), new Int4Datum(5), new Int4Datum(15));
+ runAndAssert("plus", new Int2Datum((short)10), new Int8Datum(5), new Int8Datum(15));
+ runAndAssert("plus", new Int2Datum((short)10), new Float4Datum(5.0f), new Float4Datum(15.0f));
+ runAndAssert("plus", new Int2Datum((short)10), new Float8Datum(5.0), new Float8Datum(15.0));
+
+ //minus
+ runAndAssert("minus", new Int2Datum((short)10), new Int2Datum((short)5), new Int2Datum((short)5));
+ runAndAssert("minus", new Int2Datum((short)10), new Int4Datum(5), new Int4Datum(5));
+ runAndAssert("minus", new Int2Datum((short)10), new Int8Datum(5), new Int8Datum(5));
+ runAndAssert("minus", new Int2Datum((short)10), new Float4Datum(5.0f), new Float4Datum(5.0f));
+ runAndAssert("minus", new Int2Datum((short)10), new Float8Datum(5.0), new Float8Datum(5.0));
+
+ runAndAssert("minus", new Int2Datum((short)5), new Int2Datum((short)10), new Int2Datum((short)-5));
+ runAndAssert("minus", new Int2Datum((short)5), new Int4Datum(10), new Int4Datum(-5));
+ runAndAssert("minus", new Int2Datum((short)5), new Int8Datum(10), new Int8Datum(-5));
+ runAndAssert("minus", new Int2Datum((short)5), new Float4Datum(10.0f), new Float4Datum(-5.0f));
+ runAndAssert("minus", new Int2Datum((short)5), new Float8Datum(10.0), new Float8Datum(-5.0));
+
+ //multiply
+ runAndAssert("multiply", new Int2Datum((short)10), new Int2Datum((short)5), new Int4Datum(50));
+ runAndAssert("multiply", new Int2Datum((short)10), new Int4Datum(5), new Int4Datum(50));
+ runAndAssert("multiply", new Int2Datum((short)10), new Int8Datum(5), new Int8Datum(50));
+ runAndAssert("multiply", new Int2Datum((short)10), new Float4Datum(5.0f), new Float4Datum(50.0f));
+ runAndAssert("multiply", new Int2Datum((short)10), new Float8Datum(5.0), new Float8Datum(50.0));
+
+ //divide
+ runAndAssert("divide", new Int2Datum((short)10), new Int2Datum((short)5), new Int2Datum((short)2));
+ runAndAssert("divide", new Int2Datum((short)10), new Int4Datum(5), new Int4Datum(2));
+ runAndAssert("divide", new Int2Datum((short)10), new Int8Datum(5), new Int8Datum(2));
+ runAndAssert("divide", new Int2Datum((short)10), new Float4Datum(5.0f), new Float4Datum(2.0f));
+ runAndAssert("divide", new Int2Datum((short)10), new Float8Datum(5.0), new Float8Datum(2.0));
+
+ runAndAssert("divide", new Int2Datum((short)10), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("divide", new Int2Datum((short)10), new Int4Datum(0), NullDatum.get());
+ runAndAssert("divide", new Int2Datum((short)10), new Int8Datum(0), NullDatum.get());
+ runAndAssert("divide", new Int2Datum((short)10), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("divide", new Int2Datum((short)10), new Float8Datum(0.0), NullDatum.get());
+
+ //modular
+ runAndAssert("modular", new Int2Datum((short)10), new Int2Datum((short)3), new Int2Datum((short)1));
+ runAndAssert("modular", new Int2Datum((short)10), new Int4Datum(3), new Int4Datum(1));
+ runAndAssert("modular", new Int2Datum((short)10), new Int8Datum(3), new Int8Datum(1));
+ runAndAssert("modular", new Int2Datum((short)10), new Float4Datum(3.0f), new Float4Datum(1.0f));
+ runAndAssert("modular", new Int2Datum((short)10), new Float8Datum(3.0), new Float8Datum(1.0));
+
+ runAndAssert("modular", new Int2Datum((short)10), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("modular", new Int2Datum((short)10), new Int4Datum(0), NullDatum.get());
+ runAndAssert("modular", new Int2Datum((short)10), new Int8Datum(0), NullDatum.get());
+ runAndAssert("modular", new Int2Datum((short)10), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("modular", new Int2Datum((short)10), new Float8Datum(0.0), NullDatum.get());
+ }
+
+ @Test
+ public void testInt4Datum() throws Exception {
+ //plus
+ runAndAssert("plus", new Int4Datum(10), new Int2Datum((short)5), new Int4Datum(15));
+ runAndAssert("plus", new Int4Datum(10), new Int4Datum(5), new Int4Datum(15));
+ runAndAssert("plus", new Int4Datum(10), new Int8Datum(5), new Int8Datum(15));
+ runAndAssert("plus", new Int4Datum(10), new Float4Datum(5.0f), new Float4Datum(15.0f));
+ runAndAssert("plus", new Int4Datum(10), new Float8Datum(5.0), new Float8Datum(15.0));
+
+ //minus
+ runAndAssert("minus", new Int4Datum(10), new Int2Datum((short)5), new Int4Datum(5));
+ runAndAssert("minus", new Int4Datum(10), new Int4Datum(5), new Int4Datum(5));
+ runAndAssert("minus", new Int4Datum(10), new Int8Datum(5), new Int8Datum(5));
+ runAndAssert("minus", new Int4Datum(10), new Float4Datum(5.0f), new Float4Datum(5.0f));
+ runAndAssert("minus", new Int4Datum(10), new Float8Datum(5.0), new Float8Datum(5.0));
+
+ runAndAssert("minus", new Int4Datum(5), new Int2Datum((short)10), new Int4Datum(-5));
+ runAndAssert("minus", new Int4Datum(5), new Int4Datum(10), new Int4Datum(-5));
+ runAndAssert("minus", new Int4Datum(5), new Int8Datum(10), new Int8Datum(-5));
+ runAndAssert("minus", new Int4Datum(5), new Float4Datum(10.0f), new Float4Datum(-5.0f));
+ runAndAssert("minus", new Int4Datum(5), new Float8Datum(10.0), new Float8Datum(-5.0));
+
+ //multiply
+ runAndAssert("multiply", new Int4Datum(10), new Int2Datum((short)5), new Int4Datum(50));
+ runAndAssert("multiply", new Int4Datum(10), new Int4Datum(5), new Int4Datum(50));
+ runAndAssert("multiply", new Int4Datum(10), new Int8Datum(5), new Int8Datum(50));
+ runAndAssert("multiply", new Int4Datum(10), new Float4Datum(5.0f), new Float4Datum(50.0f));
+ runAndAssert("multiply", new Int4Datum(10), new Float8Datum(5.0), new Float8Datum(50.0));
+
+ //divide
+ runAndAssert("divide", new Int4Datum(10), new Int2Datum((short)5), new Int4Datum(2));
+ runAndAssert("divide", new Int4Datum(10), new Int4Datum(5), new Int4Datum(2));
+ runAndAssert("divide", new Int4Datum(10), new Int8Datum(5), new Int8Datum(2));
+ runAndAssert("divide", new Int4Datum(10), new Float4Datum(5.0f), new Float4Datum(2.0f));
+ runAndAssert("divide", new Int4Datum(10), new Float8Datum(5.0), new Float8Datum(2.0));
+
+ runAndAssert("divide", new Int4Datum(10), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("divide", new Int4Datum(10), new Int4Datum(0), NullDatum.get());
+ runAndAssert("divide", new Int4Datum(10), new Int8Datum(0), NullDatum.get());
+ runAndAssert("divide", new Int4Datum(10), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("divide", new Int4Datum(10), new Float8Datum(0.0), NullDatum.get());
+
+ //modular
+ runAndAssert("modular", new Int4Datum(10), new Int2Datum((short)3), new Int4Datum(1));
+ runAndAssert("modular", new Int4Datum(10), new Int4Datum(3), new Int4Datum(1));
+ runAndAssert("modular", new Int4Datum(10), new Int8Datum(3), new Int8Datum(1));
+ runAndAssert("modular", new Int4Datum(10), new Float4Datum(3.0f), new Float4Datum(1.0f));
+ runAndAssert("modular", new Int4Datum(10), new Float8Datum(3.0), new Float8Datum(1.0));
+
+ runAndAssert("modular", new Int4Datum(10), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("modular", new Int4Datum(10), new Int4Datum(0), NullDatum.get());
+ runAndAssert("modular", new Int4Datum(10), new Int8Datum(0), NullDatum.get());
+ runAndAssert("modular", new Int4Datum(10), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("modular", new Int4Datum(10), new Float8Datum(0.0), NullDatum.get());
+ }
+
+ @Test
+ public void testInt8Datum() throws Exception {
+ //plus
+ runAndAssert("plus", new Int8Datum(10), new Int2Datum((short)5), new Int8Datum(15));
+ runAndAssert("plus", new Int8Datum(10), new Int4Datum(5), new Int8Datum(15));
+ runAndAssert("plus", new Int8Datum(10), new Int8Datum(5), new Int8Datum(15));
+ runAndAssert("plus", new Int8Datum(10), new Float4Datum(5.0f), new Float8Datum(15.0f));
+ runAndAssert("plus", new Int8Datum(10), new Float8Datum(5.0), new Float8Datum(15.0));
+
+ //minus
+ runAndAssert("minus", new Int8Datum(10), new Int2Datum((short)5), new Int8Datum(5));
+ runAndAssert("minus", new Int8Datum(10), new Int4Datum(5), new Int8Datum(5));
+ runAndAssert("minus", new Int8Datum(10), new Int8Datum(5), new Int8Datum(5));
+ runAndAssert("minus", new Int8Datum(10), new Float4Datum(5.0f), new Float8Datum(5.0f));
+ runAndAssert("minus", new Int8Datum(10), new Float8Datum(5.0), new Float8Datum(5.0));
+
+ runAndAssert("minus", new Int8Datum(5), new Int2Datum((short)10), new Int8Datum(-5));
+ runAndAssert("minus", new Int8Datum(5), new Int4Datum(10), new Int8Datum(-5));
+ runAndAssert("minus", new Int8Datum(5), new Int8Datum(10), new Int8Datum(-5));
+ runAndAssert("minus", new Int8Datum(5), new Float4Datum(10.0f), new Float8Datum(-5.0f));
+ runAndAssert("minus", new Int8Datum(5), new Float8Datum(10.0), new Float8Datum(-5.0));
+
+ //multiply
+ runAndAssert("multiply", new Int8Datum(10), new Int2Datum((short)5), new Int8Datum(50));
+ runAndAssert("multiply", new Int8Datum(10), new Int4Datum(5), new Int8Datum(50));
+ runAndAssert("multiply", new Int8Datum(10), new Int8Datum(5), new Int8Datum(50));
+ runAndAssert("multiply", new Int8Datum(10), new Float4Datum(5.0f), new Float8Datum(50.0f));
+ runAndAssert("multiply", new Int8Datum(10), new Float8Datum(5.0), new Float8Datum(50.0));
+
+ //divide
+ runAndAssert("divide", new Int8Datum(10), new Int2Datum((short)5), new Int8Datum(2));
+ runAndAssert("divide", new Int8Datum(10), new Int4Datum(5), new Int8Datum(2));
+ runAndAssert("divide", new Int8Datum(10), new Int8Datum(5), new Int8Datum(2));
+ runAndAssert("divide", new Int8Datum(10), new Float4Datum(5.0f), new Float8Datum(2.0f));
+ runAndAssert("divide", new Int8Datum(10), new Float8Datum(5.0), new Float8Datum(2.0));
+
+ runAndAssert("divide", new Int8Datum(10), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("divide", new Int8Datum(10), new Int4Datum(0), NullDatum.get());
+ runAndAssert("divide", new Int8Datum(10), new Int8Datum(0), NullDatum.get());
+ runAndAssert("divide", new Int8Datum(10), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("divide", new Int8Datum(10), new Float8Datum(0.0), NullDatum.get());
+
+ //modular
+ runAndAssert("modular", new Int8Datum(10), new Int2Datum((short)3), new Int8Datum(1));
+ runAndAssert("modular", new Int8Datum(10), new Int4Datum(3), new Int8Datum(1));
+ runAndAssert("modular", new Int8Datum(10), new Int8Datum(3), new Int8Datum(1));
+ runAndAssert("modular", new Int8Datum(10), new Float4Datum(3.0f), new Float8Datum(1.0f));
+ runAndAssert("modular", new Int8Datum(10), new Float8Datum(3.0), new Float8Datum(1.0));
+
+ runAndAssert("modular", new Int8Datum(10), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("modular", new Int8Datum(10), new Int4Datum(0), NullDatum.get());
+ runAndAssert("modular", new Int8Datum(10), new Int8Datum(0), NullDatum.get());
+ runAndAssert("modular", new Int8Datum(10), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("modular", new Int8Datum(10), new Float8Datum(0.0), NullDatum.get());
+ }
+
+ @Test
+ public void testFloat4Datum() throws Exception {
+ //plus
+ runAndAssert("plus", new Float4Datum(10.0f), new Int2Datum((short)5), new Float4Datum(15.0f));
+ runAndAssert("plus", new Float4Datum(10.0f), new Int4Datum(5), new Float4Datum(15.0f));
+ runAndAssert("plus", new Float4Datum(10.0f), new Int8Datum(5), new Float4Datum(15.0f));
+ runAndAssert("plus", new Float4Datum(10.0f), new Float4Datum(5.0f), new Float4Datum(15.0f));
+ runAndAssert("plus", new Float4Datum(10.0f), new Float8Datum(5.0), new Float8Datum(15.0));
+
+ //minus
+ runAndAssert("minus", new Float4Datum(10.0f), new Int2Datum((short)5), new Float4Datum(5.0f));
+ runAndAssert("minus", new Float4Datum(10.0f), new Int4Datum(5), new Float4Datum(5.0f));
+ runAndAssert("minus", new Float4Datum(10.0f), new Int8Datum(5), new Float4Datum(5.0f));
+ runAndAssert("minus", new Float4Datum(10.0f), new Float4Datum(5.0f), new Float4Datum(5.0f));
+ runAndAssert("minus", new Float4Datum(10.0f), new Float8Datum(5.0), new Float8Datum(5.0));
+
+ runAndAssert("minus", new Float4Datum(5.0f), new Int2Datum((short)10), new Float4Datum(-5.0f));
+ runAndAssert("minus", new Float4Datum(5.0f), new Int4Datum(10), new Float4Datum(-5.0f));
+ runAndAssert("minus", new Float4Datum(5.0f), new Float4Datum(10.0f), new Float4Datum(-5.0f));
+ runAndAssert("minus", new Float4Datum(5.0f), new Float4Datum(10.0f), new Float4Datum(-5.0f));
+ runAndAssert("minus", new Float4Datum(5.0f), new Float8Datum(10.0), new Float8Datum(-5.0));
+
+ //multiply
+ runAndAssert("multiply", new Float4Datum(10.0f), new Int2Datum((short)5), new Float4Datum(50.0f));
+ runAndAssert("multiply", new Float4Datum(10.0f), new Int4Datum(5), new Float4Datum(50.0f));
+ runAndAssert("multiply", new Float4Datum(10.0f), new Int8Datum(5), new Float4Datum(50.0f));
+ runAndAssert("multiply", new Float4Datum(10.0f), new Float4Datum(5.0f), new Float4Datum(50.0f));
+ runAndAssert("multiply", new Float4Datum(10.0f), new Float8Datum(5.0), new Float8Datum(50.0));
+
+ //divide
+ runAndAssert("divide", new Float4Datum(10.0f), new Int2Datum((short)5), new Float4Datum(2.0f));
+ runAndAssert("divide", new Float4Datum(10.0f), new Int4Datum(5), new Float4Datum(2.0f));
+ runAndAssert("divide", new Float4Datum(10.0f), new Int8Datum(5), new Float4Datum(2.0f));
+ runAndAssert("divide", new Float4Datum(10.0f), new Float4Datum(5.0f), new Float4Datum(2.0f));
+ runAndAssert("divide", new Float4Datum(10.0f), new Float8Datum(5.0), new Float8Datum(2.0));
+
+ runAndAssert("divide", new Float4Datum(10.0f), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("divide", new Float4Datum(10.0f), new Int4Datum(0), NullDatum.get());
+ runAndAssert("divide", new Float4Datum(10.0f), new Int8Datum(0), NullDatum.get());
+ runAndAssert("divide", new Float4Datum(10.0f), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("divide", new Float4Datum(10.0f), new Float8Datum(0.0), NullDatum.get());
+
+ //modular
+ runAndAssert("modular", new Float4Datum(10.0f), new Int2Datum((short)3), new Float4Datum(1.0f));
+ runAndAssert("modular", new Float4Datum(10.0f), new Int4Datum(3), new Float4Datum(1.0f));
+ runAndAssert("modular", new Float4Datum(10.0f), new Int8Datum(3), new Float4Datum(1.0f));
+ runAndAssert("modular", new Float4Datum(10.0f), new Float4Datum(3.0f), new Float4Datum(1.0f));
+ runAndAssert("modular", new Float4Datum(10.0f), new Float8Datum(3.0), new Float8Datum(1.0));
+
+ runAndAssert("modular", new Float4Datum(10.0f), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("modular", new Float4Datum(10.0f), new Int4Datum(0), NullDatum.get());
+ runAndAssert("modular", new Float4Datum(10.0f), new Int8Datum(0), NullDatum.get());
+ runAndAssert("modular", new Float4Datum(10.0f), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("modular", new Float4Datum(10.0f), new Float8Datum(0.0), NullDatum.get());
+ }
+
+ @Test
+ public void testFloat8Datum() throws Exception {
+ //plus
+ runAndAssert("plus", new Float8Datum(10.0), new Int2Datum((short)5), new Float8Datum(15.0));
+ runAndAssert("plus", new Float8Datum(10.0), new Int4Datum(5), new Float8Datum(15.0));
+ runAndAssert("plus", new Float8Datum(10.0), new Int8Datum(5), new Float8Datum(15.0));
+ runAndAssert("plus", new Float8Datum(10.0), new Float4Datum(5.0f), new Float8Datum(15.0));
+ runAndAssert("plus", new Float8Datum(10.0), new Float8Datum(5.0), new Float8Datum(15.0));
+
+ //minus
+ runAndAssert("minus", new Float8Datum(10.0), new Int2Datum((short)5), new Float8Datum(5.0));
+ runAndAssert("minus", new Float8Datum(10.0), new Int4Datum(5), new Float8Datum(5.0));
+ runAndAssert("minus", new Float8Datum(10.0), new Int8Datum(5), new Float8Datum(5.0));
+ runAndAssert("minus", new Float8Datum(10.0), new Float4Datum(5.0f), new Float8Datum(5.0));
+ runAndAssert("minus", new Float8Datum(10.0), new Float8Datum(5.0), new Float8Datum(5.0));
+
+ runAndAssert("minus", new Float8Datum(5.0), new Int2Datum((short)10), new Float8Datum(-5.0));
+ runAndAssert("minus", new Float8Datum(5.0), new Int4Datum(10), new Float8Datum(-5.0));
+ runAndAssert("minus", new Float8Datum(5.0), new Float8Datum(10.0), new Float8Datum(-5.0));
+ runAndAssert("minus", new Float8Datum(5.0), new Float8Datum(10.0), new Float8Datum(-5.0));
+ runAndAssert("minus", new Float8Datum(5.0), new Float8Datum(10.0), new Float8Datum(-5.0));
+
+ //multiply
+ runAndAssert("multiply", new Float8Datum(10.0), new Int2Datum((short)5), new Float8Datum(50.0));
+ runAndAssert("multiply", new Float8Datum(10.0), new Int4Datum(5), new Float8Datum(50.0));
+ runAndAssert("multiply", new Float8Datum(10.0), new Int8Datum(5), new Float8Datum(50.0));
+ runAndAssert("multiply", new Float8Datum(10.0), new Float4Datum(5.0f), new Float8Datum(50.0));
+ runAndAssert("multiply", new Float8Datum(10.0), new Float8Datum(5.0), new Float8Datum(50.0));
+
+ //divide
+ runAndAssert("divide", new Float8Datum(10.0), new Int2Datum((short)5), new Float8Datum(2.0));
+ runAndAssert("divide", new Float8Datum(10.0), new Int4Datum(5), new Float8Datum(2.0));
+ runAndAssert("divide", new Float8Datum(10.0), new Int8Datum(5), new Float8Datum(2.0));
+ runAndAssert("divide", new Float8Datum(10.0), new Float4Datum(5.0f), new Float8Datum(2.0));
+ runAndAssert("divide", new Float8Datum(10.0), new Float8Datum(5.0), new Float8Datum(2.0));
+
+ runAndAssert("divide", new Float8Datum(10.0), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("divide", new Float8Datum(10.0), new Int4Datum(0), NullDatum.get());
+ runAndAssert("divide", new Float8Datum(10.0), new Int8Datum(0), NullDatum.get());
+ runAndAssert("divide", new Float8Datum(10.0), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("divide", new Float8Datum(10.0), new Float8Datum(0.0), NullDatum.get());
+
+ //modular
+ runAndAssert("modular", new Float8Datum(10.0), new Int2Datum((short)3), new Float8Datum(1.0));
+ runAndAssert("modular", new Float8Datum(10.0), new Int4Datum(3), new Float8Datum(1.0));
+ runAndAssert("modular", new Float8Datum(10.0), new Int8Datum(3), new Float8Datum(1.0));
+ runAndAssert("modular", new Float8Datum(10.0), new Float4Datum(3.0f), new Float8Datum(1.0));
+ runAndAssert("modular", new Float8Datum(10.0), new Float8Datum(3.0), new Float8Datum(1.0));
+
+ //modular by zero
+ runAndAssert("modular", new Float8Datum(10.0), new Int2Datum((short) 0), NullDatum.get());
+ runAndAssert("modular", new Float8Datum(10.0), new Int4Datum(0), NullDatum.get());
+ runAndAssert("modular", new Float8Datum(10.0), new Int8Datum(0), NullDatum.get());
+ runAndAssert("modular", new Float8Datum(10.0), new Float4Datum(0.0f), NullDatum.get());
+ runAndAssert("modular", new Float8Datum(10.0), new Float8Datum(0.0), NullDatum.get());
+ }
+
+ private void runAndAssert(String op, Datum left, Datum right, Datum expected) throws Exception {
+ Datum result = null;
+ try {
+ if ("plus".equals(op)) {
+ result = left.plus(right);
+ } else if ("minus".equals(op)) {
+ result = left.minus(right);
+ } else if ("multiply".equals(op)) {
+ result = left.multiply(right);
+ } else if ("divide".equals(op)) {
+ result = left.divide(right);
+ if (right.asFloat8() == 0) {
+ if (Datum.abortWhenDivideByZero) {
+ fail("should throw DivideByZeroException in the case of nullIfZero false");
+ }
+ }
+ } else if ("modular".equals(op)) {
+ result = left.modular(right);
+ if (right.asFloat8() == 0) {
+ if (Datum.abortWhenDivideByZero) {
+ fail("should throw DivideByZeroException in the case of nullIfZero false");
+ }
+ }
+ }
+ assertEquals(expected.type(), result.type());
+ assertEquals(expected, result);
+ } catch (ArithmeticException e) {
+ if (!Datum.abortWhenDivideByZero) {
+ fail(op + " throws DivideByZeroException");
+ } else {
+ //success
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/072b5a3a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java
index 8068180..e54c40c 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestMathFunctions.java
@@ -20,13 +20,11 @@ package org.apache.tajo.engine.function;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.engine.eval.ExprTestBase;
-import org.apache.tajo.exception.InvalidOperationException;
import org.junit.Test;
import java.io.IOException;
import static org.apache.tajo.common.TajoDataTypes.Type.*;
-import static org.junit.Assert.fail;
public class TestMathFunctions extends ExprTestBase {
@Test
@@ -440,13 +438,14 @@ public class TestMathFunctions extends ExprTestBase {
@Test
public void testRoundWithSpecifiedPrecision() throws IOException {
+ // TODO - in order to make this test possible, testSimpleEval should take session variables. Now, we disable it.
// divide zero
- try {
- testSimpleEval("select round(10.0/0.0,2) ", new String[]{""});
- fail("10.0/0 should throw InvalidOperationException");
- } catch (InvalidOperationException e) {
- //success
- }
+// try {
+// testSimpleEval("select round(10.0/0.0,2) ", new String[]{""});
+// fail("10.0/0 should throw InvalidOperationException");
+// } catch (InvalidOperationException e) {
+// //success
+// }
testSimpleEval("select round(42.4382,2) ", new String[]{"42.44"});
testSimpleEval("select round(-42.4382,2) ", new String[]{"-42.44"});