You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2014/07/23 02:48:17 UTC
[1/7] TAJO-965: Upgrade Bytes class and move some methods to others.
Repository: tajo
Updated Branches:
refs/heads/index_support 334ea2ef9 -> c9b8f511e
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java b/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
index 5c13f8f..6c732c7 100644
--- a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
+++ b/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
@@ -22,8 +22,7 @@ import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.*;
public class TestStringUtil {
@@ -89,4 +88,41 @@ public class TestStringUtil {
assertEquals("\\u0020", StringUtils.unicodeEscapedDelimiter(spaceDelimiter));
assertEquals(spaceDelimiter, StringEscapeUtils.unescapeJava(StringUtils.unicodeEscapedDelimiter(spaceDelimiter)));
}
+
+ @Test
+ public void testAsciiBytes() {
+ String asciiText = "abcde 12345 ABCDE";
+ assertArrayEquals(asciiText.getBytes(), BytesUtils.toASCIIBytes(asciiText.toCharArray()));
+ }
+
+ @Test
+ public void testSplitBytes() {
+ String text = "abcde|12345|ABCDE";
+ char separatorChar = '|';
+
+ String[] textArray = org.apache.commons.lang.StringUtils.splitPreserveAllTokens(text, separatorChar);
+ byte[][] bytesArray = BytesUtils.splitPreserveAllTokens(text.getBytes(), separatorChar);
+
+ assertEquals(textArray.length, bytesArray.length);
+ for (int i = 0; i < textArray.length; i++){
+ assertArrayEquals(textArray[i].getBytes(), bytesArray[i]);
+ }
+ }
+
+ @Test
+ public void testSplitProjectionBytes() {
+ String text = "abcde|12345|ABCDE";
+ int[] target = new int[]{ 1 };
+ char separatorChar = '|';
+
+ String[] textArray = org.apache.commons.lang.StringUtils.splitPreserveAllTokens(text, separatorChar);
+ byte[][] bytesArray = BytesUtils.splitPreserveAllTokens(text.getBytes(), separatorChar, target);
+
+ assertEquals(textArray.length, bytesArray.length);
+
+ assertNull(bytesArray[0]);
+ assertNotNull(bytesArray[1]);
+ assertArrayEquals(textArray[1].getBytes(), bytesArray[1]);
+ assertNull(bytesArray[2]);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 0742a80..ad80ddf 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -38,10 +38,10 @@ import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.LazyTuple;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.VTuple;
-import org.apache.tajo.util.Bytes;
+import org.apache.tajo.util.BytesUtils;
import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.datetime.DateTimeUtil;
import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.datetime.DateTimeUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -170,7 +170,7 @@ public class ExprTestBase {
}
lazyTuple =
- new LazyTuple(inputSchema, Bytes.splitPreserveAllTokens(csvTuple.getBytes(), delimiter, targetIdx),0);
+ new LazyTuple(inputSchema, BytesUtils.splitPreserveAllTokens(csvTuple.getBytes(), delimiter, targetIdx),0);
vtuple = new VTuple(inputSchema.size());
for (int i = 0; i < inputSchema.size(); i++) {
// If null value occurs, null datum is manually inserted to an input tuple.
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index 17b9229..8e26ec6 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -40,7 +40,7 @@ import org.apache.tajo.storage.compress.CodecPool;
import org.apache.tajo.storage.exception.AlreadyExistsStorageException;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.rcfile.NonSyncByteArrayOutputStream;
-import org.apache.tajo.util.Bytes;
+import org.apache.tajo.util.BytesUtils;
import java.io.*;
import java.util.ArrayList;
@@ -466,8 +466,8 @@ public class CSVFile {
offset = fileOffsets.get(currentIdx);
}
- byte[][] cells = Bytes.splitPreserveAllTokens(buffer.getData(), startOffsets.get(currentIdx),
- rowLengthList.get(currentIdx), delimiter, targetColumnIndexes);
+ byte[][] cells = BytesUtils.splitPreserveAllTokens(buffer.getData(), startOffsets.get(currentIdx),
+ rowLengthList.get(currentIdx), delimiter, targetColumnIndexes);
currentIdx++;
return new LazyTuple(schema, cells, offset, nullChars, serde);
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java
index 2068260..db36771 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java
@@ -35,7 +35,6 @@ import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.storage.exception.AlreadyExistsStorageException;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.BitArray;
-import org.apache.tajo.util.Bytes;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -121,7 +120,7 @@ public class RowFile {
private void readHeader() throws IOException {
SYNC_INTERVAL = in.readInt();
- Bytes.readFully(in, this.sync, 0, SYNC_HASH_SIZE);
+ StorageUtil.readFully(in, this.sync, 0, SYNC_HASH_SIZE);
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 95bb96f..5b2d711 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -32,7 +32,10 @@ import org.apache.tajo.util.KeyValueSet;
import parquet.hadoop.ParquetOutputFormat;
import sun.nio.ch.DirectBuffer;
+import java.io.DataInput;
+import java.io.EOFException;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
@@ -199,4 +202,42 @@ public class StorageUtil extends StorageConstants {
}
}
}
+
+ public static int readFully(InputStream is, byte[] buffer, int offset, int length)
+ throws IOException {
+ int nread = 0;
+ while (nread < length) {
+ int nbytes = is.read(buffer, offset + nread, length - nread);
+ if (nbytes < 0) {
+ return nread > 0 ? nread : nbytes;
+ }
+ nread += nbytes;
+ }
+ return nread;
+ }
+
+ /**
+ * Similar to readFully(). Skips bytes in a loop.
+ * @param in The DataInput to skip bytes from
+ * @param len number of bytes to skip.
+ * @throws java.io.IOException if it could not skip requested number of bytes
+ * for any reason (including EOF)
+ */
+ public static void skipFully(DataInput in, int len) throws IOException {
+ int amt = len;
+ while (amt > 0) {
+ long ret = in.skipBytes(amt);
+ if (ret == 0) {
+ // skip may return 0 even if we're not at EOF. Luckily, we can
+ // use the read() method to figure out if we're at the end.
+ int b = in.readByte();
+ if (b == -1) {
+ throw new EOFException( "Premature EOF from inputStream after " +
+ "skipping " + (len - amt) + " byte(s).");
+ }
+ ret = 1;
+ }
+ amt -= ret;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java
index ad732c7..d2ccdc7 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java
@@ -26,6 +26,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.*;
import org.apache.tajo.datum.protobuf.ProtobufJsonFormat;
import org.apache.tajo.util.Bytes;
+import org.apache.tajo.util.NumberUtil;
import java.io.IOException;
import java.io.OutputStream;
@@ -132,11 +133,11 @@ public class TextSerializerDeserializer implements SerializerDeserializer {
case INT1:
case INT2:
datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
- : DatumFactory.createInt2((short) Bytes.parseInt(bytes, offset, length));
+ : DatumFactory.createInt2((short) NumberUtil.parseInt(bytes, offset, length));
break;
case INT4:
datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
- : DatumFactory.createInt4(Bytes.parseInt(bytes, offset, length));
+ : DatumFactory.createInt4(NumberUtil.parseInt(bytes, offset, length));
break;
case INT8:
datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
@@ -148,7 +149,7 @@ public class TextSerializerDeserializer implements SerializerDeserializer {
break;
case FLOAT8:
datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
- : DatumFactory.createFloat8(Bytes.parseDouble(bytes, offset, length));
+ : DatumFactory.createFloat8(NumberUtil.parseDouble(bytes, offset, length));
break;
case TEXT: {
byte[] chars = new byte[length];
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java b/tajo-storage/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java
index a7a144b..5d43bd5 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java
@@ -30,12 +30,12 @@ import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder;
import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleComparator;
import org.apache.tajo.storage.index.IndexMethod;
import org.apache.tajo.storage.index.IndexWriter;
import org.apache.tajo.storage.index.OrderIndexReader;
-import org.apache.tajo.util.Bytes;
import java.io.Closeable;
import java.io.FileNotFoundException;
@@ -335,7 +335,7 @@ public class BSTIndex implements IndexMethod {
// schema
int schemaByteSize = indexIn.readInt();
byte [] schemaBytes = new byte[schemaByteSize];
- Bytes.readFully(indexIn, schemaBytes, 0, schemaByteSize);
+ StorageUtil.readFully(indexIn, schemaBytes, 0, schemaByteSize);
SchemaProto.Builder builder = SchemaProto.newBuilder();
builder.mergeFrom(schemaBytes);
@@ -346,7 +346,7 @@ public class BSTIndex implements IndexMethod {
// comparator
int compByteSize = indexIn.readInt();
byte [] compBytes = new byte[compByteSize];
- Bytes.readFully(indexIn, compBytes, 0, compByteSize);
+ StorageUtil.readFully(indexIn, compBytes, 0, compByteSize);
TupleComparatorProto.Builder compProto = TupleComparatorProto.newBuilder();
compProto.mergeFrom(compBytes);
@@ -358,11 +358,11 @@ public class BSTIndex implements IndexMethod {
this.entryNum = indexIn.readInt();
if (entryNum > 0) { // if there is no any entry, do not read firstKey/lastKey values
byte [] minBytes = new byte[indexIn.readInt()];
- Bytes.readFully(indexIn, minBytes, 0, minBytes.length);
+ StorageUtil.readFully(indexIn, minBytes, 0, minBytes.length);
this.firstKey = rowStoreDecoder.toTuple(minBytes);
byte [] maxBytes = new byte[indexIn.readInt()];
- Bytes.readFully(indexIn, maxBytes, 0, maxBytes.length);
+ StorageUtil.readFully(indexIn, maxBytes, 0, maxBytes.length);
this.lastKey = rowStoreDecoder.toTuple(maxBytes);
}
}
@@ -484,7 +484,7 @@ public class BSTIndex implements IndexMethod {
for (int i = 0; i < entryNum; i++) {
counter++;
buf = new byte[in.readInt()];
- Bytes.readFully(in, buf, 0, buf.length);
+ StorageUtil.readFully(in, buf, 0, buf.length);
dataSubIndex[i] = rowStoreDecoder.toTuple(buf);
int offsetNum = in.readInt();
@@ -506,7 +506,7 @@ public class BSTIndex implements IndexMethod {
byte[] buf;
for (int i = 0; i < counter; i++) {
buf = new byte[in.readInt()];
- Bytes.readFully(in, buf, 0, buf.length);
+ StorageUtil.readFully(in, buf, 0, buf.length);
dataSubIndex[i] = rowStoreDecoder.toTuple(buf);
int offsetNum = in.readInt();
@@ -535,7 +535,7 @@ public class BSTIndex implements IndexMethod {
byte[] buf;
for (int i = 0; i < entryNum; i++) {
buf = new byte[in.readInt()];
- Bytes.readFully(in, buf, 0, buf.length);
+ StorageUtil.readFully(in, buf, 0, buf.length);
keyTuple = rowStoreDecoder.toTuple(buf);
dataIndex[i] = keyTuple;
this.offsetIndex[i] = in.readLong();
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
index 4cc37b3..78498c7 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
@@ -39,7 +39,6 @@ import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
-import org.apache.tajo.util.Bytes;
import java.io.Closeable;
import java.io.*;
@@ -489,7 +488,7 @@ public class RCFile {
}
if (skipTotal != 0) {
- Bytes.skipFully(in, skipTotal);
+ StorageUtil.skipFully(in, skipTotal);
skipTotal = 0;
}
@@ -528,7 +527,7 @@ public class RCFile {
}
if (skipTotal != 0) {
- Bytes.skipFully(in, skipTotal);
+ StorageUtil.skipFully(in, skipTotal);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
index 9eb1b2d..b150a9a 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
@@ -43,7 +43,7 @@ import org.apache.tajo.datum.ProtobufDatum;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.exception.AlreadyExistsStorageException;
import org.apache.tajo.storage.rcfile.NonSyncByteArrayOutputStream;
-import org.apache.tajo.util.Bytes;
+import org.apache.tajo.util.BytesUtils;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -177,16 +177,16 @@ public class SequenceFileAppender extends FileAppender {
switch (schema.getColumn(j).getDataType().getType()) {
case TEXT:
- Bytes.writeVLong(os, datum.asTextBytes().length);
+ BytesUtils.writeVLong(os, datum.asTextBytes().length);
break;
case PROTOBUF:
ProtobufDatum protobufDatum = (ProtobufDatum) datum;
- Bytes.writeVLong(os, protobufDatum.asByteArray().length);
+ BytesUtils.writeVLong(os, protobufDatum.asByteArray().length);
break;
case CHAR:
case INET4:
case BLOB:
- Bytes.writeVLong(os, datum.asByteArray().length);
+ BytesUtils.writeVLong(os, datum.asByteArray().length);
break;
default:
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
index ccf3d9e..32d1d57 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
@@ -36,7 +36,7 @@ import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
-import org.apache.tajo.util.Bytes;
+import org.apache.tajo.util.BytesUtils;
import java.io.IOException;
@@ -164,7 +164,7 @@ public class SequenceFileScanner extends FileScanner {
} else {
Text text = new Text();
reader.getCurrentValue(text);
- cells = Bytes.splitPreserveAllTokens(text.getBytes(), delimiter, projectionMap);
+ cells = BytesUtils.splitPreserveAllTokens(text.getBytes(), delimiter, projectionMap);
totalBytes += (long)text.getBytes().length;
tuple = new LazyTuple(schema, cells, 0, nullChars, serde);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
index 19209bd..e15ca6e 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
@@ -31,7 +31,7 @@ import org.apache.tajo.storage.LazyTuple;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.compress.CodecPool;
import org.apache.tajo.storage.fragment.FileFragment;
-import org.apache.tajo.util.Bytes;
+import org.apache.tajo.util.BytesUtils;
import java.io.DataInputStream;
import java.io.IOException;
@@ -221,10 +221,10 @@ public class CSVFileScanner extends FileScannerV2 {
if (prevTailLen == 0) {
tail = new byte[0];
- tuples = Bytes.splitPreserveAllTokens(buf, rbyte, (char) LF);
+ tuples = BytesUtils.splitPreserveAllTokens(buf, rbyte, (char) LF);
} else {
byte[] lastRow = ArrayUtils.addAll(tail, buf);
- tuples = Bytes.splitPreserveAllTokens(lastRow, rbyte + tail.length, (char) LF);
+ tuples = BytesUtils.splitPreserveAllTokens(lastRow, rbyte + tail.length, (char) LF);
tail = null;
}
@@ -294,7 +294,7 @@ public class CSVFileScanner extends FileScannerV2 {
offset = this.tupleOffsets[currentIdx];
}
- byte[][] cells = Bytes.splitPreserveAllTokens(tuples[currentIdx++], delimiter, targetColumnIndexes);
+ byte[][] cells = BytesUtils.splitPreserveAllTokens(tuples[currentIdx++], delimiter, targetColumnIndexes);
return new LazyTuple(schema, cells, offset);
} catch (Throwable t) {
LOG.error(t.getMessage(), t);
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
index 47dce74..ac58598 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.io.compress.*;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.storage.rcfile.*;
-import org.apache.tajo.util.Bytes;
import java.io.*;
import java.rmi.server.UID;
@@ -485,7 +485,7 @@ public class RCFile {
}
if (skipTotal != 0) {
- Bytes.skipFully(in, skipTotal);
+ StorageUtil.skipFully(in, skipTotal);
skipTotal = 0;
}
@@ -512,7 +512,7 @@ public class RCFile {
}
if (skipTotal != 0) {
- Bytes.skipFully(in, skipTotal);
+ StorageUtil.skipFully(in, skipTotal);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java
index cb2f7a6..c6149f7 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestLazyTuple.java
@@ -23,7 +23,7 @@ import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
-import org.apache.tajo.util.Bytes;
+import org.apache.tajo.util.BytesUtils;
import org.junit.Before;
import org.junit.Test;
@@ -69,7 +69,7 @@ public class TestLazyTuple {
sb.append(DatumFactory.createInet4("192.168.0.1")).append('|');
sb.append(new String(nullbytes)).append('|');
sb.append(NullDatum.get());
- textRow = Bytes.splitPreserveAllTokens(sb.toString().getBytes(), '|');
+ textRow = BytesUtils.splitPreserveAllTokens(sb.toString().getBytes(), '|');
serde = new TextSerializerDeserializer();
}
@@ -220,7 +220,7 @@ public class TestLazyTuple {
@Test
public void testInvalidNumber() {
- byte[][] bytes = Bytes.splitPreserveAllTokens(" 1| |2 ||".getBytes(), '|');
+ byte[][] bytes = BytesUtils.splitPreserveAllTokens(" 1| |2 ||".getBytes(), '|');
Schema schema = new Schema();
schema.addColumn("col1", TajoDataTypes.Type.INT2);
schema.addColumn("col2", TajoDataTypes.Type.INT4);
[2/7] git commit: TAJO-965: Upgrade Bytes class and move some methods
to others.
Posted by ji...@apache.org.
TAJO-965: Upgrade Bytes class and move some methods to others.
Closes #86
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/85b7031a
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/85b7031a
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/85b7031a
Branch: refs/heads/index_support
Commit: 85b7031aa58971365a526c44c1b31c3385b7993d
Parents: 6dac6a5
Author: Hyunsik Choi <hy...@apache.org>
Authored: Tue Jul 22 11:26:01 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Tue Jul 22 11:26:01 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../main/java/org/apache/tajo/util/Bytes.java | 1184 ++++++++----------
.../java/org/apache/tajo/util/BytesUtils.java | 182 +++
.../java/org/apache/tajo/util/NumberUtil.java | 348 ++++-
.../java/org/apache/tajo/util/TestBytes.java | 111 --
.../org/apache/tajo/util/TestNumberUtil.java | 47 +-
.../org/apache/tajo/util/TestStringUtil.java | 40 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 6 +-
.../java/org/apache/tajo/storage/CSVFile.java | 6 +-
.../java/org/apache/tajo/storage/RowFile.java | 3 +-
.../org/apache/tajo/storage/StorageUtil.java | 41 +
.../storage/TextSerializerDeserializer.java | 7 +-
.../apache/tajo/storage/index/bst/BSTIndex.java | 16 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 5 +-
.../sequencefile/SequenceFileAppender.java | 8 +-
.../sequencefile/SequenceFileScanner.java | 4 +-
.../apache/tajo/storage/v2/CSVFileScanner.java | 8 +-
.../java/org/apache/tajo/storage/v2/RCFile.java | 6 +-
.../org/apache/tajo/storage/TestLazyTuple.java | 6 +-
19 files changed, 1221 insertions(+), 809 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 4010042..86cef01 100644
--- a/CHANGES
+++ b/CHANGES
@@ -97,6 +97,8 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-965: Upgrade Bytes class and move some methods to others. (hyunsik)
+
TAJO-961: TajoCli should exit when at least one query faces error while
executing a SQL script. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java b/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java
index 448274a..405ec2f 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/Bytes.java
@@ -21,30 +21,48 @@ package org.apache.tajo.util;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.io.WritableUtils;
import sun.misc.Unsafe;
-import java.io.*;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.nio.charset.Charset;
import java.security.AccessController;
import java.security.PrivilegedAction;
-import java.util.ArrayList;
+import java.security.SecureRandom;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
+import static com.google.common.base.Preconditions.*;
+
/**
* Utility class that handles byte arrays, conversions to/from other types,
* comparisons, hash code generation, manufacturing keys for HashMaps or
* HashSets, etc.
*/
public class Bytes {
+ //HConstants.UTF8_ENCODING should be updated if this changed
+ /** When we encode strings, we always specify UTF8 encoding */
+ private static final String UTF8_ENCODING = "UTF-8";
+
+ //HConstants.UTF8_CHARSET should be updated if this changed
+ /** When we encode strings, we always specify UTF8 encoding */
+ private static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
+
+ //HConstants.EMPTY_BYTE_ARRAY should be updated if this changed
+ private static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
private static final Log LOG = LogFactory.getLog(Bytes.class);
@@ -88,7 +106,6 @@ public class Bytes {
*/
public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE;
-
/**
* Estimate of size cost to pay beyond payload in jvm for instance of byte [].
* Estimate based on study of jhat and jprofiler numbers.
@@ -98,8 +115,20 @@ public class Bytes {
public static final int ESTIMATED_HEAP_TAX = 16;
/**
+ * Returns length of the byte array, returning 0 if the array is null.
+ * Useful for calculating sizes.
+ * @param b byte array, which can be null
+ * @return 0 if b is null, otherwise returns length
+ */
+ final public static int len(byte[] b) {
+ return b == null ? 0 : b.length;
+ }
+
+ /**
* Byte array comparator class.
*/
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
public static class ByteArrayComparator implements RawComparator<byte []> {
/**
* Constructor
@@ -107,35 +136,65 @@ public class Bytes {
public ByteArrayComparator() {
super();
}
+ @Override
public int compare(byte [] left, byte [] right) {
return compareTo(left, right);
}
+ @Override
public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
return LexicographicalComparerHolder.BEST_COMPARER.
- compareTo(b1, s1, l1, b2, s2, l2);
+ compareTo(b1, s1, l1, b2, s2, l2);
+ }
+ }
+
+ /**
+ * A {@link ByteArrayComparator} that treats the empty array as the largest value.
+ * This is useful for comparing row end keys for regions.
+ */
+ // TODO: unfortunately, HBase uses byte[0] as both start and end keys for region
+ // boundaries. Thus semantically, we should treat empty byte array as the smallest value
+ // while comparing row keys, start keys etc; but as the largest value for comparing
+ // region boundaries for endKeys.
+ @InterfaceAudience.Public
+ @InterfaceStability.Stable
+ public static class RowEndKeyComparator extends ByteArrayComparator {
+ @Override
+ public int compare(byte[] left, byte[] right) {
+ return compare(left, 0, left.length, right, 0, right.length);
+ }
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ if (b1 == b2 && s1 == s2 && l1 == l2) {
+ return 0;
+ }
+ if (l1 == 0) {
+ return l2; //0 or positive
+ }
+ if (l2 == 0) {
+ return -1;
+ }
+ return super.compare(b1, s1, l1, b2, s2, l2);
}
}
/**
* Pass this to TreeMaps where byte [] are keys.
*/
- public static Comparator<byte []> BYTES_COMPARATOR =
- new ByteArrayComparator();
+ public final static Comparator<byte []> BYTES_COMPARATOR = new ByteArrayComparator();
/**
* Use comparing byte arrays, byte-by-byte
*/
- public static RawComparator<byte []> BYTES_RAWCOMPARATOR =
- new ByteArrayComparator();
+ public final static RawComparator<byte []> BYTES_RAWCOMPARATOR = new ByteArrayComparator();
/**
* Read byte-array written with a WritableableUtils.vint prefix.
* @param in Input to read from.
* @return byte array read off <code>in</code>
- * @throws java.io.IOException e
+ * @throws IOException e
*/
public static byte [] readByteArray(final DataInput in)
- throws IOException {
+ throws IOException {
int len = WritableUtils.readVInt(in);
if (len < 0) {
throw new NegativeArraySizeException(Integer.toString(len));
@@ -163,10 +222,10 @@ public class Bytes {
* Write byte-array with a WritableableUtils.vint prefix.
* @param out output stream to be written to
* @param b array to write
- * @throws java.io.IOException e
+ * @throws IOException e
*/
public static void writeByteArray(final DataOutput out, final byte [] b)
- throws IOException {
+ throws IOException {
if(b == null) {
WritableUtils.writeVInt(out, 0);
} else {
@@ -180,11 +239,11 @@ public class Bytes {
* @param b array
* @param offset offset into array
* @param length length past offset
- * @throws java.io.IOException e
+ * @throws IOException e
*/
public static void writeByteArray(final DataOutput out, final byte [] b,
- final int offset, final int length)
- throws IOException {
+ final int offset, final int length)
+ throws IOException {
WritableUtils.writeVInt(out, length);
out.write(b, offset, length);
}
@@ -199,7 +258,7 @@ public class Bytes {
* @return New offset in src array.
*/
public static int writeByteArray(final byte [] tgt, final int tgtOffset,
- final byte [] src, final int srcOffset, final int srcLength) {
+ final byte [] src, final int srcOffset, final int srcLength) {
byte [] vint = vintToBytes(srcLength);
System.arraycopy(vint, 0, tgt, tgtOffset, vint.length);
int offset = tgtOffset + vint.length;
@@ -207,11 +266,6 @@ public class Bytes {
return offset + srcLength;
}
- public static void writeVLong(ByteArrayOutputStream byteStream, long l) {
- byte[] vLongBytes = Bytes.vlongToBytes(l);
- byteStream.write(vLongBytes, 0, vLongBytes.length);
- }
-
/**
* Put bytes at the specified byte array position.
* @param tgtBytes the byte array
@@ -222,7 +276,7 @@ public class Bytes {
* @return incremented offset
*/
public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes,
- int srcOffset, int srcLength) {
+ int srcOffset, int srcLength) {
System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
return tgtOffset + srcLength;
}
@@ -240,14 +294,37 @@ public class Bytes {
}
/**
- * Returns a new byte array, copied from the passed ByteBuffer.
- * @param bb A ByteBuffer
+ * Add the whole content of the ByteBuffer to the bytes arrays. The ByteBuffer is modified.
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param buf ByteBuffer to write out
+ * @return incremented offset
+ */
+ public static int putByteBuffer(byte[] bytes, int offset, ByteBuffer buf) {
+ int len = buf.remaining();
+ buf.get(bytes, offset, len);
+ return offset + len;
+ }
+
+ /**
+ * Returns a new byte array, copied from the given {@code buf},
+ * from the index 0 (inclusive) to the limit (exclusive),
+ * regardless of the current position.
+ * The position and the other index parameters are not changed.
+ *
+ * @param buf a byte buffer
* @return the byte array
+ * @see #getBytes(ByteBuffer)
*/
- public static byte[] toBytes(ByteBuffer bb) {
- int length = bb.limit();
- byte [] result = new byte[length];
- System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length);
+ public static byte[] toBytes(ByteBuffer buf) {
+ ByteBuffer dup = buf.duplicate();
+ dup.position(0);
+ return readBytes(dup);
+ }
+
+ private static byte[] readBytes(ByteBuffer buf) {
+ byte [] result = new byte[buf.remaining()];
+ buf.get(result);
return result;
}
@@ -276,8 +353,7 @@ public class Bytes {
/**
* This method will convert utf8 encoded bytes into a string. If
- * an UnsupportedEncodingException occurs, this method will eat it
- * and return null instead.
+ * the given byte array is null, this method will return null.
*
* @param b Presumed UTF-8 encoded byte array.
* @param off offset into array
@@ -291,12 +367,7 @@ public class Bytes {
if (len == 0) {
return "";
}
- try {
- return new String(b, off, len, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- LOG.error("UTF-8 not supported?", e);
- return null;
- }
+ return new String(b, off, len, UTF8_CHARSET);
}
/**
@@ -311,18 +382,25 @@ public class Bytes {
return "null";
return toStringBinary(b, 0, b.length);
}
-
+
/**
- * Converts the given byte buffer, from its array offset to its limit, to
- * a string. The position and the mark are ignored.
+ * Converts the given byte buffer to a printable representation,
+ * from the index 0 (inclusive) to the limit (exclusive),
+ * regardless of the current position.
+ * The position and the other index parameters are not changed.
*
* @param buf a byte buffer
* @return a string representation of the buffer's binary contents
+ * @see #toBytes(ByteBuffer)
+ * @see #getBytes(ByteBuffer)
*/
public static String toStringBinary(ByteBuffer buf) {
if (buf == null)
return "null";
- return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
+ if (buf.hasArray()) {
+ return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit());
+ }
+ return toStringBinary(toBytes(buf));
}
/**
@@ -337,21 +415,19 @@ public class Bytes {
*/
public static String toStringBinary(final byte [] b, int off, int len) {
StringBuilder result = new StringBuilder();
- try {
- String first = new String(b, off, len, "ISO-8859-1");
- for (int i = 0; i < first.length() ; ++i ) {
- int ch = first.charAt(i) & 0xFF;
- if ( (ch >= '0' && ch <= '9')
- || (ch >= 'A' && ch <= 'Z')
- || (ch >= 'a' && ch <= 'z')
- || " `~!@#$%^&*()-_=+[]{}\\|;:'\",.<>/?".indexOf(ch) >= 0 ) {
- result.append(first.charAt(i));
- } else {
- result.append(String.format("\\x%02X", ch));
- }
+ // Just in case we are passed a 'len' that is > buffer length...
+ if (off >= b.length) return result.toString();
+ if (off + len > b.length) len = b.length - off;
+ for (int i = off; i < off + len ; ++i ) {
+ int ch = b[i] & 0xFF;
+ if ( (ch >= '0' && ch <= '9')
+ || (ch >= 'A' && ch <= 'Z')
+ || (ch >= 'a' && ch <= 'z')
+ || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
+ result.append((char)ch);
+ } else {
+ result.append(String.format("\\x%02X", ch));
}
- } catch (UnsupportedEncodingException e) {
- LOG.error("ISO-8859-1 not supported?", e);
}
return result.toString();
}
@@ -359,7 +435,7 @@ public class Bytes {
private static boolean isHexDigit(char c) {
return
(c >= 'A' && c <= 'F') ||
- (c >= '0' && c <= '9');
+ (c >= '0' && c <= '9');
}
/**
@@ -376,19 +452,12 @@ public class Bytes {
}
public static byte [] toBytesBinary(String in) {
- // this may be bigger than we need, but lets be safe.
+ // this may be bigger than we need, but let's be safe.
byte [] b = new byte[in.length()];
int size = 0;
for (int i = 0; i < in.length(); ++i) {
char ch = in.charAt(i);
- if (ch == '\\') {
- // begin hex escape:
- char next = in.charAt(i+1);
- if (next != 'x') {
- // invalid escape sequence, ignore this one.
- b[size++] = (byte)ch;
- continue;
- }
+ if (ch == '\\' && in.length() > i+1 && in.charAt(i+1) == 'x') {
// ok, take next 2 hex digits.
char hd1 = in.charAt(i+2);
char hd2 = in.charAt(i+3);
@@ -420,26 +489,7 @@ public class Bytes {
* @return the byte array
*/
public static byte[] toBytes(String s) {
- try {
- return s.getBytes("UTF-8");
- } catch (UnsupportedEncodingException e) {
- LOG.error("UTF-8 not supported?", e);
- return null;
- }
- }
-
- /**
- * Converts a char array to a ascii byte array.
- *
- * @param chars string
- * @return the byte array
- */
- public static byte[] toASCIIBytes(char[] chars) {
- byte[] buffer = new byte[chars.length];
- for (int i = 0; i < chars.length; i++) {
- buffer[i] = (byte) chars[i];
- }
- return buffer;
+ return s.getBytes(UTF8_CHARSET);
}
/**
@@ -526,16 +576,16 @@ public class Bytes {
}
private static IllegalArgumentException
- explainWrongLengthOrOffset(final byte[] bytes,
- final int offset,
- final int length,
- final int expectedLength) {
+ explainWrongLengthOrOffset(final byte[] bytes,
+ final int offset,
+ final int length,
+ final int expectedLength) {
String reason;
if (length != expectedLength) {
reason = "Wrong length: " + length + ", expected " + expectedLength;
} else {
- reason = "offset (" + offset + ") + length (" + length + ") exceed the"
- + " capacity of the array: " + bytes.length;
+ reason = "offset (" + offset + ") + length (" + length + ") exceed the"
+ + " capacity of the array: " + bytes.length;
}
return new IllegalArgumentException(reason);
}
@@ -640,7 +690,9 @@ public class Bytes {
}
/**
- * Convert an int value to a byte array
+ * Convert an int value to a byte array. Big-endian. Same as what DataOutputStream.writeInt
+ * does.
+ *
* @param val value
* @return the byte array
*/
@@ -695,6 +747,28 @@ public class Bytes {
}
/**
+ * Converts a byte array to an int value
+ * @param bytes byte array
+ * @param offset offset into array
+ * @param length how many bytes should be considered for creating int
+ * @return the int value
+ * @throws IllegalArgumentException if there's not enough room in the array at the offset
+ * indicated.
+ */
+ public static int readAsInt(byte[] bytes, int offset, final int length) {
+ if (offset + length > bytes.length) {
+ throw new IllegalArgumentException("offset (" + offset + ") + length (" + length
+ + ") exceed the" + " capacity of the array: " + bytes.length);
+ }
+ int n = 0;
+ for(int i = offset; i < (offset + length); i++) {
+ n <<= 8;
+ n ^= bytes[i] & 0xFF;
+ }
+ return n;
+ }
+
+ /**
* Put an int value out to the specified byte array position.
* @param bytes the byte array
* @param offset position in the array
@@ -769,17 +843,16 @@ public class Bytes {
}
/**
- * This method will get a sequence of bytes from pos -> limit,
- * but will restore pos after.
- * @param buf
- * @return
+ * Returns a new byte array, copied from the given {@code buf},
+ * from the position (inclusive) to the limit (exclusive).
+ * The position and the other index parameters are not changed.
+ *
+ * @param buf a byte buffer
+ * @return the byte array
+ * @see #toBytes(ByteBuffer)
*/
public static byte[] getBytes(ByteBuffer buf) {
- int savedPos = buf.position();
- byte [] newBytes = new byte[buf.remaining()];
- buf.get(newBytes);
- buf.position(savedPos);
- return newBytes;
+ return readBytes(buf.duplicate());
}
/**
@@ -803,6 +876,29 @@ public class Bytes {
}
/**
+ * Put an int value as short out to the specified byte array position. Only the lower 2 bytes of
+ * the short will be put into the array. The caller of the API need to make sure they will not
+ * loose the value by doing so. This is useful to store an unsigned short which is represented as
+ * int in other parts.
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val value to write out
+ * @return incremented offset
+ * @throws IllegalArgumentException if the byte array given doesn't have
+ * enough room at the offset specified.
+ */
+ public static int putAsShort(byte[] bytes, int offset, int val) {
+ if (bytes.length - offset < SIZEOF_SHORT) {
+ throw new IllegalArgumentException("Not enough room to put a short at"
+ + " offset " + offset + " in a " + bytes.length + " byte array");
+ }
+ bytes[offset+1] = (byte) val;
+ val >>= 8;
+ bytes[offset] = (byte) val;
+ return offset + SIZEOF_SHORT;
+ }
+
+ /**
* Convert a BigDecimal value to a byte array
*
* @param val
@@ -832,29 +928,18 @@ public class Bytes {
*
* @param bytes
* @param offset
- * @return the char value
- */
- public static BigDecimal toBigDecimal(byte[] bytes, int offset) {
- return toBigDecimal(bytes, offset, bytes.length);
- }
-
- /**
- * Converts a byte array to a BigDecimal value
- *
- * @param bytes
- * @param offset
* @param length
* @return the char value
*/
public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
if (bytes == null || length < SIZEOF_INT + 1 ||
- (offset + length > bytes.length)) {
+ (offset + length > bytes.length)) {
return null;
}
- int scale = toInt(bytes, 0);
+ int scale = toInt(bytes, offset);
byte[] tcBytes = new byte[length - SIZEOF_INT];
- System.arraycopy(bytes, SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
+ System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
return new BigDecimal(new BigInteger(tcBytes), scale);
}
@@ -876,7 +961,7 @@ public class Bytes {
offset = putInt(result, offset, val.scale());
return putBytes(result, offset, valueBytes, 0, valueBytes.length);
}
-
+
/**
* @param vint Integer to make a vint of.
* @return Vint as bytes array.
@@ -916,45 +1001,6 @@ public class Bytes {
}
/**
- * @param n Long to make a VLong of.
- * @return VLong as bytes array.
- */
- public static byte[] vlongToBytes(long n) {
- byte [] result;
- int offset = 0;
- if (n >= -112 && n <= 127) {
- result = new byte[1];
- result[offset] = (byte) n;
- return result;
- }
-
- int len = -112;
- if (n < 0) {
- n ^= -1L; // take one's complement'
- len = -120;
- }
-
- long tmp = n;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- int size = WritableUtils.decodeVIntSize((byte)len);
-
- result = new byte[size];
- result[offset++] = (byte) len;
- len = (len < -120) ? -(len + 120) : -(len + 112);
-
- for (int idx = len; idx != 0; idx--) {
- int shiftbits = (idx - 1) * 8;
- long mask = 0xFFL << shiftbits;
- result[offset++] = (byte)((n & mask) >> shiftbits);
- }
- return result;
- }
-
- /**
* @param buffer buffer to convert
* @return vint bytes as an integer.
*/
@@ -982,47 +1028,19 @@ public class Bytes {
* @return deserialized long from stream.
*/
public static long readVLong(final byte [] buffer, final int offset)
- throws IOException {
- byte firstByte = buffer[offset];
- int length = (byte) WritableUtils.decodeVIntSize(firstByte);
- if (length == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < length - 1; idx++) {
- byte b = buffer[offset + 1 + idx];
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
- }
-
- /**
- * Reads a zero-compressed encoded int from input stream and returns it.
- * @param buffer Binary array
- * @param offset Offset into array at which vint begins.
- * @throws java.io.IOException e
- * @return deserialized long from stream.
- */
- public static int readVInt(final byte [] buffer, final int offset)
throws IOException {
byte firstByte = buffer[offset];
- int length = (byte) WritableUtils.decodeVIntSize(firstByte);
- if (length == 1) {
+ int len = WritableUtils.decodeVIntSize(firstByte);
+ if (len == 1) {
return firstByte;
}
- int i = 0;
- for (int idx = 0; idx < length - 1; idx++) {
+ long i = 0;
+ for (int idx = 0; idx < len-1; idx++) {
byte b = buffer[offset + 1 + idx];
i = i << 8;
i = i | (b & 0xFF);
}
- return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1) : i);
- }
-
- public static byte getVIntSize(byte[] bytes, int offset) {
- byte firstByte = bytes[offset];
- return (byte) WritableUtils.decodeVIntSize(firstByte);
+ return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
/**
@@ -1032,7 +1050,7 @@ public class Bytes {
*/
public static int compareTo(final byte [] left, final byte [] right) {
return LexicographicalComparerHolder.BEST_COMPARER.
- compareTo(left, 0, left.length, right, 0, right.length);
+ compareTo(left, 0, left.length, right, 0, right.length);
}
/**
@@ -1047,20 +1065,15 @@ public class Bytes {
* @return 0 if equal, < 0 if left is less than right, etc.
*/
public static int compareTo(byte[] buffer1, int offset1, int length1,
- byte[] buffer2, int offset2, int length2) {
+ byte[] buffer2, int offset2, int length2) {
return LexicographicalComparerHolder.BEST_COMPARER.
- compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
+ compareTo(buffer1, offset1, length1, buffer2, offset2, length2);
}
-
- /**
- * The number of bytes required to represent a primitive {@code long}
- * value.
- */
- public static final int LONG_BYTES = Long.SIZE / Byte.SIZE;
-
+
interface Comparer<T> {
- abstract public int compareTo(T buffer1, int offset1, int length1,
- T buffer2, int offset2, int length2);
+ int compareTo(
+ T buffer1, int offset1, int length1, T buffer2, int offset2, int length2
+ );
}
@VisibleForTesting
@@ -1070,7 +1083,7 @@ public class Bytes {
/**
* Provides a lexicographical comparer implementation; either a Java
- * implementation or a faster implementation based on {@link sun.misc.Unsafe}.
+ * implementation or a faster implementation based on {@link Unsafe}.
*
* <p>Uses reflection to gracefully fall back to the Java implementation if
* {@code Unsafe} isn't available.
@@ -1079,7 +1092,7 @@ public class Bytes {
static class LexicographicalComparerHolder {
static final String UNSAFE_COMPARER_NAME =
LexicographicalComparerHolder.class.getName() + "$UnsafeComparer";
-
+
static final Comparer<byte[]> BEST_COMPARER = getBestComparer();
/**
* Returns the Unsafe-using Comparer, or falls back to the pure-Java
@@ -1092,19 +1105,19 @@ public class Bytes {
// yes, UnsafeComparer does implement Comparer<byte[]>
@SuppressWarnings("unchecked")
Comparer<byte[]> comparer =
- (Comparer<byte[]>) theClass.getEnumConstants()[0];
+ (Comparer<byte[]>) theClass.getEnumConstants()[0];
return comparer;
} catch (Throwable t) { // ensure we really catch *everything*
return lexicographicalComparerJavaImpl();
}
}
-
+
enum PureJavaComparer implements Comparer<byte[]> {
INSTANCE;
@Override
public int compareTo(byte[] buffer1, int offset1, int length1,
- byte[] buffer2, int offset2, int length2) {
+ byte[] buffer2, int offset2, int length2) {
// Short circuit equal case
if (buffer1 == buffer2 &&
offset1 == offset2 &&
@@ -1162,7 +1175,7 @@ public class Bytes {
}
static final boolean littleEndian =
- ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
+ ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
/**
* Returns true if x1 is less than x2, when both values are treated as
@@ -1185,7 +1198,7 @@ public class Bytes {
*/
@Override
public int compareTo(byte[] buffer1, int offset1, int length1,
- byte[] buffer2, int offset2, int length2) {
+ byte[] buffer2, int offset2, int length2) {
// Short circuit equal case
if (buffer1 == buffer2 &&
offset1 == offset2 &&
@@ -1193,7 +1206,7 @@ public class Bytes {
return 0;
}
int minLength = Math.min(length1, length2);
- int minWords = minLength / LONG_BYTES;
+ int minWords = minLength / SIZEOF_LONG;
int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
@@ -1202,7 +1215,7 @@ public class Bytes {
* time is no slower than comparing 4 bytes at a time even on 32-bit.
* On the other hand, it is substantially faster on 64-bit.
*/
- for (int i = 0; i < minWords * LONG_BYTES; i += LONG_BYTES) {
+ for (int i = 0; i < minWords * SIZEOF_LONG; i += SIZEOF_LONG) {
long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
long diff = lw ^ rw;
@@ -1237,7 +1250,7 @@ public class Bytes {
}
// The epilogue to cover the last (minLength % 8) elements.
- for (int i = minWords * LONG_BYTES; i < minLength; i++) {
+ for (int i = minWords * SIZEOF_LONG; i < minLength; i++) {
int a = (buffer1[offset1 + i] & 0xff);
int b = (buffer2[offset2 + i] & 0xff);
if (a != b) {
@@ -1261,7 +1274,7 @@ public class Bytes {
if (left == null || right == null) return false;
if (left.length != right.length) return false;
if (left.length == 0) return true;
-
+
// Since we're often comparing adjacent sorted data,
// it's usual to have equal arrays except for the very last byte
// so check that first
@@ -1269,7 +1282,7 @@ public class Bytes {
return compareTo(left, right) == 0;
}
-
+
public static boolean equals(final byte[] left, int leftOffset, int leftLen,
final byte[] right, int rightOffset, int rightLen) {
// short circuit case
@@ -1285,16 +1298,37 @@ public class Bytes {
if (leftLen == 0) {
return true;
}
-
+
// Since we're often comparing adjacent sorted data,
// it's usual to have equal arrays except for the very last byte
// so check that first
if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) return false;
return LexicographicalComparerHolder.BEST_COMPARER.
- compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
+ compareTo(left, leftOffset, leftLen, right, rightOffset, rightLen) == 0;
}
-
+
+
+ /**
+ * @param a left operand
+ * @param buf right operand
+ * @return True if equal
+ */
+ public static boolean equals(byte[] a, ByteBuffer buf) {
+ if (a == null) return buf == null;
+ if (buf == null) return false;
+ if (a.length != buf.remaining()) return false;
+
+ // Thou shalt not modify the original byte buffer in what should be read only operations.
+ ByteBuffer b = buf.duplicate();
+ for (byte anA : a) {
+ if (anA != b.get()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Return true if the byte array on the right is a prefix of the byte
@@ -1302,15 +1336,16 @@ public class Bytes {
*/
public static boolean startsWith(byte[] bytes, byte[] prefix) {
return bytes != null && prefix != null &&
- bytes.length >= prefix.length &&
- LexicographicalComparerHolder.BEST_COMPARER.
- compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
+ bytes.length >= prefix.length &&
+ LexicographicalComparerHolder.BEST_COMPARER.
+ compareTo(bytes, 0, prefix.length, prefix, 0, prefix.length) == 0;
}
/**
* @param b bytes to hash
- * @return Runs {@link org.apache.hadoop.io.WritableComparator#hashBytes(byte[], int)} on the
+ * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
* passed in array. This method is what {@link org.apache.hadoop.io.Text} and
+ * {@link ImmutableBytesWritable} use calculating hash code.
*/
public static int hashCode(final byte [] b) {
return hashCode(b, b.length);
@@ -1319,8 +1354,9 @@ public class Bytes {
/**
* @param b value
* @param length length of the value
- * @return Runs {@link org.apache.hadoop.io.WritableComparator#hashBytes(byte[], int)} on the
+ * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
* passed in array. This method is what {@link org.apache.hadoop.io.Text} and
+ * {@link ImmutableBytesWritable} use calculating hash code.
*/
public static int hashCode(final byte [] b, final int length) {
return WritableComparator.hashBytes(b, length);
@@ -1351,7 +1387,7 @@ public class Bytes {
* @return New array that has a in lower half and b in upper half.
*/
public static byte [] add(final byte [] a, final byte [] b) {
- return add(a, b, new byte [0]);
+ return add(a, b, EMPTY_BYTE_ARRAY);
}
/**
@@ -1432,114 +1468,49 @@ public class Bytes {
* @return Array of dividing values
*/
public static byte [][] split(final byte [] a, final byte [] b, final int num) {
- byte[][] ret = new byte[num+2][];
+ return split(a, b, false, num);
+ }
+
+ /**
+ * Split passed range. Expensive operation relatively. Uses BigInteger math.
+ * Useful splitting ranges for MapReduce jobs.
+ * @param a Beginning of range
+ * @param b End of range
+ * @param inclusive Whether the end of range is prefix-inclusive or is
+ * considered an exclusive boundary. Automatic splits are generally exclusive
+ * and manual splits with an explicit range utilize an inclusive end of range.
+ * @param num Number of times to split range. Pass 1 if you want to split
+ * the range in two; i.e. one split.
+ * @return Array of dividing values
+ */
+ public static byte[][] split(final byte[] a, final byte[] b,
+ boolean inclusive, final int num) {
+ byte[][] ret = new byte[num + 2][];
int i = 0;
- Iterable<byte[]> iter = iterateOnSplits(a, b, num);
- if (iter == null) return null;
+ Iterable<byte[]> iter = iterateOnSplits(a, b, inclusive, num);
+ if (iter == null)
+ return null;
for (byte[] elem : iter) {
ret[i++] = elem;
}
return ret;
}
- public static byte[][] splitPreserveAllTokens(byte[] str, char separatorChar, int[] target) {
- return splitWorker(str, 0, -1, separatorChar, true, target);
- }
-
- public static byte[][] splitPreserveAllTokens(byte[] str, int offset, int length, char separatorChar, int[] target) {
- return splitWorker(str, offset, length, separatorChar, true, target);
- }
-
- public static byte[][] splitPreserveAllTokens(byte[] str, char separatorChar) {
- return splitWorker(str, 0, -1, separatorChar, true, null);
- }
-
- public static byte[][] splitPreserveAllTokens(byte[] str, int length, char separatorChar) {
- return splitWorker(str, 0, length, separatorChar, true, null);
- }
-
/**
- * Performs the logic for the <code>split</code> and
- * <code>splitPreserveAllTokens</code> methods that do not return a
- * maximum array length.
- *
- * @param str the String to parse, may be <code>null</code>
- * @param length amount of bytes to str
- * @param separatorChar the ascii separate character
- * @param preserveAllTokens if <code>true</code>, adjacent separators are
- * @param target the projection target
- * treated as empty token separators; if <code>false</code>, adjacent
- * separators are treated as one separator.
- * @return an array of parsed Strings, <code>null</code> if null String input
- */
- private static byte[][] splitWorker(byte[] str, int offset, int length, char separatorChar, boolean preserveAllTokens, int[] target) {
- // Performance tuned for 2.0 (JDK1.4)
-
- if (str == null) {
- return null;
- }
- int len = length;
- if (len == 0) {
- return new byte[1][0];
- }else if(len < 0){
- len = str.length - offset;
- }
-
- List list = new ArrayList();
- int i = 0, start = 0;
- boolean match = false;
- boolean lastMatch = false;
- int currentTarget = 0;
- int currentIndex = 0;
- while (i < len) {
- if (str[i + offset] == separatorChar) {
- if (match || preserveAllTokens) {
- if (target == null) {
- byte[] bytes = new byte[i - start];
- System.arraycopy(str, start + offset, bytes, 0, bytes.length);
- list.add(bytes);
- } else if (target.length > currentTarget && currentIndex == target[currentTarget]) {
- byte[] bytes = new byte[i - start];
- System.arraycopy(str, start + offset, bytes, 0, bytes.length);
- list.add(bytes);
- currentTarget++;
- } else {
- list.add(null);
- }
- currentIndex++;
- match = false;
- lastMatch = true;
- }
- start = ++i;
- continue;
- }
- lastMatch = false;
- match = true;
- i++;
- }
- if (match || (preserveAllTokens && lastMatch)) {
- if (target == null) {
- byte[] bytes = new byte[i - start];
- System.arraycopy(str, start + offset, bytes, 0, bytes.length);
- list.add(bytes);
- } else if (target.length > currentTarget && currentIndex == target[currentTarget]) {
- byte[] bytes = new byte[i - start];
- System.arraycopy(str, start + offset, bytes, 0, bytes.length);
- list.add(bytes); //str.substring(start, i));
- currentTarget++;
- } else {
- list.add(null);
- }
- currentIndex++;
- }
- return (byte[][]) list.toArray(new byte[list.size()][]);
+ * Iterate over keys within the passed range, splitting at an [a,b) boundary.
+ */
+ public static Iterable<byte[]> iterateOnSplits(final byte[] a,
+ final byte[] b, final int num)
+ {
+ return iterateOnSplits(a, b, false, num);
}
+
/**
- * Iterate over keys within the passed inclusive range.
+ * Iterate over keys within the passed range.
*/
public static Iterable<byte[]> iterateOnSplits(
- final byte[] a, final byte[]b, final int num)
- {
+ final byte[] a, final byte[]b, boolean inclusive, final int num)
+ {
byte [] aPadded;
byte [] bPadded;
if (a.length < b.length) {
@@ -1556,12 +1527,15 @@ public class Bytes {
throw new IllegalArgumentException("b <= a");
}
if (num <= 0) {
- throw new IllegalArgumentException("num cannot be < 0");
+ throw new IllegalArgumentException("num cannot be <= 0");
}
byte [] prependHeader = {1, 0};
final BigInteger startBI = new BigInteger(add(prependHeader, aPadded));
final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
- final BigInteger diffBI = stopBI.subtract(startBI);
+ BigInteger diffBI = stopBI.subtract(startBI);
+ if (inclusive) {
+ diffBI = diffBI.add(BigInteger.ONE);
+ }
final BigInteger splitsBI = BigInteger.valueOf(num + 1);
if(diffBI.compareTo(splitsBI) < 0) {
return null;
@@ -1576,7 +1550,7 @@ public class Bytes {
final Iterator<byte[]> iterator = new Iterator<byte[]>() {
private int i = -1;
-
+
@Override
public boolean hasNext() {
return i < num+1;
@@ -1587,7 +1561,7 @@ public class Bytes {
i++;
if (i == 0) return a;
if (i == num + 1) return b;
-
+
BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i)));
byte [] padded = curBI.toByteArray();
if (padded[1] == 0)
@@ -1601,9 +1575,9 @@ public class Bytes {
public void remove() {
throw new UnsupportedOperationException();
}
-
+
};
-
+
return new Iterable<byte[]>() {
@Override
public Iterator<byte[]> iterator() {
@@ -1613,6 +1587,18 @@ public class Bytes {
}
/**
+ * @param bytes array to hash
+ * @param offset offset to start from
+ * @param length length to hash
+ * */
+ public static int hashCode(byte[] bytes, int offset, int length) {
+ int hash = 1;
+ for (int i = offset; i < offset + length; i++)
+ hash = (31 * hash) + (int) bytes[i];
+ return hash;
+ }
+
+ /**
* @param t operands
* @return Array of byte arrays made from passed array of Text
*/
@@ -1660,7 +1646,7 @@ public class Bytes {
* ranging from -(N + 1) to N - 1.
*/
public static int binarySearch(byte [][]arr, byte []key, int offset,
- int length, RawComparator<byte []> comparator) {
+ int length, RawComparator<?> comparator) {
int low = 0;
int high = arr.length - 1;
@@ -1673,10 +1659,10 @@ public class Bytes {
// key lives above the midpoint
if (cmp > 0)
low = mid + 1;
- // key lives below the midpoint
+ // key lives below the midpoint
else if (cmp < 0)
high = mid - 1;
- // BAM. how often does this really happen?
+ // BAM. how often does this really happen?
else
return mid;
}
@@ -1703,11 +1689,11 @@ public class Bytes {
newvalue = new byte[SIZEOF_LONG];
}
System.arraycopy(val, 0, newvalue, newvalue.length - val.length,
- val.length);
+ val.length);
val = newvalue;
} else if (val.length > SIZEOF_LONG) {
throw new IllegalArgumentException("Increment Bytes - value too big: " +
- val.length);
+ val.length);
}
if(amount == 0) return val;
if(val[0] < 0){
@@ -1770,7 +1756,7 @@ public class Bytes {
* Writes a string as a fixed-size field, padded with zeros.
*/
public static void writeStringFixedSize(final DataOutput out, String s,
- int size) throws IOException {
+ int size) throws IOException {
byte[] b = toBytes(s);
if (b.length > size) {
throw new IOException("Trying to write " + b.length + " bytes (" +
@@ -1785,7 +1771,7 @@ public class Bytes {
/**
* Reads a fixed-size field and interprets it as a string padded with zeros.
*/
- public static String readStringFixedSize(final DataInput in, int size)
+ public static String readStringFixedSize(final DataInput in, int size)
throws IOException {
byte[] b = new byte[size];
in.readFully(b);
@@ -1796,377 +1782,279 @@ public class Bytes {
return toString(b, 0, n);
}
- public static int readFully(InputStream is, byte[] buffer, int offset, int length)
- throws IOException {
- int nread = 0;
- while (nread < length) {
- int nbytes = is.read(buffer, offset + nread, length - nread);
- if (nbytes < 0) {
- return nread > 0 ? nread : nbytes;
- }
- nread += nbytes;
- }
- return nread;
+ /**
+ * Copy the byte array given in parameter and return an instance
+ * of a new byte array with the same length and the same content.
+ * @param bytes the byte array to duplicate
+ * @return a copy of the given byte array
+ */
+ public static byte [] copy(byte [] bytes) {
+ if (bytes == null) return null;
+ byte [] result = new byte[bytes.length];
+ System.arraycopy(bytes, 0, result, 0, bytes.length);
+ return result;
}
/**
- * Similar to readFully(). Skips bytes in a loop.
- * @param in The DataInput to skip bytes from
- * @param len number of bytes to skip.
- * @throws IOException if it could not skip requested number of bytes
- * for any reason (including EOF)
+ * Copy the byte array given in parameter and return an instance
+ * of a new byte array with the same length and the same content.
+ * @param bytes the byte array to copy from
+ * @return a copy of the given designated byte array
+ * @param offset
+ * @param length
*/
- public static void skipFully(DataInput in, int len) throws IOException {
- int amt = len;
- while (amt > 0) {
- long ret = in.skipBytes(amt);
- if (ret == 0) {
- // skip may return 0 even if we're not at EOF. Luckily, we can
- // use the read() method to figure out if we're at the end.
- int b = in.readByte();
- if (b == -1) {
- throw new EOFException( "Premature EOF from inputStream after " +
- "skipping " + (len - amt) + " byte(s).");
- }
- ret = 1;
- }
- amt -= ret;
- }
+ public static byte [] copy(byte [] bytes, final int offset, final int length) {
+ if (bytes == null) return null;
+ byte [] result = new byte[length];
+ System.arraycopy(bytes, offset, result, 0, length);
+ return result;
}
/**
- * Parses the byte array argument as if it was an int value and returns the
- * result. Throws NumberFormatException if the byte array does not represent an
- * int quantity.
- *
- * @return int the value represented by the argument
- * @throws NumberFormatException if the argument could not be parsed as an int quantity.
+ * Search sorted array "a" for byte "key". I can't remember if I wrote this or copied it from
+ * somewhere. (mcorgan)
+ * @param a Array to search. Entries must be sorted and unique.
+ * @param fromIndex First index inclusive of "a" to include in the search.
+ * @param toIndex Last index exclusive of "a" to include in the search.
+ * @param key The byte to search for.
+ * @return The index of key if found. If not found, return -(index + 1), where negative indicates
+ * "not found" and the "index + 1" handles the "-0" case.
*/
- public static int parseInt(byte[] bytes, int start, int length) {
- return parseInt(bytes, start, length, 10);
+ public static int unsignedBinarySearch(byte[] a, int fromIndex, int toIndex, byte key) {
+ int unsignedKey = key & 0xff;
+ int low = fromIndex;
+ int high = toIndex - 1;
+
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ int midVal = a[mid] & 0xff;
+
+ if (midVal < unsignedKey) {
+ low = mid + 1;
+ } else if (midVal > unsignedKey) {
+ high = mid - 1;
+ } else {
+ return mid; // key found
+ }
+ }
+ return -(low + 1); // key not found.
}
/**
- * Parses the byte array argument as if it was an int value and returns the
- * result. Throws NumberFormatException if the byte array does not represent an
- * int quantity. The second argument specifies the radix to use when parsing
- * the value.
+ * Treat the byte[] as an unsigned series of bytes, most significant bits first. Start by adding
+ * 1 to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes.
*
- * @param radix the base to use for conversion.
- * @return the value represented by the argument
- * @throws NumberFormatException if the argument could not be parsed as an int quantity.
+ * @param input The byte[] to increment.
+ * @return The incremented copy of "in". May be same length or 1 byte longer.
*/
- public static int parseInt(byte[] bytes, int start, int length, int radix) {
- if (bytes == null) {
- throw new NumberFormatException("String is null");
+ public static byte[] unsignedCopyAndIncrement(final byte[] input) {
+ byte[] copy = copy(input);
+ if (copy == null) {
+ throw new IllegalArgumentException("cannot increment null array");
}
- if (radix < Character.MIN_RADIX || radix > Character.MAX_RADIX) {
- throw new NumberFormatException("Invalid radix: " + radix);
- }
- if (length == 0) {
- throw new NumberFormatException("Empty byte array!");
- }
- int offset = start;
- boolean negative = bytes[start] == '-';
- if (negative || bytes[start] == '+') {
- offset++;
- if (length == 1) {
- throw new NumberFormatException(new String(bytes, start,
- length));
+ for (int i = copy.length - 1; i >= 0; --i) {
+ if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum
+ copy[i] = 0;
+ } else {
+ ++copy[i];
+ return copy;
}
}
-
- return parse(bytes, start, length, offset, radix, negative);
+ // we maxed out the array
+ byte[] out = new byte[copy.length + 1];
+ out[0] = 1;
+ System.arraycopy(copy, 0, out, 1, copy.length);
+ return out;
}
- /**
- * @param bytes
- * @param start
- * @param length
- * @param radix the base to use for conversion.
- * @param offset the starting position after the sign (if exists)
- * @param radix the base to use for conversion.
- * @param negative whether the number is negative.
- * @return the value represented by the argument
- * @throws NumberFormatException if the argument could not be parsed as an int quantity.
- */
- private static int parse(byte[] bytes, int start, int length, int offset,
- int radix, boolean negative) {
- byte separator = '.';
- int max = Integer.MIN_VALUE / radix;
- int result = 0, end = start + length;
- while (offset < end) {
- int digit = digit(bytes[offset++], radix);
- if (digit == -1) {
- if (bytes[offset - 1] == separator) {
- // We allow decimals and will return a truncated integer in that case.
- // Therefore we won't throw an exception here (checking the fractional
- // part happens below.)
- break;
- }
- throw new NumberFormatException(new String(bytes, start,
- length));
- }
- if (max > result) {
- throw new NumberFormatException(new String(bytes, start,
- length));
- }
- int next = result * radix - digit;
- if (next > result) {
- throw new NumberFormatException(new String(bytes, start,
- length));
+ public static boolean equals(List<byte[]> a, List<byte[]> b) {
+ if (a == null) {
+ if (b == null) {
+ return true;
}
- result = next;
+ return false;
}
-
- // This is the case when we've encountered a decimal separator. The fractional
- // part will not change the number, but we will verify that the fractional part
- // is well formed.
- while (offset < end) {
- int digit = digit(bytes[offset++], radix);
- if (digit == -1) {
- throw new NumberFormatException(new String(bytes, start,
- length));
- }
+ if (b == null) {
+ return false;
}
-
- if (!negative) {
- result = -result;
- if (result < 0) {
- throw new NumberFormatException(new String(bytes, start,
- length));
+ if (a.size() != b.size()) {
+ return false;
+ }
+ for (int i = 0; i < a.size(); ++i) {
+ if (!Bytes.equals(a.get(i), b.get(i))) {
+ return false;
}
}
- return result;
+ return true;
}
-
/**
- * Returns the digit represented by character b.
+ * Returns the index of the first appearance of the value {@code target} in
+ * {@code array}.
*
- * @param b The ascii code of the character
- * @param radix The radix
- * @return -1 if it's invalid
- */
- private static int digit(int b, int radix) {
- int r = -1;
- if (b >= '0' && b <= '9') {
- r = b - '0';
- } else if (b >= 'A' && b <= 'Z') {
- r = b - 'A' + 10;
- } else if (b >= 'a' && b <= 'z') {
- r = b - 'a' + 10;
- }
- if (r >= radix) {
- r = -1;
+ * @param array an array of {@code byte} values, possibly empty
+ * @param target a primitive {@code byte} value
+ * @return the least index {@code i} for which {@code array[i] == target}, or
+ * {@code -1} if no such index exists.
+ */
+ public static int indexOf(byte[] array, byte target) {
+ for (int i = 0; i < array.length; i++) {
+ if (array[i] == target) {
+ return i;
+ }
}
- return r;
+ return -1;
}
/**
- * Returns the digit represented by character b, radix is 10
+ * Returns the start position of the first occurrence of the specified {@code
+ * target} within {@code array}, or {@code -1} if there is no such occurrence.
*
- * @param b The ascii code of the character
- * @return -1 if it's invalid
- */
- private static boolean isDigit(int b) {
- return (b >= '0' && b <= '9');
- }
-
- private static final int maxExponent = 511; /* Largest possible base 10 exponent. Any
- * exponent larger than this will already
- * produce underflow or overflow, so there's
- * no need to worry about additional digits.
- */
- public static final double powersOf10[] = { /* Table giving binary powers of 10. Entry */
- 10., /* is 10^2^i. Used to convert decimal */
- 100., /* exponents into floating-point numbers. */
- 1.0e4,
- 1.0e8,
- 1.0e16,
- 1.0e32,
- 1.0e64,
- 1.0e128,
- 1.0e256
- };
-
- /**
- * Parses the byte array argument as if it was a double value and returns the
- * result. Throws NumberFormatException if the byte array does not represent a
- * double value.
+ * <p>More formally, returns the lowest index {@code i} such that {@code
+ * java.util.Arrays.copyOfRange(array, i, i + target.length)} contains exactly
+ * the same elements as {@code target}.
*
- * @return double, the value represented by the argument
- * @throws NumberFormatException if the argument could not be parsed as a double
+ * @param array the array to search for the sequence {@code target}
+ * @param target the array to search for as a sub-sequence of {@code array}
*/
- public static double parseDouble(byte[] bytes, int start, int length) {
- if (bytes == null) {
- throw new NumberFormatException("String is null");
- }
- if (length == 0) {
- throw new NumberFormatException("Empty byte array!");
- }
-
- /*
- * Strip off leading blanks
- */
- int offset = start;
- int end = start + length;
-
- while (offset < end && bytes[offset] == ' ') {
- offset++;
- }
- if (offset == end) {
- throw new NumberFormatException("blank byte array!");
+ public static int indexOf(byte[] array, byte[] target) {
+ checkNotNull(array, "array");
+ checkNotNull(target, "target");
+ if (target.length == 0) {
+ return 0;
}
- /*
- * check for a sign.
- */
- boolean sign = false;
- if (bytes[offset] == '-') {
- sign = true;
- offset++;
- } else if (bytes[offset] == '+') {
- offset++;
- }
- if (offset == end) {
- throw new NumberFormatException("the byte array only has a sign!");
- }
-
- /*
- * Count the number of digits in the mantissa (including the decimal
- * point), and also locate the decimal point.
- */
- int mantSize = 0; /* Number of digits in mantissa. */
- int decicalOffset = -1; /* Number of mantissa digits BEFORE decimal point. */
- for (; offset < end; offset++) {
- if (!isDigit(bytes[offset])) {
- if ((bytes[offset] != '.') || (decicalOffset >= 0)) {
- break;
+ outer:
+ for (int i = 0; i < array.length - target.length + 1; i++) {
+ for (int j = 0; j < target.length; j++) {
+ if (array[i + j] != target[j]) {
+ continue outer;
}
- decicalOffset = mantSize;
}
- mantSize++;
+ return i;
}
+ return -1;
+ }
- int exponentOffset = offset; /* Temporarily holds location of exponent in bytes. */
+ /**
+ * @param array an array of {@code byte} values, possibly empty
+ * @param target a primitive {@code byte} value
+ * @return {@code true} if {@code target} is present as an element anywhere in {@code array}.
+ */
+ public static boolean contains(byte[] array, byte target) {
+ return indexOf(array, target) > -1;
+ }
- /*
- * Now suck up the digits in the mantissa. Use two integers to
- * collect 9 digits each (this is faster than using floating-point).
- * If the mantissa has more than 18 digits, ignore the extras, since
- * they can't affect the value anyway.
- */
- offset -= mantSize;
- if (decicalOffset < 0) {
- decicalOffset = mantSize;
- } else {
- mantSize -= 1; /* One of the digits was the decimal point. */
- }
- int fracExponent; /* Exponent that derives from the fractional
- * part. Under normal circumstatnces, it is
- * the negative of the number of digits in F.
- * However, if I is very long, the last digits
- * of I get dropped (otherwise a long I with a
- * large negative exponent could cause an
- * unnecessary overflow on I alone). In this
- * case, fracExp is incremented one for each
- * dropped digit. */
- if (mantSize > 18) {
- fracExponent = decicalOffset - 18;
- mantSize = 18;
- } else {
- fracExponent = decicalOffset - mantSize;
- }
+ /**
+ * @param array an array of {@code byte} values, possibly empty
+ * @param target an array of {@code byte}
+ * @return {@code true} if {@code target} is present anywhere in {@code array}
+ */
+ public static boolean contains(byte[] array, byte[] target) {
+ return indexOf(array, target) > -1;
+ }
- if (mantSize == 0) {
- return 0.0;
- }
+ /**
+ * Fill given array with zeros.
+ * @param b array which needs to be filled with zeros
+ */
+ public static void zero(byte[] b) {
+ zero(b, 0, b.length);
+ }
- int frac1 = 0;
- for (; mantSize > 9; mantSize--) {
- int b = bytes[offset];
- offset++;
- if (b == '.') {
- b = bytes[offset];
- offset++;
- }
- frac1 = 10 * frac1 + (b - '0');
- }
- int frac2 = 0;
- for (; mantSize > 0; mantSize--) {
- int b = bytes[offset];
- offset++;
- if (b == '.') {
- b = bytes[offset];
- offset++;
- }
- frac2 = 10 * frac2 + (b - '0');
- }
- double fraction = (1.0e9 * frac1) + frac2;
+ /**
+ * Fill given array with zeros at the specified position.
+ * @param b
+ * @param offset
+ * @param length
+ */
+ public static void zero(byte[] b, int offset, int length) {
+ checkPositionIndex(offset, b.length, "offset");
+ checkArgument(length > 0, "length must be greater than 0");
+ checkPositionIndex(offset + length, b.length, "offset + length");
+ Arrays.fill(b, offset, offset + length, (byte) 0);
+ }
- /*
- * Skim off the exponent.
- */
- int exponent = 0; /* Exponent read from "EX" field. */
- offset = exponentOffset;
- boolean expSign = false;
-
- if (offset < end) {
- if ((bytes[offset] != 'E') && (bytes[offset] != 'e')) {
- throw new NumberFormatException(new String(bytes, start,
- length));
- }
+ private static final SecureRandom RNG = new SecureRandom();
- // (bytes[offset] == 'E') || (bytes[offset] == 'e')
- offset++;
+ /**
+ * Fill given array with random bytes.
+ * @param b array which needs to be filled with random bytes
+ */
+ public static void random(byte[] b) {
+ RNG.nextBytes(b);
+ }
- if (bytes[offset] == '-') {
- expSign = true;
- offset++;
- } else if (bytes[offset] == '+') {
- offset++;
- }
+ /**
+ * Fill given array with random bytes at the specified position.
+ * @param b
+ * @param offset
+ * @param length
+ */
+ public static void random(byte[] b, int offset, int length) {
+ checkPositionIndex(offset, b.length, "offset");
+ checkArgument(length > 0, "length must be greater than 0");
+ checkPositionIndex(offset + length, b.length, "offset + length");
+ byte[] buf = new byte[length];
+ RNG.nextBytes(buf);
+ System.arraycopy(buf, 0, b, offset, length);
+ }
- for (; offset < end; offset++) {
- if (isDigit(bytes[offset])) {
- exponent = exponent * 10 + (bytes[offset] - '0');
- } else {
- throw new NumberFormatException(new String(bytes, start,
- length));
- }
- }
+ /**
+ * Create a max byte array with the specified max byte count
+ * @param maxByteCount the length of returned byte array
+ * @return the created max byte array
+ */
+ public static byte[] createMaxByteArray(int maxByteCount) {
+ byte[] maxByteArray = new byte[maxByteCount];
+ for (int i = 0; i < maxByteArray.length; i++) {
+ maxByteArray[i] = (byte) 0xff;
}
+ return maxByteArray;
+ }
- exponent = expSign ? (fracExponent - exponent) : (fracExponent + exponent);
-
- /*
- * Generate a floating-point number that represents the exponent.
- * Do this by processing the exponent one bit at a time to combine
- * many powers of 2 of 10. Then combine the exponent with the
- * fraction.
- */
- if (exponent < 0) {
- expSign = true;
- exponent = -exponent;
- } else {
- expSign = false;
- }
- if (exponent > maxExponent) {
- throw new NumberFormatException(new String(bytes, start,
- length));
+ /**
+ * Create a byte array which is multiple given bytes
+ * @param srcBytes
+ * @param multiNum
+ * @return byte array
+ */
+ public static byte[] multiple(byte[] srcBytes, int multiNum) {
+ if (multiNum <= 0) {
+ return new byte[0];
}
-
- double dblExp = 1.0;
- for (int i = 0; exponent != 0; exponent >>= 1, i++) {
- if ((exponent & 01) == 01) {
- dblExp *= powersOf10[i];
- }
+ byte[] result = new byte[srcBytes.length * multiNum];
+ for (int i = 0; i < multiNum; i++) {
+ System.arraycopy(srcBytes, 0, result, i * srcBytes.length,
+ srcBytes.length);
}
+ return result;
+ }
- fraction = (expSign) ? (fraction / dblExp) : (fraction * dblExp);
-
- return sign ? (-fraction) : fraction;
+ /**
+ * Convert a byte array into a hex string
+ * @param b
+ */
+ public static String toHex(byte[] b) {
+ checkArgument(b.length > 0, "length must be greater than 0");
+ return String.format("%x", new BigInteger(1, b));
}
+ /**
+ * Create a byte array from a string of hash digits. The length of the
+ * string must be a multiple of 2
+ * @param hex
+ */
+ public static byte[] fromHex(String hex) {
+ checkArgument(hex.length() > 0, "length must be greater than 0");
+ checkArgument(hex.length() % 2 == 0, "length must be a multiple of 2");
+ // Make sure letters are upper case
+ hex = hex.toUpperCase();
+ byte[] b = new byte[hex.length() / 2];
+ for (int i = 0; i < b.length; i++) {
+ b[i] = (byte)((toBinaryFromHex((byte)hex.charAt(2 * i)) << 4) +
+ toBinaryFromHex((byte)hex.charAt((2 * i + 1))));
+ }
+ return b;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
new file mode 100644
index 0000000..5f309c2
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/util/BytesUtils.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util;
+
+import org.apache.hadoop.io.WritableUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Extra utilities for bytes
+ */
+public class BytesUtils {
+ /**
+ * @param n Long to make a VLong of.
+ * @return VLong as bytes array.
+ */
+ public static byte[] vlongToBytes(long n) {
+ byte [] result;
+ int offset = 0;
+ if (n >= -112 && n <= 127) {
+ result = new byte[1];
+ result[offset] = (byte) n;
+ return result;
+ }
+
+ int len = -112;
+ if (n < 0) {
+ n ^= -1L; // take one's complement'
+ len = -120;
+ }
+
+ long tmp = n;
+ while (tmp != 0) {
+ tmp = tmp >> 8;
+ len--;
+ }
+
+ int size = WritableUtils.decodeVIntSize((byte) len);
+
+ result = new byte[size];
+ result[offset++] = (byte) len;
+ len = (len < -120) ? -(len + 120) : -(len + 112);
+
+ for (int idx = len; idx != 0; idx--) {
+ int shiftbits = (idx - 1) * 8;
+ long mask = 0xFFL << shiftbits;
+ result[offset++] = (byte)((n & mask) >> shiftbits);
+ }
+ return result;
+ }
+
+ public static void writeVLong(ByteArrayOutputStream byteStream, long l) {
+ byte[] vLongBytes = vlongToBytes(l);
+ byteStream.write(vLongBytes, 0, vLongBytes.length);
+ }
+
+ /**
+ * Converts a char array to a ascii byte array.
+ *
+ * @param chars string
+ * @return the byte array
+ */
+ static byte[] toASCIIBytes(char[] chars) {
+ byte[] buffer = new byte[chars.length];
+ for (int i = 0; i < chars.length; i++) {
+ buffer[i] = (byte) chars[i];
+ }
+ return buffer;
+ }
+
+ public static byte[][] splitPreserveAllTokens(byte[] str, char separatorChar, int[] target) {
+ return splitWorker(str, 0, -1, separatorChar, true, target);
+ }
+
+ public static byte[][] splitPreserveAllTokens(byte[] str, int offset, int length, char separatorChar, int[] target) {
+ return splitWorker(str, offset, length, separatorChar, true, target);
+ }
+
+ public static byte[][] splitPreserveAllTokens(byte[] str, char separatorChar) {
+ return splitWorker(str, 0, -1, separatorChar, true, null);
+ }
+
+ public static byte[][] splitPreserveAllTokens(byte[] str, int length, char separatorChar) {
+ return splitWorker(str, 0, length, separatorChar, true, null);
+ }
+
+ /**
+ * Performs the logic for the <code>split</code> and
+ * <code>splitPreserveAllTokens</code> methods that do not return a
+ * maximum array length.
+ *
+ * @param str the String to parse, may be <code>null</code>
+ * @param length amount of bytes to str
+ * @param separatorChar the ascii separate character
+ * @param preserveAllTokens if <code>true</code>, adjacent separators are
+ * @param target the projection target
+ * treated as empty token separators; if <code>false</code>, adjacent
+ * separators are treated as one separator.
+ * @return an array of parsed Strings, <code>null</code> if null String input
+ */
+ private static byte[][] splitWorker(byte[] str, int offset, int length, char separatorChar,
+ boolean preserveAllTokens, int[] target) {
+ // Performance tuned for 2.0 (JDK1.4)
+
+ if (str == null) {
+ return null;
+ }
+ int len = length;
+ if (len == 0) {
+ return new byte[1][0];
+ }else if(len < 0){
+ len = str.length - offset;
+ }
+
+ List list = new ArrayList();
+ int i = 0, start = 0;
+ boolean match = false;
+ boolean lastMatch = false;
+ int currentTarget = 0;
+ int currentIndex = 0;
+ while (i < len) {
+ if (str[i + offset] == separatorChar) {
+ if (match || preserveAllTokens) {
+ if (target == null) {
+ byte[] bytes = new byte[i - start];
+ System.arraycopy(str, start + offset, bytes, 0, bytes.length);
+ list.add(bytes);
+ } else if (target.length > currentTarget && currentIndex == target[currentTarget]) {
+ byte[] bytes = new byte[i - start];
+ System.arraycopy(str, start + offset, bytes, 0, bytes.length);
+ list.add(bytes);
+ currentTarget++;
+ } else {
+ list.add(null);
+ }
+ currentIndex++;
+ match = false;
+ lastMatch = true;
+ }
+ start = ++i;
+ continue;
+ }
+ lastMatch = false;
+ match = true;
+ i++;
+ }
+ if (match || (preserveAllTokens && lastMatch)) {
+ if (target == null) {
+ byte[] bytes = new byte[i - start];
+ System.arraycopy(str, start + offset, bytes, 0, bytes.length);
+ list.add(bytes);
+ } else if (target.length > currentTarget && currentIndex == target[currentTarget]) {
+ byte[] bytes = new byte[i - start];
+ System.arraycopy(str, start + offset, bytes, 0, bytes.length);
+ list.add(bytes); //str.substring(start, i));
+ currentTarget++;
+ } else {
+ list.add(null);
+ }
+ currentIndex++;
+ }
+ return (byte[][]) list.toArray(new byte[list.size()][]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java
index c8205eb..d52b804 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java
@@ -20,7 +20,24 @@ package org.apache.tajo.util;
public class NumberUtil {
- public static long unsigned32(int n) {
+ public static final double[] powersOf10 = { /* Table giving binary powers of 10. Entry */
+ 10., /* is 10^2^i. Used to convert decimal */
+ 100., /* exponents into floating-point numbers. */
+ 1.0e4,
+ 1.0e8,
+ 1.0e16,
+ 1.0e32,
+ 1.0e64,
+ 1.0e128,
+ 1.0e256
+ };
+ private static final int maxExponent = 511; /* Largest possible base 10 exponent. Any
+ * exponent larger than this will already
+ * produce underflow or overflow, so there's
+ * no need to worry about additional digits.
+ */
+
+ public static long unsigned32(int n) {
return n & 0xFFFFFFFFL;
}
@@ -29,27 +46,27 @@ public class NumberUtil {
}
public static byte[] toAsciiBytes(Number i){
- return Bytes.toASCIIBytes(String.valueOf(i).toCharArray());
+ return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray());
}
public static byte[] toAsciiBytes(short i){
- return Bytes.toASCIIBytes(String.valueOf(i).toCharArray());
+ return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray());
}
public static byte[] toAsciiBytes(int i){
- return Bytes.toASCIIBytes(String.valueOf(i).toCharArray());
+ return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray());
}
public static byte[] toAsciiBytes(long i){
- return Bytes.toASCIIBytes(String.valueOf(i).toCharArray());
+ return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray());
}
public static byte[] toAsciiBytes(float i){
- return Bytes.toASCIIBytes(String.valueOf(i).toCharArray());
+ return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray());
}
public static byte[] toAsciiBytes(double i){
- return Bytes.toASCIIBytes(String.valueOf(i).toCharArray());
+ return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray());
}
private static void benchmark(int num){
@@ -75,7 +92,320 @@ public class NumberUtil {
+ " ms, " + "Total: " + size / (1024 * 1024) + "MB");
}
- public static void main(String[] args) throws Exception {
- benchmark(1024 * 1024 * 10);
+ /**
+ * Returns the digit represented by character b.
+ *
+ * @param b The ascii code of the character
+ * @param radix The radix
+ * @return -1 if it's invalid
+ */
+ static int digit(int b, int radix) {
+ int r = -1;
+ if (b >= '0' && b <= '9') {
+ r = b - '0';
+ } else if (b >= 'A' && b <= 'Z') {
+ r = b - 'A' + 10;
+ } else if (b >= 'a' && b <= 'z') {
+ r = b - 'a' + 10;
+ }
+ if (r >= radix) {
+ r = -1;
+ }
+ return r;
+ }
+
+ /**
+ * Returns the digit represented by character b, radix is 10
+ *
+ * @param b The ascii code of the character
+ * @return -1 if it's invalid
+ */
+ private static boolean isDigit(int b) {
+ return (b >= '0' && b <= '9');
+ }
+
+ /**
+ * Parses the byte array argument as if it was a double value and returns the
+ * result. Throws NumberFormatException if the byte array does not represent a
+ * double value.
+ *
+ * @return double, the value represented by the argument
+ * @throws NumberFormatException if the argument could not be parsed as a double
+ */
+ public static double parseDouble(byte[] bytes, int start, int length) {
+ if (bytes == null) {
+ throw new NumberFormatException("String is null");
+ }
+ if (length == 0) {
+ throw new NumberFormatException("Empty byte array!");
+ }
+
+ /*
+ * Strip off leading blanks
+ */
+ int offset = start;
+ int end = start + length;
+
+ while (offset < end && bytes[offset] == ' ') {
+ offset++;
+ }
+ if (offset == end) {
+ throw new NumberFormatException("blank byte array!");
+ }
+
+ /*
+ * check for a sign.
+ */
+ boolean sign = false;
+ if (bytes[offset] == '-') {
+ sign = true;
+ offset++;
+ } else if (bytes[offset] == '+') {
+ offset++;
+ }
+ if (offset == end) {
+ throw new NumberFormatException("the byte array only has a sign!");
+ }
+
+ /*
+ * Count the number of digits in the mantissa (including the decimal
+ * point), and also locate the decimal point.
+ */
+ int mantSize = 0; /* Number of digits in mantissa. */
+ int decicalOffset = -1; /* Number of mantissa digits BEFORE decimal point. */
+ for (; offset < end; offset++) {
+ if (!isDigit(bytes[offset])) {
+ if ((bytes[offset] != '.') || (decicalOffset >= 0)) {
+ break;
+ }
+ decicalOffset = mantSize;
+ }
+ mantSize++;
+ }
+
+ int exponentOffset = offset; /* Temporarily holds location of exponent in bytes. */
+
+ /*
+ * Now suck up the digits in the mantissa. Use two integers to
+ * collect 9 digits each (this is faster than using floating-point).
+ * If the mantissa has more than 18 digits, ignore the extras, since
+ * they can't affect the value anyway.
+ */
+ offset -= mantSize;
+ if (decicalOffset < 0) {
+ decicalOffset = mantSize;
+ } else {
+ mantSize -= 1; /* One of the digits was the decimal point. */
+ }
+ int fracExponent; /* Exponent that derives from the fractional
+ * part. Under normal circumstatnces, it is
+ * the negative of the number of digits in F.
+ * However, if I is very long, the last digits
+ * of I get dropped (otherwise a long I with a
+ * large negative exponent could cause an
+ * unnecessary overflow on I alone). In this
+ * case, fracExp is incremented one for each
+ * dropped digit. */
+ if (mantSize > 18) {
+ fracExponent = decicalOffset - 18;
+ mantSize = 18;
+ } else {
+ fracExponent = decicalOffset - mantSize;
+ }
+
+ if (mantSize == 0) {
+ return 0.0;
+ }
+
+ int frac1 = 0;
+ for (; mantSize > 9; mantSize--) {
+ int b = bytes[offset];
+ offset++;
+ if (b == '.') {
+ b = bytes[offset];
+ offset++;
+ }
+ frac1 = 10 * frac1 + (b - '0');
+ }
+ int frac2 = 0;
+ for (; mantSize > 0; mantSize--) {
+ int b = bytes[offset];
+ offset++;
+ if (b == '.') {
+ b = bytes[offset];
+ offset++;
+ }
+ frac2 = 10 * frac2 + (b - '0');
+ }
+ double fraction = (1.0e9 * frac1) + frac2;
+
+ /*
+ * Skim off the exponent.
+ */
+ int exponent = 0; /* Exponent read from "EX" field. */
+ offset = exponentOffset;
+ boolean expSign = false;
+
+ if (offset < end) {
+ if ((bytes[offset] != 'E') && (bytes[offset] != 'e')) {
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+
+ // (bytes[offset] == 'E') || (bytes[offset] == 'e')
+ offset++;
+
+ if (bytes[offset] == '-') {
+ expSign = true;
+ offset++;
+ } else if (bytes[offset] == '+') {
+ offset++;
+ }
+
+ for (; offset < end; offset++) {
+ if (isDigit(bytes[offset])) {
+ exponent = exponent * 10 + (bytes[offset] - '0');
+ } else {
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+ }
+ }
+
+ exponent = expSign ? (fracExponent - exponent) : (fracExponent + exponent);
+
+ /*
+ * Generate a floating-point number that represents the exponent.
+ * Do this by processing the exponent one bit at a time to combine
+ * many powers of 2 of 10. Then combine the exponent with the
+ * fraction.
+ */
+ if (exponent < 0) {
+ expSign = true;
+ exponent = -exponent;
+ } else {
+ expSign = false;
+ }
+ if (exponent > maxExponent) {
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+
+ double dblExp = 1.0;
+ for (int i = 0; exponent != 0; exponent >>= 1, i++) {
+ if ((exponent & 01) == 01) {
+ dblExp *= powersOf10[i];
+ }
+ }
+
+ fraction = (expSign) ? (fraction / dblExp) : (fraction * dblExp);
+
+ return sign ? (-fraction) : fraction;
+ }
+
+ /**
+ * Parses the byte array argument as if it was an int value and returns the
+ * result. Throws NumberFormatException if the byte array does not represent an
+ * int quantity.
+ *
+ * @return int the value represented by the argument
+ * @throws NumberFormatException if the argument could not be parsed as an int quantity.
+ */
+ public static int parseInt(byte[] bytes, int start, int length) {
+ return parseInt(bytes, start, length, 10);
+ }
+
+ /**
+ * Parses the byte array argument as if it was an int value and returns the
+ * result. Throws NumberFormatException if the byte array does not represent an
+ * int quantity. The second argument specifies the radix to use when parsing
+ * the value.
+ *
+ * @param radix the base to use for conversion.
+ * @return the value represented by the argument
+ * @throws NumberFormatException if the argument could not be parsed as an int quantity.
+ */
+ public static int parseInt(byte[] bytes, int start, int length, int radix) {
+ if (bytes == null) {
+ throw new NumberFormatException("String is null");
+ }
+ if (radix < Character.MIN_RADIX || radix > Character.MAX_RADIX) {
+ throw new NumberFormatException("Invalid radix: " + radix);
+ }
+ if (length == 0) {
+ throw new NumberFormatException("Empty byte array!");
+ }
+ int offset = start;
+ boolean negative = bytes[start] == '-';
+ if (negative || bytes[start] == '+') {
+ offset++;
+ if (length == 1) {
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+ }
+
+ return parseIntInternal(bytes, start, length, offset, radix, negative);
+ }
+
+ /**
+ * @param bytes
+ * @param start
+ * @param length
+ * @param radix the base to use for conversion.
+ * @param offset the starting position after the sign (if exists)
+ * @param radix the base to use for conversion.
+ * @param negative whether the number is negative.
+ * @return the value represented by the argument
+ * @throws NumberFormatException if the argument could not be parsed as an int quantity.
+ */
+ private static int parseIntInternal(byte[] bytes, int start, int length, int offset,
+ int radix, boolean negative) {
+ byte separator = '.';
+ int max = Integer.MIN_VALUE / radix;
+ int result = 0, end = start + length;
+ while (offset < end) {
+ int digit = digit(bytes[offset++], radix);
+ if (digit == -1) {
+ if (bytes[offset - 1] == separator) {
+ // We allow decimals and will return a truncated integer in that case.
+ // Therefore we won't throw an exception here (checking the fractional
+ // part happens below.)
+ break;
+ }
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+ if (max > result) {
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+ int next = result * radix - digit;
+ if (next > result) {
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+ result = next;
+ }
+
+ // This is the case when we've encountered a decimal separator. The fractional
+ // part will not change the number, but we will verify that the fractional part
+ // is well formed.
+ while (offset < end) {
+ int digit = digit(bytes[offset++], radix);
+ if (digit == -1) {
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+ }
+
+ if (!negative) {
+ result = -result;
+ if (result < 0) {
+ throw new NumberFormatException(new String(bytes, start,
+ length));
+ }
+ }
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-common/src/test/java/org/apache/tajo/util/TestBytes.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestBytes.java b/tajo-common/src/test/java/org/apache/tajo/util/TestBytes.java
deleted file mode 100644
index c26bf7d..0000000
--- a/tajo-common/src/test/java/org/apache/tajo/util/TestBytes.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.util;
-
-import org.apache.commons.lang.StringUtils;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-public class TestBytes {
-
- @Test
- public void testAsciiBytes() {
- String asciiText = "abcde 12345 ABCDE";
- assertArrayEquals(asciiText.getBytes(), Bytes.toASCIIBytes(asciiText.toCharArray()));
- }
-
- @Test
- public void testSplitBytes() {
- String text = "abcde|12345|ABCDE";
- char separatorChar = '|';
-
- String[] textArray = StringUtils.splitPreserveAllTokens(text, separatorChar);
- byte[][] bytesArray = Bytes.splitPreserveAllTokens(text.getBytes(), separatorChar);
-
- assertEquals(textArray.length, bytesArray.length);
- for (int i = 0; i < textArray.length; i++){
- assertArrayEquals(textArray[i].getBytes(), bytesArray[i]);
- }
- }
-
- @Test
- public void testSplitProjectionBytes() {
- String text = "abcde|12345|ABCDE";
- int[] target = new int[]{ 1 };
- char separatorChar = '|';
-
- String[] textArray = StringUtils.splitPreserveAllTokens(text, separatorChar);
- byte[][] bytesArray = Bytes.splitPreserveAllTokens(text.getBytes(), separatorChar, target);
-
- assertEquals(textArray.length, bytesArray.length);
-
- assertNull(bytesArray[0]);
- assertNotNull(bytesArray[1]);
- assertArrayEquals(textArray[1].getBytes(), bytesArray[1]);
- assertNull(bytesArray[2]);
- }
-
- @Test
- public void testParseInt() {
- int int1 = 0;
- byte[] bytes1 = Double.toString(int1).getBytes();
- assertEquals(int1, Bytes.parseInt(bytes1, 0, bytes1.length));
-
- int int2 = -7;
- byte[] bytes2 = Double.toString(int2).getBytes();
- assertEquals(int2, Bytes.parseInt(bytes2, 0, bytes2.length));
-
- int int3 = +128;
- byte[] bytes3 = Double.toString(int3).getBytes();
- assertEquals(int3, Bytes.parseInt(bytes3, 0, bytes3.length));
-
- int int4 = 4;
- byte[] bytes4 = Double.toString(int4).getBytes();
- assertEquals(int4, Bytes.parseInt(bytes4, 0, bytes4.length));
-
- byte[] bytes5 = "0123-456789".getBytes();
- assertEquals(-456, Bytes.parseInt(bytes5, 4, 4));
-
- }
-
- @Test
- public void testParseDouble() {
- double double1 = 2.0015E7;
- byte[] bytes1 = Double.toString(double1).getBytes();
- assertEquals(double1, Bytes.parseDouble(bytes1, 0, bytes1.length), 0.0);
-
- double double2 = 1.345E-7;
- byte[] bytes2 = Double.toString(double2).getBytes();
- assertEquals(double2, Bytes.parseDouble(bytes2, 0, bytes2.length), 0.0);
-
- double double3 = -1.345E-7;
- byte[] bytes3 = Double.toString(double3).getBytes();
- assertEquals(double3, Bytes.parseDouble(bytes3, 0, bytes3.length), 0.0);
-
- double double4 = 4;
- byte[] bytes4 = Double.toString(double4).getBytes();
- assertEquals(double4, Bytes.parseDouble(bytes4, 0, bytes4.length), 0.0);
-
- byte[] bytes5 = "0123456789.012345E012345".getBytes();
- assertEquals(6789.012345E01, Bytes.parseDouble(bytes5, 6, 14), 0.0);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/85b7031a/tajo-common/src/test/java/org/apache/tajo/util/TestNumberUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestNumberUtil.java b/tajo-common/src/test/java/org/apache/tajo/util/TestNumberUtil.java
index 25851cd..e4e3b62 100644
--- a/tajo-common/src/test/java/org/apache/tajo/util/TestNumberUtil.java
+++ b/tajo-common/src/test/java/org/apache/tajo/util/TestNumberUtil.java
@@ -23,9 +23,9 @@ import org.junit.Test;
import java.util.Random;
import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
public class TestNumberUtil {
-
@Test
public void testNumberToAsciiBytes() {
Random r = new Random(System.currentTimeMillis());
@@ -45,4 +45,49 @@ public class TestNumberUtil {
n = r.nextDouble();
assertArrayEquals(String.valueOf(n.doubleValue()).getBytes(), NumberUtil.toAsciiBytes(n.doubleValue()));
}
+
+ @Test
+ public void testParseInt() {
+ int int1 = 0;
+ byte[] bytes1 = Double.toString(int1).getBytes();
+ assertEquals(int1, NumberUtil.parseInt(bytes1, 0, bytes1.length));
+
+ int int2 = -7;
+ byte[] bytes2 = Double.toString(int2).getBytes();
+ assertEquals(int2, NumberUtil.parseInt(bytes2, 0, bytes2.length));
+
+ int int3 = +128;
+ byte[] bytes3 = Double.toString(int3).getBytes();
+ assertEquals(int3, NumberUtil.parseInt(bytes3, 0, bytes3.length));
+
+ int int4 = 4;
+ byte[] bytes4 = Double.toString(int4).getBytes();
+ assertEquals(int4, NumberUtil.parseInt(bytes4, 0, bytes4.length));
+
+ byte[] bytes5 = "0123-456789".getBytes();
+ assertEquals(-456, NumberUtil.parseInt(bytes5, 4, 4));
+
+ }
+
+ @Test
+ public void testParseDouble() {
+ double double1 = 2.0015E7;
+ byte[] bytes1 = Double.toString(double1).getBytes();
+ assertEquals(double1, NumberUtil.parseDouble(bytes1, 0, bytes1.length), 0.0);
+
+ double double2 = 1.345E-7;
+ byte[] bytes2 = Double.toString(double2).getBytes();
+ assertEquals(double2, NumberUtil.parseDouble(bytes2, 0, bytes2.length), 0.0);
+
+ double double3 = -1.345E-7;
+ byte[] bytes3 = Double.toString(double3).getBytes();
+ assertEquals(double3, NumberUtil.parseDouble(bytes3, 0, bytes3.length), 0.0);
+
+ double double4 = 4;
+ byte[] bytes4 = Double.toString(double4).getBytes();
+ assertEquals(double4, NumberUtil.parseDouble(bytes4, 0, bytes4.length), 0.0);
+
+ byte[] bytes5 = "0123456789.012345E012345".getBytes();
+ assertEquals(6789.012345E01, NumberUtil.parseDouble(bytes5, 6, 14), 0.0);
+ }
}
[3/7] git commit: TAJO-939: Refactoring the column resolver in
LogicalPlan.
Posted by ji...@apache.org.
TAJO-939: Refactoring the column resolver in LogicalPlan.
Closes #71
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/862435bc
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/862435bc
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/862435bc
Branch: refs/heads/index_support
Commit: 862435bc9622f9a8dde07417158b904554c36e60
Parents: 85b7031
Author: Hyunsik Choi <hy...@apache.org>
Authored: Tue Jul 22 11:29:50 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Tue Jul 22 11:29:50 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../java/org/apache/tajo/catalog/Schema.java | 19 +-
.../tajo/engine/planner/ExprAnnotator.java | 26 +-
.../tajo/engine/planner/ExprNormalizer.java | 13 +-
.../apache/tajo/engine/planner/LogicalPlan.java | 256 ++--------------
.../engine/planner/LogicalPlanPreprocessor.java | 32 +-
.../tajo/engine/planner/LogicalPlanner.java | 49 ++--
.../planner/logical/TableSubQueryNode.java | 1 -
.../planner/nameresolver/NameResolver.java | 291 +++++++++++++++++++
.../planner/nameresolver/NameResolvingMode.java | 80 +++++
.../planner/nameresolver/ResolverByLegacy.java | 126 ++++++++
.../planner/nameresolver/ResolverByRels.java | 38 +++
.../nameresolver/ResolverByRelsAndSubExprs.java | 42 +++
.../nameresolver/ResolverBySubExprsAndRels.java | 42 +++
.../tajo/engine/eval/TestEvalTreeUtil.java | 4 +-
.../tajo/engine/query/TestCaseByCases.java | 7 +
.../tajo/engine/query/TestSelectQuery.java | 27 ++
.../TestCaseByCases/testTAJO917Case1.sql | 13 +
.../TestSelectQuery/testNonQualifiedNames.sql | 1 +
...testSelectColumnAliasExistingInRelation1.sql | 1 +
...testSelectColumnAliasExistingInRelation2.sql | 1 +
.../TestCaseByCases/testTAJO917Case1.result | 7 +
.../testNonQualifiedNames.result | 7 +
...tSelectColumnAliasExistingInRelation1.result | 4 +
...tSelectColumnAliasExistingInRelation2.result | 7 +
25 files changed, 830 insertions(+), 266 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 86cef01..8208fd7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -97,6 +97,8 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-939: Refactoring the column resolver in LogicalPlan. (hyunsik)
+
TAJO-965: Upgrade Bytes class and move some methods to others. (hyunsik)
TAJO-961: TajoCli should exit when at least one query faces error while
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index 35d2fe9..3aee5d2 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@ -95,11 +95,22 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
* @param qualifier The qualifier
*/
public void setQualifier(String qualifier) {
+ Schema copy = null;
+ try {
+ copy = (Schema) clone();
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+
+ fields.clear();
fieldsByQualifiedName.clear();
- for (int i = 0; i < size(); i++) {
- Column column = fields.get(i);
- fields.set(i, new Column(qualifier + "." + column.getSimpleName(), column.getDataType()));
- fieldsByQualifiedName.put(fields.get(i).getQualifiedName(), i);
+ fieldsByName.clear();
+
+ Column newColumn;
+ for (int i = 0; i < copy.size(); i++) {
+ Column column = copy.getColumn(i);
+ newColumn = new Column(qualifier + "." + column.getSimpleName(), column.getDataType());
+ addColumn(newColumn);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java
index 2c386b2..39fd08a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java
@@ -32,6 +32,8 @@ import org.apache.tajo.engine.eval.*;
import org.apache.tajo.engine.function.AggFunction;
import org.apache.tajo.engine.function.GeneralFunction;
import org.apache.tajo.engine.planner.logical.NodeType;
+import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
+import org.apache.tajo.engine.planner.nameresolver.NameResolver;
import org.apache.tajo.exception.InternalException;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
@@ -66,16 +68,19 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva
static class Context {
LogicalPlan plan;
LogicalPlan.QueryBlock currentBlock;
+ NameResolvingMode columnRsvLevel;
- public Context(LogicalPlan plan, LogicalPlan.QueryBlock block) {
+ public Context(LogicalPlan plan, LogicalPlan.QueryBlock block, NameResolvingMode colRsvLevel) {
this.plan = plan;
this.currentBlock = block;
+ this.columnRsvLevel = colRsvLevel;
}
}
- public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr)
+ public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr,
+ NameResolvingMode colRsvLevel)
throws PlanningException {
- Context context = new Context(plan, block);
+ Context context = new Context(plan, block, colRsvLevel);
return AlgebraicUtil.eliminateConstantExprs(visit(context, new Stack<Expr>(), expr));
}
@@ -540,7 +545,20 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva
@Override
public EvalNode visitColumnReference(Context ctx, Stack<Expr> stack, ColumnReferenceExpr expr)
throws PlanningException {
- Column column = ctx.plan.resolveColumn(ctx.currentBlock, expr);
+ Column column;
+
+ switch (ctx.columnRsvLevel) {
+ case LEGACY:
+ column = ctx.plan.resolveColumn(ctx.currentBlock, expr);
+ break;
+ case RELS_ONLY:
+ case RELS_AND_SUBEXPRS:
+ case SUBEXPRS_AND_RELS:
+ column = NameResolver.resolve(ctx.plan, ctx.currentBlock, expr, ctx.columnRsvLevel);
+ break;
+ default:
+ throw new PlanningException("Unsupported column resolving level: " + ctx.columnRsvLevel.name());
+ }
return new FieldEval(column);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java
index 81bbd41..5b61b74 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java
@@ -19,15 +19,15 @@
package org.apache.tajo.engine.planner;
import com.google.common.collect.Sets;
-import com.google.common.collect.Sets;
import org.apache.tajo.algebra.*;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.engine.exception.NoSuchColumnException;
+import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
+import org.apache.tajo.engine.planner.nameresolver.NameResolver;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
-import java.util.Set;
import java.util.Stack;
/**
@@ -330,11 +330,18 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR
@Override
public Expr visitColumnReference(ExprNormalizedResult ctx, Stack<Expr> stack, ColumnReferenceExpr expr)
throws PlanningException {
+
+ if (ctx.block.isAliasedName(expr.getCanonicalName())) {
+ String originalName = ctx.block.getOriginalName(expr.getCanonicalName());
+ expr.setName(originalName);
+ return expr;
+ }
// if a column reference is not qualified, it finds and sets the qualified column name.
if (!(expr.hasQualifier() && CatalogUtil.isFQTableName(expr.getQualifier()))) {
if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName()) && expr.getType() == OpType.Column) {
try {
- String normalized = ctx.plan.getNormalizedColumnName(ctx.block, expr);
+ String normalized =
+ NameResolver.resolve(ctx.plan, ctx.block, expr, NameResolvingMode.LEGACY).getQualifiedName();
expr.setName(normalized);
} catch (NoSuchColumnException nsc) {
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
index 4e1d313..86bacef 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
@@ -19,23 +19,20 @@
package org.apache.tajo.engine.planner;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
import org.apache.commons.lang.ObjectUtils;
import org.apache.tajo.algebra.*;
import org.apache.tajo.annotation.NotThreadSafe;
-import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.engine.eval.EvalNode;
-import org.apache.tajo.engine.exception.AmbiguousFieldException;
-import org.apache.tajo.engine.exception.NoSuchColumnException;
-import org.apache.tajo.engine.exception.VerifyException;
import org.apache.tajo.engine.planner.graph.DirectedGraphCursor;
import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.engine.planner.logical.LogicalRootNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.planner.logical.RelationNode;
+import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
+import org.apache.tajo.engine.planner.nameresolver.NameResolver;
import org.apache.tajo.util.TUtil;
import java.lang.reflect.Constructor;
@@ -49,7 +46,6 @@ public class LogicalPlan {
/** the prefix character for virtual tables */
public static final char VIRTUAL_TABLE_PREFIX='#';
public static final char NONAMED_COLUMN_PREFIX='?';
- public static final char NONAMED_WINDOW_PREFIX='^';
/** it indicates the root block */
public static final String ROOT_BLOCK = VIRTUAL_TABLE_PREFIX + "ROOT";
@@ -58,7 +54,6 @@ public class LogicalPlan {
private int nextPid = 0;
private Integer noNameBlockId = 0;
private Integer noNameColumnId = 0;
- private Integer noNameWindowId = 0;
/** a map from between a block name to a block plan */
private Map<String, QueryBlock> queryBlocks = new LinkedHashMap<String, QueryBlock>();
@@ -266,230 +261,8 @@ public class LogicalPlan {
return queryBlockGraph;
}
- public String getNormalizedColumnName(QueryBlock block, ColumnReferenceExpr columnRef)
- throws PlanningException {
- Column found = resolveColumn(block, columnRef);
- if (found == null) {
- throw new NoSuchColumnException(columnRef.getCanonicalName());
- }
- return found.getQualifiedName();
- }
-
- public String resolveDatabase(QueryBlock block, String tableName) throws PlanningException {
- List<String> found = new ArrayList<String>();
- for (RelationNode relation : block.getRelations()) {
- // check alias name or table name
- if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) ||
- CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) {
- // obtain the database name
- found.add(CatalogUtil.extractQualifier(relation.getTableName()));
- }
- }
-
- if (found.size() == 0) {
- return null;
- } else if (found.size() > 1) {
- throw new PlanningException("Ambiguous table name \"" + tableName + "\"");
- }
-
- return found.get(0);
- }
-
- /**
- * It resolves a column.
- */
public Column resolveColumn(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException {
- if (columnRef.hasQualifier()) {
- return resolveColumnWithQualifier(block, columnRef);
- } else {
- return resolveColumnWithoutQualifier(block, columnRef);
- }
- }
-
- private Column resolveColumnWithQualifier(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException {
- String qualifier;
- String canonicalName;
- String qualifiedName;
-
- if (CatalogUtil.isFQTableName(columnRef.getQualifier())) {
- qualifier = columnRef.getQualifier();
- canonicalName = columnRef.getCanonicalName();
- } else {
- String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier());
- if (resolvedDatabaseName == null) {
- throw new NoSuchColumnException(columnRef.getQualifier());
- }
- qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier());
- canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
- }
- qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
-
- RelationNode relationOp = block.getRelation(qualifier);
-
- // if a column name is outside of this query block
- if (relationOp == null) {
- // TODO - nested query can only refer outer query block? or not?
- for (QueryBlock eachBlock : queryBlocks.values()) {
- if (eachBlock.existsRelation(qualifier)) {
- relationOp = eachBlock.getRelation(qualifier);
- }
- }
- }
-
- // If we cannot find any relation against a qualified column name
- if (relationOp == null) {
- throw new NoSuchColumnException(canonicalName);
- }
-
- if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) {
- String changedName = CatalogUtil.buildFQName(
- relationOp.getCanonicalName(),
- CatalogUtil.extractSimpleName(canonicalName));
- canonicalName = changedName;
- }
-
- Schema schema = relationOp.getTableSchema();
- Column column = schema.getColumn(canonicalName);
- if (column == null) {
- throw new NoSuchColumnException(canonicalName);
- }
-
- // If code reach here, a column is found.
- // But, it may be aliased from bottom logical node.
- // If the column is aliased, the found name may not be used in upper node.
-
- // Here, we try to check if column reference is already aliased.
- // If so, it replaces the name with aliased name.
- LogicalNode currentNode = block.getCurrentNode();
-
- // The condition (currentNode.getInSchema().contains(column)) means
- // the column can be used at the current node. So, we don't need to find aliase name.
- Schema currentNodeSchema = null;
- if (currentNode != null) {
- if (currentNode instanceof RelationNode) {
- currentNodeSchema = ((RelationNode) currentNode).getTableSchema();
- } else {
- currentNodeSchema = currentNode.getInSchema();
- }
- }
-
- if (currentNode != null && !currentNodeSchema.contains(column)
- && currentNode.getType() != NodeType.TABLE_SUBQUERY) {
- List<Column> candidates = TUtil.newList();
- if (block.namedExprsMgr.isAliased(qualifiedName)) {
- String alias = block.namedExprsMgr.getAlias(canonicalName);
- Column found = resolveColumn(block, new ColumnReferenceExpr(alias));
- if (found != null) {
- candidates.add(found);
- }
- }
- if (!candidates.isEmpty()) {
- return ensureUniqueColumn(candidates);
- }
- }
-
- return column;
- }
-
- private Column resolveColumnWithoutQualifier(QueryBlock block,
- ColumnReferenceExpr columnRef)throws PlanningException {
-
- List<Column> candidates = TUtil.newList();
-
- // It tries to find a full qualified column name from all relations in the current block.
- for (RelationNode rel : block.getRelations()) {
- Column found = rel.getTableSchema().getColumn(columnRef.getName());
- if (found != null) {
- candidates.add(found);
- }
- }
-
- if (!candidates.isEmpty()) {
- return ensureUniqueColumn(candidates);
- }
-
- // Trying to find the column within the current block
- if (block.currentNode != null && block.currentNode.getInSchema() != null) {
- Column found = block.currentNode.getInSchema().getColumn(columnRef.getCanonicalName());
- if (found != null) {
- return found;
- }
- }
-
- if (block.getLatestNode() != null) {
- Column found = block.getLatestNode().getOutSchema().getColumn(columnRef.getName());
- if (found != null) {
- return found;
- }
- }
-
-
- // Trying to find columns from aliased references.
- if (block.namedExprsMgr.isAliased(columnRef.getCanonicalName())) {
- String originalName = block.namedExprsMgr.getAlias(columnRef.getCanonicalName());
- Column found = resolveColumn(block, new ColumnReferenceExpr(originalName));
- if (found != null) {
- candidates.add(found);
- }
- }
- if (!candidates.isEmpty()) {
- return ensureUniqueColumn(candidates);
- }
-
- // This is an exception case. It means that there are some bugs in other parts.
- LogicalNode blockRootNode = block.getRoot();
- if (blockRootNode != null && blockRootNode.getOutSchema().getColumn(columnRef.getCanonicalName()) != null) {
- throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName());
- }
-
- // Trying to find columns from other relations in other blocks
- for (QueryBlock eachBlock : queryBlocks.values()) {
- for (RelationNode rel : eachBlock.getRelations()) {
- Column found = rel.getTableSchema().getColumn(columnRef.getName());
- if (found != null) {
- candidates.add(found);
- }
- }
- }
-
- if (!candidates.isEmpty()) {
- return ensureUniqueColumn(candidates);
- }
-
- // Trying to find columns from schema in current block.
- if (block.getSchema() != null) {
- Column found = block.getSchema().getColumn(columnRef.getName());
- if (found != null) {
- candidates.add(found);
- }
- }
-
- if (!candidates.isEmpty()) {
- return ensureUniqueColumn(candidates);
- }
-
- throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName());
- }
-
- private static Column ensureUniqueColumn(List<Column> candidates)
- throws VerifyException {
- if (candidates.size() == 1) {
- return candidates.get(0);
- } else if (candidates.size() > 2) {
- StringBuilder sb = new StringBuilder();
- boolean first = true;
- for (Column column : candidates) {
- if (first) {
- first = false;
- } else {
- sb.append(", ");
- }
- sb.append(column);
- }
- throw new AmbiguousFieldException("Ambiguous Column Name: " + sb.toString());
- } else {
- return null;
- }
+ return NameResolver.resolve(this, block, columnRef, NameResolvingMode.LEGACY);
}
public String getQueryGraphAsString() {
@@ -601,7 +374,8 @@ public class LogicalPlan {
// transient states
private final Map<String, RelationNode> canonicalNameToRelationMap = TUtil.newHashMap();
- private final Map<String, List<String>> aliasMap = TUtil.newHashMap();
+ private final Map<String, List<String>> relationAliasMap = TUtil.newHashMap();
+ private final Map<String, String> columnAliasMap = TUtil.newHashMap();
private final Map<OpType, List<Expr>> operatorToExprMap = TUtil.newHashMap();
private final List<RelationNode> relationList = TUtil.newList();
private boolean hasWindowFunction = false;
@@ -669,7 +443,7 @@ public class LogicalPlan {
}
public boolean isAlreadyRenamedTableName(String name) {
- return aliasMap.containsKey(name);
+ return relationAliasMap.containsKey(name);
}
public RelationNode getRelation(String name) {
@@ -677,8 +451,8 @@ public class LogicalPlan {
return canonicalNameToRelationMap.get(name);
}
- if (aliasMap.containsKey(name)) {
- return canonicalNameToRelationMap.get(aliasMap.get(name).get(0));
+ if (relationAliasMap.containsKey(name)) {
+ return canonicalNameToRelationMap.get(relationAliasMap.get(name).get(0));
}
return null;
@@ -686,7 +460,7 @@ public class LogicalPlan {
public void addRelation(RelationNode relation) {
if (relation.hasAlias()) {
- TUtil.putToNestedList(aliasMap, relation.getTableName(), relation.getCanonicalName());
+ TUtil.putToNestedList(relationAliasMap, relation.getTableName(), relation.getCanonicalName());
}
canonicalNameToRelationMap.put(relation.getCanonicalName(), relation);
relationList.add(relation);
@@ -700,6 +474,18 @@ public class LogicalPlan {
return this.canonicalNameToRelationMap.size() > 0;
}
+ public void addColumnAlias(String original, String alias) {
+ columnAliasMap.put(alias, original);
+ }
+
+ public boolean isAliasedName(String alias) {
+ return columnAliasMap.containsKey(alias);
+ }
+
+ public String getOriginalName(String alias) {
+ return columnAliasMap.get(alias);
+ }
+
public void setSchema(Schema schema) {
this.schema = schema;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
index 4f1218f..84fe6c2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
@@ -26,6 +26,8 @@ import org.apache.tajo.engine.eval.FieldEval;
import org.apache.tajo.engine.exception.NoSuchColumnException;
import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock;
import org.apache.tajo.engine.planner.logical.*;
+import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
+import org.apache.tajo.engine.planner.nameresolver.NameResolver;
import org.apache.tajo.engine.utils.SchemaUtil;
import org.apache.tajo.master.session.Session;
import org.apache.tajo.util.TUtil;
@@ -196,7 +198,19 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
expr.setNamedExprs(rewrittenTargets.toArray(new NamedExpr[rewrittenTargets.size()]));
}
+ // 1) Normalize field names into full qualified names
+ // 2) Register explicit column aliases to block
NamedExpr[] projectTargetExprs = expr.getNamedExprs();
+ NameRefInSelectListNormalizer normalizer = new NameRefInSelectListNormalizer();
+ for (int i = 0; i < expr.getNamedExprs().length; i++) {
+ NamedExpr namedExpr = projectTargetExprs[i];
+ normalizer.visit(ctx, new Stack<Expr>(), namedExpr.getExpr());
+
+ if (namedExpr.getExpr().getType() == OpType.Column && namedExpr.hasAlias()) {
+ ctx.currentBlock.addColumnAlias(((ColumnReferenceExpr)namedExpr.getExpr()).getCanonicalName(),
+ namedExpr.getAlias());
+ }
+ }
Target [] targets;
targets = new Target[projectTargetExprs.length];
@@ -217,6 +231,8 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
ProjectionNode projectionNode = ctx.plan.createNode(ProjectionNode.class);
projectionNode.setInSchema(child.getOutSchema());
projectionNode.setOutSchema(PlannerUtil.targetToSchema(targets));
+
+ ctx.currentBlock.setSchema(projectionNode.getOutSchema());
return projectionNode;
}
@@ -267,7 +283,8 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
for (int i = 0; i < finalTargetNum; i++) {
NamedExpr namedExpr = projection.getNamedExprs()[i];
- EvalNode evalNode = annotator.createEvalNode(ctx.plan, ctx.currentBlock, namedExpr.getExpr());
+ EvalNode evalNode = annotator.createEvalNode(ctx.plan, ctx.currentBlock, namedExpr.getExpr(),
+ NameResolvingMode.SUBEXPRS_AND_RELS);
if (namedExpr.hasAlias()) {
targets[i] = new Target(evalNode, namedExpr.getAlias());
@@ -449,4 +466,17 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
insertNode.setOutSchema(child.getOutSchema());
return insertNode;
}
+
+ class NameRefInSelectListNormalizer extends SimpleAlgebraVisitor<PreprocessContext, Object> {
+ @Override
+ public Expr visitColumnReference(PreprocessContext ctx, Stack<Expr> stack, ColumnReferenceExpr expr)
+ throws PlanningException {
+
+ String normalized = NameResolver.resolve(ctx.plan, ctx.currentBlock, expr,
+ NameResolvingMode.RELS_ONLY).getQualifiedName();
+ expr.setName(normalized);
+
+ return expr;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index ea517c0..a4820cb 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -40,6 +40,7 @@ import org.apache.tajo.engine.eval.*;
import org.apache.tajo.engine.exception.VerifyException;
import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock;
import org.apache.tajo.engine.planner.logical.*;
+import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule;
import org.apache.tajo.engine.utils.SchemaUtil;
import org.apache.tajo.master.session.Session;
@@ -51,6 +52,7 @@ import org.apache.tajo.util.TUtil;
import java.util.*;
import static org.apache.tajo.algebra.CreateTable.PartitionType;
+
import static org.apache.tajo.engine.planner.ExprNormalizer.ExprNormalizedResult;
import static org.apache.tajo.engine.planner.LogicalPlan.BlockType;
import static org.apache.tajo.engine.planner.LogicalPlanPreprocessor.PreprocessContext;
@@ -260,7 +262,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Target [] rawTargets = new Target[projection.getNamedExprs().length];
for (int i = 0; i < projection.getNamedExprs().length; i++) {
NamedExpr namedExpr = projection.getNamedExprs()[i];
- EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr());
+ EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(),
+ NameResolvingMode.RELS_AND_SUBEXPRS);
rawTargets[i] = new Target(evalNode, referenceNames[i]);
}
// it's for debugging or unit testing
@@ -383,7 +386,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
for (int i = 0; i < targets.length; i++) {
NamedExpr namedExpr = projection.getNamedExprs()[i];
- EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr());
+ EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolvingMode.RELS_ONLY);
if (namedExpr.hasAlias()) {
targets[i] = new Target(evalNode, namedExpr.getAlias());
} else {
@@ -407,7 +410,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
targets[i] = block.namedExprsMgr.getTarget(referenceNames[i]);
} else {
NamedExpr namedExpr = block.namedExprsMgr.getNamedExpr(referenceNames[i]);
- EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr());
+ EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(),
+ NameResolvingMode.RELS_AND_SUBEXPRS);
block.namedExprsMgr.markAsEvaluated(referenceNames[i], evalNode);
targets[i] = new Target(evalNode, referenceNames[i]);
}
@@ -553,7 +557,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
for (Iterator<NamedExpr> it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) {
NamedExpr rawTarget = it.next();
try {
- EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr());
+ EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(),
+ NameResolvingMode.SUBEXPRS_AND_RELS);
if (evalNode.getType() == EvalType.WINDOW_FUNCTION) {
winFuncRefs.add(rawTarget.getAlias());
winFuncs.add((WindowFunctionEval) evalNode);
@@ -678,7 +683,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
try {
// check if at least distinct aggregation function
includeDistinctFunction |= PlannerUtil.existsDistinctAggregationFunction(rawTarget.getExpr());
- EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr());
+ EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(),
+ NameResolvingMode.SUBEXPRS_AND_RELS);
if (evalNode.getType() == EvalType.AGG_FUNCTION) {
aggEvalNames.add(rawTarget.getAlias());
aggEvals.add((AggregationFunctionCallEval) evalNode);
@@ -712,7 +718,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
EvalNode firstFetNum;
LogicalNode child;
if (limit.getFetchFirstNum().getType() == OpType.Literal) {
- firstFetNum = exprAnnotator.createEvalNode(context.plan, block, limit.getFetchFirstNum());
+ firstFetNum = exprAnnotator.createEvalNode(context.plan, block, limit.getFetchFirstNum(),
+ NameResolvingMode.RELS_ONLY);
////////////////////////////////////////////////////////
// Visit and Build Child Plan
@@ -739,7 +746,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
firstFetNum = block.namedExprsMgr.getTarget(referName).getEvalTree();
} else {
NamedExpr namedExpr = block.namedExprsMgr.getNamedExpr(referName);
- firstFetNum = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr());
+ firstFetNum = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr(),
+ NameResolvingMode.SUBEXPRS_AND_RELS);
block.namedExprsMgr.markAsEvaluated(referName, firstFetNum);
}
}
@@ -835,7 +843,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
havingCondition = block.namedExprsMgr.getTarget(referName).getEvalTree();
} else {
NamedExpr namedExpr = block.namedExprsMgr.getNamedExpr(referName);
- havingCondition = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr());
+ havingCondition = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr(),
+ NameResolvingMode.SUBEXPRS_AND_RELS);
block.namedExprsMgr.markAsEvaluated(referName, havingCondition);
}
@@ -902,7 +911,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
NamedExpr namedExpr = iterator.next();
try {
includeDistinctFunction |= PlannerUtil.existsDistinctAggregationFunction(namedExpr.getExpr());
- EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, namedExpr.getExpr());
+ EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, namedExpr.getExpr(),
+ NameResolvingMode.SUBEXPRS_AND_RELS);
if (evalNode.getType() == EvalType.AGG_FUNCTION) {
block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode);
aggEvalNames.add(namedExpr.getAlias());
@@ -988,7 +998,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
selectionNode.setOutSchema(child.getOutSchema());
// Create EvalNode for a search condition.
- EvalNode searchCondition = exprAnnotator.createEvalNode(context.plan, block, selection.getQual());
+ EvalNode searchCondition = exprAnnotator.createEvalNode(context.plan, block, selection.getQual(),
+ NameResolvingMode.RELS_AND_SUBEXPRS);
EvalNode simplified = AlgebraicUtil.eliminateConstantExprs(searchCondition);
// set selection condition
selectionNode.setQual(simplified);
@@ -1041,7 +1052,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
// Create EvalNode for a search condition.
EvalNode joinCondition = null;
if (join.hasQual()) {
- EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, join.getQual());
+ EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, join.getQual(),
+ NameResolvingMode.LEGACY);
joinCondition = AlgebraicUtil.eliminateConstantExprs(evalNode);
}
@@ -1071,7 +1083,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
for (Iterator<NamedExpr> it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) {
NamedExpr namedExpr = it.next();
try {
- evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr());
+ evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolvingMode.LEGACY);
if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, evalNode, joinNode, stack.peek().getType() != OpType.Join)) {
block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode);
newlyEvaluatedExprs.add(namedExpr.getAlias());
@@ -1141,7 +1153,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
for (Iterator<NamedExpr> it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) {
NamedExpr namedExpr = it.next();
try {
- evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr());
+ evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolvingMode.LEGACY);
if (EvalTreeUtil.findDistinctAggFunction(evalNode).size() == 0) {
block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode);
newlyEvaluatedExprs.add(namedExpr.getAlias());
@@ -1193,7 +1205,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
for (Iterator<NamedExpr> iterator = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); iterator.hasNext();) {
NamedExpr rawTarget = iterator.next();
try {
- EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr());
+ EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(),
+ NameResolvingMode.RELS_ONLY);
if (checkIfBeEvaluatedAtRelation(block, evalNode, scanNode)) {
block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode);
newlyEvaluatedExprsReferences.add(rawTarget.getAlias()); // newly added exr
@@ -1208,8 +1221,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
// The fact the some expr is included in newlyEvaluatedExprsReferences means that it is already evaluated.
// So, we get a raw expression and then creates a target.
for (String reference : newlyEvaluatedExprsReferences) {
- NamedExpr refrrer = block.namedExprsMgr.getNamedExpr(reference);
- EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, refrrer.getExpr());
+ NamedExpr refrer = block.namedExprsMgr.getNamedExpr(reference);
+ EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, refrer.getExpr(),
+ NameResolvingMode.RELS_ONLY);
targets.add(new Target(evalNode, reference));
}
@@ -1267,7 +1281,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Set<String> newlyEvaluatedExprs = TUtil.newHashSet();
for (NamedExpr rawTarget : block.namedExprsMgr.getAllNamedExprs()) {
try {
- EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr());
+ EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(),
+ NameResolvingMode.RELS_ONLY);
if (checkIfBeEvaluatedAtRelation(block, evalNode, subQueryNode)) {
block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode);
newlyEvaluatedExprs.add(rawTarget.getAlias()); // newly added exr
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
index 4d0090b..3c808fc 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java
@@ -86,7 +86,6 @@ public class TableSubQueryNode extends RelationNode implements Projectable {
} else {
setOutSchema(SchemaUtil.clone(this.subQuery.getOutSchema()));
}
- getOutSchema().setQualifier(this.tableName);
}
public LogicalNode getSubQuery() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java
new file mode 100644
index 0000000..aee5d43
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.planner.nameresolver;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.tajo.algebra.ColumnReferenceExpr;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.engine.exception.AmbiguousFieldException;
+import org.apache.tajo.engine.exception.NoSuchColumnException;
+import org.apache.tajo.engine.exception.VerifyException;
+import org.apache.tajo.engine.planner.LogicalPlan;
+import org.apache.tajo.engine.planner.PlanningException;
+import org.apache.tajo.engine.planner.logical.RelationNode;
+import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.TUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * NameResolver utility
+ */
+public abstract class NameResolver {
+
+ public static Map<NameResolvingMode, NameResolver> resolverMap = Maps.newHashMap();
+
+ static {
+ resolverMap.put(NameResolvingMode.RELS_ONLY, new ResolverByRels());
+ resolverMap.put(NameResolvingMode.RELS_AND_SUBEXPRS, new ResolverByRelsAndSubExprs());
+ resolverMap.put(NameResolvingMode.SUBEXPRS_AND_RELS, new ResolverBySubExprsAndRels());
+ resolverMap.put(NameResolvingMode.LEGACY, new ResolverByLegacy());
+ }
+
+ abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ throws PlanningException;
+
+ /**
+ * Try to find the database name
+ *
+ * @param block the current block
+ * @param tableName The table name
+ * @return The found database name
+ * @throws PlanningException
+ */
+ public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableName) throws PlanningException {
+ List<String> found = new ArrayList<String>();
+ for (RelationNode relation : block.getRelations()) {
+ // check alias name or table name
+ if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) ||
+ CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) {
+ // obtain the database name
+ found.add(CatalogUtil.extractQualifier(relation.getTableName()));
+ }
+ }
+
+ if (found.size() == 0) {
+ return null;
+ } else if (found.size() > 1) {
+ throw new PlanningException("Ambiguous table name \"" + tableName + "\"");
+ }
+
+ return found.get(0);
+ }
+
+ public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
+ NameResolvingMode mode) throws PlanningException {
+ if (!resolverMap.containsKey(mode)) {
+ throw new PlanningException("Unsupported name resolving level: " + mode.name());
+ }
+ return resolverMap.get(mode).resolve(plan, block, column);
+ }
+
+ /**
+ * Try to find a column from all relations within a given query block.
+ * If a given column reference is qualified, it tries to resolve the name
+ * from only the relation corresponding to the qualifier.
+ *
+ * @param plan The logical plan
+ * @param block The current query block
+ * @param columnRef The column reference to be found
+ * @return The found column
+ * @throws PlanningException
+ */
+ static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef) throws PlanningException {
+ String qualifier;
+ String canonicalName;
+
+ if (columnRef.hasQualifier()) {
+ Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef);
+ qualifier = normalized.getFirst();
+ canonicalName = normalized.getSecond();
+
+ RelationNode relationOp = block.getRelation(qualifier);
+
+ // If we cannot find any relation against a qualified column name
+ if (relationOp == null) {
+ throw null;
+ }
+
+ // Please consider a query case:
+ // select lineitem.l_orderkey from lineitem a order by lineitem.l_orderkey;
+ //
+ // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still can be used.
+ // The below code makes it available. Otherwise, it cannot find any match in the relation schema.
+ if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) {
+ canonicalName =
+ CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName));
+ }
+
+ Schema schema = relationOp.getTableSchema();
+ Column column = schema.getColumn(canonicalName);
+
+ return column;
+ } else {
+ return resolveFromAllRelsInBlock(block, columnRef);
+ }
+ }
+
+ /**
+ * Try to find the column from the current node and child node. It can find subexprs generated from the optimizer.
+ *
+ * @param block The current query block
+ * @param columnRef The column reference to be found
+ * @return The found column
+ */
+ static Column resolveFromCurrentAndChildNode(LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ throws NoSuchColumnException {
+
+ if (block.getCurrentNode() != null && block.getCurrentNode().getInSchema() != null) {
+ Column found = block.getCurrentNode().getInSchema().getColumn(columnRef.getCanonicalName());
+ if (found != null) {
+ return found;
+ } else if (block.getLatestNode() != null) {
+ found = block.getLatestNode().getOutSchema().getColumn(columnRef.getName());
+ if (found != null) {
+ return found;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * It tries to find a full qualified column name from all relations in the current block.
+ *
+ * @param block The current query block
+ * @param columnRef The column reference to be found
+ * @return The found column
+ */
+ static Column resolveFromAllRelsInBlock(LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef) throws VerifyException {
+ List<Column> candidates = TUtil.newList();
+
+ for (RelationNode rel : block.getRelations()) {
+ Column found = rel.getTableSchema().getColumn(columnRef.getName());
+ if (found != null) {
+ candidates.add(found);
+ }
+ }
+
+ if (!candidates.isEmpty()) {
+ return ensureUniqueColumn(candidates);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Trying to find a column from all relations in other blocks
+ *
+ * @param plan The logical plan
+ * @param columnRef The column reference to be found
+ * @return The found column
+ */
+ static Column resolveFromAllRelsInAllBlocks(LogicalPlan plan, ColumnReferenceExpr columnRef) throws VerifyException {
+
+ List<Column> candidates = Lists.newArrayList();
+
+ // from all relations of all query blocks
+ for (LogicalPlan.QueryBlock eachBlock : plan.getQueryBlocks()) {
+
+ for (RelationNode rel : eachBlock.getRelations()) {
+ Column found = rel.getTableSchema().getColumn(columnRef.getName());
+ if (found != null) {
+ candidates.add(found);
+ }
+ }
+ }
+
+ if (!candidates.isEmpty()) {
+ return NameResolver.ensureUniqueColumn(candidates);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Try to find a column from the final schema of the current block.
+ *
+ * @param block The current query block
+ * @param columnRef The column reference to be found
+ * @return The found column
+ */
+ static Column resolveAliasedName(LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) throws VerifyException {
+ List<Column> candidates = Lists.newArrayList();
+
+ if (block.getSchema() != null) {
+ Column found = block.getSchema().getColumn(columnRef.getName());
+ if (found != null) {
+ candidates.add(found);
+ }
+ }
+
+ if (!candidates.isEmpty()) {
+ return NameResolver.ensureUniqueColumn(candidates);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * It returns a pair of names, which the first value is ${database}.${table} and the second value
+ * is a simple column name.
+ *
+ * @param block The current block
+ * @param columnRef The column name
+ * @return A pair of normalized qualifier and column name
+ * @throws PlanningException
+ */
+ static Pair<String, String> normalizeQualifierAndCanonicalName(LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef)
+ throws PlanningException {
+ String qualifier;
+ String canonicalName;
+
+ if (CatalogUtil.isFQTableName(columnRef.getQualifier())) {
+ qualifier = columnRef.getQualifier();
+ canonicalName = columnRef.getCanonicalName();
+ } else {
+ String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier());
+ if (resolvedDatabaseName == null) {
+ throw new NoSuchColumnException(columnRef.getQualifier());
+ }
+ qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier());
+ canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
+ }
+
+ return new Pair<String, String>(qualifier, canonicalName);
+ }
+
+ static Column ensureUniqueColumn(List<Column> candidates) throws VerifyException {
+ if (candidates.size() == 1) {
+ return candidates.get(0);
+ } else if (candidates.size() > 2) {
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for (Column column : candidates) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(", ");
+ }
+ sb.append(column);
+ }
+ throw new AmbiguousFieldException("Ambiguous Column Name: " + sb.toString());
+ } else {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java
new file mode 100644
index 0000000..dcbb5f1
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.planner.nameresolver;
+
+/**
+ *
+ * <h2>Motivation</h2>
+ *
+ * Please take a look at the following example query:
+ *
+ * <pre>
+ * select (l_orderkey + l_orderkey) l_orderkey from lineitem where l_orderkey > 2 order by l_orderkey;
+ * </pre>
+ *
+ * Although <code>l_orderkey</code> seems to be ambiguous, the above usages are available in commercial DBMSs.
+ * In order to eliminate the ambiguity, Tajo follows the behaviors of PostgreSQL.
+ *
+ * <h2>Resolving Modes</h2>
+ *
+ * From the behaviors of PostgreSQL, we found that there are three kinds of name resolving modes.
+ * Each definition is as follows:
+ *
+ * <ul>
+ * <li><b>RELS_ONLY</b> finds a column from the relations in the current block.
+ * <li><b>RELS_AND_SUBEXPRS</b> finds a column from the all relations in the current block and
+ * from aliased temporal fields; a temporal field means an explicitly aliased expression. If there are duplicated
+ * columns in the relation and temporal fields, this level firstly chooses the field in a relation.</li>
+ * <li><b>SUBEXPRS_AND_RELS</b> is very similar to <code>RELS_AND_SUBEXPRS</code>. The main difference is that it
+ * firstly chooses an aliased temporal field instead of the fields in a relation.</li>
+ * </ul>
+ *
+ * <h2>The relationship between resolving modes and operators</h3>
+ *
+ * <ul>
+ * <li>fields in select list and LIMIT are resolved in the REL_ONLY mode.</li>
+ * <li>fields in WHERE clause are resolved in the RELS_AND_SUBEXPRS mode.</li>
+ * <li>fields in GROUP BY, HAVING, and ORDER BY are resolved in the SUBEXPRS_AND_RELS mode.</li>
+ * </ul>
+ *
+ * <h2>Example</h2>
+ *
+ * Please revisit the aforementioned example:
+ *
+ * <pre>
+ * select (l_orderkey + l_orderkey) l_orderkey from lineitem where l_orderkey > 2 order by l_orderkey;
+ * </pre>
+ *
+ * With the above rules and the relationship between modes and operators, we can easily identify which reference
+ * points to which field.
+ * <ol>
+ * <li><code>l_orderkey</code> included in <code>(l_orderkey + l_orderkey)</code> points to the field
+ * in the relation <code>lineitem</code>.</li>
+ * <li><code>l_orderkey</code> included in WHERE clause also points to the field in the relation
+ * <code>lineitem</code>.</li>
+ * <li><code>l_orderkey</code> included in ORDER BY clause points to the temporal field
+ * <code>(l_orderkey + l_orderkey)</code>.</li>
+ * </ol>
+ */
+public enum NameResolvingMode {
+ RELS_ONLY, // finding from only relations
+ RELS_AND_SUBEXPRS, // finding from relations and subexprs in a place
+ SUBEXPRS_AND_RELS, // finding from subexprs and relations in a place
+ LEGACY // Finding in a legacy manner (globally)
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java
new file mode 100644
index 0000000..396bc1b
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.planner.nameresolver;
+
+import org.apache.tajo.algebra.ColumnReferenceExpr;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.engine.exception.NoSuchColumnException;
+import org.apache.tajo.engine.planner.LogicalPlan;
+import org.apache.tajo.engine.planner.PlanningException;
+import org.apache.tajo.engine.planner.logical.LogicalNode;
+import org.apache.tajo.engine.planner.logical.NodeType;
+import org.apache.tajo.engine.planner.logical.RelationNode;
+import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.TUtil;
+
+import java.util.List;
+
+public class ResolverByLegacy extends NameResolver {
+ @Override
+ public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ throws PlanningException {
+
+ if (columnRef.hasQualifier()) {
+ return resolveColumnWithQualifier(plan, block, columnRef);
+ } else {
+ return resolveColumnWithoutQualifier(plan, block, columnRef);
+ }
+ }
+
+ private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef) throws PlanningException {
+ final String qualifier;
+ String canonicalName;
+ final String qualifiedName;
+
+ Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef);
+ qualifier = normalized.getFirst();
+ canonicalName = normalized.getSecond();
+ qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
+
+ Column found = resolveFromRelsWithinBlock(plan, block, columnRef);
+ if (found == null) {
+ throw new NoSuchColumnException(columnRef.getCanonicalName());
+ }
+
+ // If code reach here, a column is found.
+ // But, it may be aliased from bottom logical node.
+ // If the column is aliased, the found name may not be used in upper node.
+
+ // Here, we try to check if column reference is already aliased.
+ // If so, it replaces the name with aliased name.
+ LogicalNode currentNode = block.getCurrentNode();
+
+ // The condition (currentNode.getInSchema().contains(column)) means
+ // the column can be used at the current node. So, we don't need to find aliase name.
+ Schema currentNodeSchema = null;
+ if (currentNode != null) {
+ if (currentNode instanceof RelationNode) {
+ currentNodeSchema = ((RelationNode) currentNode).getTableSchema();
+ } else {
+ currentNodeSchema = currentNode.getInSchema();
+ }
+ }
+
+ if (currentNode != null && !currentNodeSchema.contains(found)
+ && currentNode.getType() != NodeType.TABLE_SUBQUERY) {
+ List<Column> candidates = TUtil.newList();
+ if (block.getNamedExprsManager().isAliased(qualifiedName)) {
+ String alias = block.getNamedExprsManager().getAlias(canonicalName);
+ found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolvingMode.LEGACY);
+ if (found != null) {
+ candidates.add(found);
+ }
+ }
+ if (!candidates.isEmpty()) {
+ return ensureUniqueColumn(candidates);
+ }
+ }
+
+ return found;
+ }
+
+ static Column resolveColumnWithoutQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef)throws PlanningException {
+
+ Column found = resolveFromAllRelsInBlock(block, columnRef);
+ if (found != null) {
+ return found;
+ }
+
+ found = resolveAliasedName(block, columnRef);
+ if (found != null) {
+ return found;
+ }
+
+ found = resolveFromCurrentAndChildNode(block, columnRef);
+ if (found != null) {
+ return found;
+ }
+
+ found = resolveFromAllRelsInAllBlocks(plan, columnRef);
+ if (found != null) {
+ return found;
+ }
+
+ throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java
new file mode 100644
index 0000000..9713e52
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.planner.nameresolver;
+
+import org.apache.tajo.algebra.ColumnReferenceExpr;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.engine.exception.NoSuchColumnException;
+import org.apache.tajo.engine.planner.LogicalPlan;
+import org.apache.tajo.engine.planner.PlanningException;
+
+public class ResolverByRels extends NameResolver {
+ @Override
+ public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ throws PlanningException {
+
+ Column column = resolveFromRelsWithinBlock(plan, block, columnRef);
+ if (column == null) {
+ throw new NoSuchColumnException(columnRef.getCanonicalName());
+ }
+ return column;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java
new file mode 100644
index 0000000..7ca3c53
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.planner.nameresolver;
+
+import org.apache.tajo.algebra.ColumnReferenceExpr;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.engine.exception.NoSuchColumnException;
+import org.apache.tajo.engine.planner.LogicalPlan;
+import org.apache.tajo.engine.planner.PlanningException;
+
+public class ResolverByRelsAndSubExprs extends NameResolver {
+ @Override
+ public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ throws PlanningException {
+
+ Column column = resolveFromRelsWithinBlock(plan, block, columnRef);
+ if (column == null) {
+ column = resolveFromCurrentAndChildNode(block, columnRef);
+ }
+
+ if (column == null) {
+ throw new NoSuchColumnException(columnRef.getCanonicalName());
+ }
+ return column;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java
new file mode 100644
index 0000000..7337ece
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.planner.nameresolver;
+
+import org.apache.tajo.algebra.ColumnReferenceExpr;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.engine.exception.NoSuchColumnException;
+import org.apache.tajo.engine.planner.LogicalPlan;
+import org.apache.tajo.engine.planner.PlanningException;
+
+public class ResolverBySubExprsAndRels extends NameResolver {
+ @Override
+ public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ throws PlanningException {
+
+ Column column = resolveFromCurrentAndChildNode(block, columnRef);
+ if (column == null) {
+ column = resolveFromRelsWithinBlock(plan, block, columnRef);
+ }
+
+ if (column == null) {
+ throw new NoSuchColumnException(columnRef.getCanonicalName());
+ }
+ return column;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
index 92770ec..7bb619d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
@@ -39,6 +39,7 @@ import org.apache.tajo.engine.planner.PlanningException;
import org.apache.tajo.engine.planner.Target;
import org.apache.tajo.engine.planner.logical.GroupbyNode;
import org.apache.tajo.engine.planner.logical.NodeType;
+import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode;
import org.apache.tajo.exception.InternalException;
import org.apache.tajo.master.TajoMaster;
import org.apache.tajo.master.session.Session;
@@ -154,7 +155,8 @@ public class TestEvalTreeUtil {
}
Selection selection = plan.getRootBlock().getSingletonExpr(OpType.Filter);
- return planner.getExprAnnotator().createEvalNode(plan, plan.getRootBlock(), selection.getQual());
+ return planner.getExprAnnotator().createEvalNode(plan, plan.getRootBlock(), selection.getQual(),
+ NameResolvingMode.RELS_AND_SUBEXPRS);
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
index 73df4e1..459a4c1 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
@@ -133,4 +133,11 @@ public class TestCaseByCases extends QueryTestCaseBase {
assertEquals(expected, resultSetToString(res));
cleanupQuery(res);
}
+
+ @Test
+ public final void testTAJO917Case1() throws Exception {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index 639c3ef..8898067 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -49,6 +49,14 @@ public class TestSelectQuery extends QueryTestCaseBase {
}
@Test
+ public final void testNonQualifiedNames() throws Exception {
+ // select l_orderkey, l_partkey from lineitem;
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
public final void testNonFromSelect1() throws Exception {
// select upper('abc');
ResultSet res = executeQuery();
@@ -113,6 +121,25 @@ public class TestSelectQuery extends QueryTestCaseBase {
}
@Test
+ public final void testSelectColumnAliasExistingInRelation1() throws Exception {
+ // We intend that 'l_orderkey' in where clause points to "default.lineitem.l_orderkey"
+ // select (l_orderkey + l_orderkey) l_orderkey from lineitem where l_orderkey > 2;
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testSelectColumnAliasExistingInRelation2() throws Exception {
+ // We intend that 'l_orderkey' in orderby clause points to (-l_orderkey).
+ // select (-l_orderkey) as l_orderkey from lineitem order by l_orderkey;
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+
+ @Test
public final void testSelectSameConstantsWithDifferentAliases() throws Exception {
// select l_orderkey, '20130819' as date1, '20130819' as date2 from lineitem where l_orderkey > -1;
ResultSet res = executeQuery();
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO917Case1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO917Case1.sql b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO917Case1.sql
new file mode 100644
index 0000000..5b3039c
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO917Case1.sql
@@ -0,0 +1,13 @@
+select
+ temp.r_regionkey as r_regionkey
+from
+ (
+ select
+ region.r_regionkey as r_regionkey
+ from
+ region
+ ) temp
+join
+ region b
+on
+ temp.r_regionkey = b.r_regionkey;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/resources/queries/TestSelectQuery/testNonQualifiedNames.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testNonQualifiedNames.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testNonQualifiedNames.sql
new file mode 100644
index 0000000..0c176b7
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testNonQualifiedNames.sql
@@ -0,0 +1 @@
+select l_orderkey, l_partkey from lineitem;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation1.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation1.sql
new file mode 100644
index 0000000..91170e3
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation1.sql
@@ -0,0 +1 @@
+select (l_orderkey + l_orderkey) l_orderkey from lineitem where l_orderkey > 2;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation2.sql
new file mode 100644
index 0000000..89f63fd
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation2.sql
@@ -0,0 +1 @@
+select (-l_orderkey) as l_orderkey from lineitem order by l_orderkey;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO917Case1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO917Case1.result b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO917Case1.result
new file mode 100644
index 0000000..5dbe646
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO917Case1.result
@@ -0,0 +1,7 @@
+r_regionkey
+-------------------------------
+0
+1
+2
+3
+4
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/resources/results/TestSelectQuery/testNonQualifiedNames.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testNonQualifiedNames.result b/tajo-core/src/test/resources/results/TestSelectQuery/testNonQualifiedNames.result
new file mode 100644
index 0000000..1378536
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectQuery/testNonQualifiedNames.result
@@ -0,0 +1,7 @@
+l_orderkey,l_partkey
+-------------------------------
+1,1
+1,1
+2,2
+3,2
+3,3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation1.result b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation1.result
new file mode 100644
index 0000000..55e2b42
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation1.result
@@ -0,0 +1,4 @@
+l_orderkey
+-------------------------------
+6
+6
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/862435bc/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation2.result b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation2.result
new file mode 100644
index 0000000..f0cf700
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation2.result
@@ -0,0 +1,7 @@
+l_orderkey
+-------------------------------
+-3
+-3
+-2
+-1
+-1
\ No newline at end of file
[7/7] git commit: TAJO-837: Register index meta information at
Catalog. (jihoon)
Posted by ji...@apache.org.
TAJO-837: Register index meta information at Catalog. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/c9b8f511
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/c9b8f511
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/c9b8f511
Branch: refs/heads/index_support
Commit: c9b8f511e646f1315469f305d9da08216a93152a
Parents: 7cd5706
Author: Jihoon Son <ji...@apache.org>
Authored: Wed Jul 23 09:47:32 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Jul 23 09:47:32 2014 +0900
----------------------------------------------------------------------
CHANGES | 4 +
.../org/apache/tajo/algebra/CreateTable.java | 2 +-
.../java/org/apache/tajo/algebra/DropIndex.java | 69 ++++++++
.../java/org/apache/tajo/algebra/OpType.java | 1 +
.../tajo/catalog/AbstractCatalogClient.java | 42 -----
.../org/apache/tajo/catalog/CatalogService.java | 5 -
.../src/main/proto/CatalogProtocol.proto | 2 -
.../java/org/apache/tajo/catalog/Column.java | 1 -
.../java/org/apache/tajo/catalog/IndexDesc.java | 146 +++++++++++++---
.../src/main/proto/CatalogProtos.proto | 17 +-
.../org/apache/tajo/catalog/TestIndexDesc.java | 59 ++++---
.../org/apache/tajo/catalog/CatalogServer.java | 44 -----
.../tajo/catalog/store/AbstractDBStore.java | 163 ++++++------------
.../apache/tajo/catalog/store/CatalogStore.java | 11 +-
.../apache/tajo/catalog/store/DerbyStore.java | 6 -
.../org/apache/tajo/catalog/store/MemStore.java | 32 +---
.../main/resources/schemas/derby/indexes.sql | 11 +-
.../main/resources/schemas/mariadb/indexes.sql | 13 +-
.../main/resources/schemas/mysql/indexes.sql | 13 +-
.../org/apache/tajo/catalog/TestCatalog.java | 31 ++--
.../java/org/apache/tajo/client/TajoClient.java | 49 +++++-
tajo-client/src/main/proto/ClientProtos.proto | 4 +
.../main/proto/TajoMasterClientProtocol.proto | 4 +
.../org/apache/tajo/engine/parser/SQLParser.g4 | 11 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 7 +-
.../tajo/engine/planner/AlgebraVisitor.java | 1 +
.../tajo/engine/planner/BaseAlgebraVisitor.java | 8 +
.../engine/planner/BasicLogicalPlanVisitor.java | 9 +
.../engine/planner/LogicalPlanPreprocessor.java | 7 +
.../tajo/engine/planner/LogicalPlanVisitor.java | 3 +
.../tajo/engine/planner/LogicalPlanner.java | 10 +-
.../apache/tajo/engine/planner/PlannerUtil.java | 16 +-
.../engine/planner/logical/CreateIndexNode.java | 18 ++
.../engine/planner/logical/DropIndexNode.java | 89 ++++++++++
.../tajo/engine/planner/logical/NodeType.java | 1 +
.../engine/planner/physical/StoreIndexExec.java | 15 +-
.../org/apache/tajo/master/GlobalEngine.java | 169 ++++++++++++++++---
.../tajo/master/TajoMasterClientService.java | 42 ++++-
.../apache/tajo/worker/TaskAttemptContext.java | 8 -
.../java/org/apache/tajo/QueryTestCaseBase.java | 23 ++-
.../tajo/engine/planner/TestLogicalPlanner.java | 1 -
.../planner/physical/TestPhysicalPlanner.java | 3 +-
.../tajo/engine/query/TestCreateIndex.java | 32 +++-
43 files changed, 819 insertions(+), 383 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8a1aae6..69bc8cf 100644
--- a/CHANGES
+++ b/CHANGES
@@ -4,6 +4,10 @@ Release 0.9.0 - unreleased
NEW FEATURES
+ TAJO-837: Register index meta information at Catalog. (jihoon)
+
+ TAJO-836: create index support. (jihoon)
+
TAJO-20: INSERT INTO ... SELECT. (Hyoungjun Kim via hyunsik)
TAJO-774: Implement logical plan part and physical executor for window
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
index 3121711..ad1a6f4 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
@@ -24,9 +24,9 @@ import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
import org.apache.tajo.util.TUtil;
+import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.HashMap;
-import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java
new file mode 100644
index 0000000..f72a12a
--- /dev/null
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.algebra;
+
+import com.google.gson.annotations.Expose;
+import com.google.gson.annotations.SerializedName;
+import org.apache.tajo.util.TUtil;
+
+import java.util.Objects;
+
+public class DropIndex extends Expr {
+
+ @Expose @SerializedName("IndexName")
+ private String indexName;
+ @Expose @SerializedName("IfExists")
+ private boolean ifExists;
+
+ public DropIndex(String indexName, boolean ifExists) {
+ super(OpType.DropIndex);
+ this.indexName = indexName;
+ this.ifExists = ifExists;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(indexName, ifExists);
+ }
+
+ @Override
+ boolean equalsTo(Expr expr) {
+ if (expr instanceof DropIndex) {
+ DropIndex another = (DropIndex) expr;
+ return TUtil.checkEquals(indexName, another.indexName) &&
+ ifExists == another.ifExists;
+ }
+ return false;
+ }
+
+ public Object clone() throws CloneNotSupportedException {
+ DropIndex clone = (DropIndex) super.clone();
+ clone.indexName = indexName;
+ clone.ifExists = ifExists;
+ return clone;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ public boolean isIfExists() {
+ return ifExists;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
index 2ef22f7..11c5ad2 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
@@ -51,6 +51,7 @@ public enum OpType {
AlterTablespace(AlterTablespace.class),
AlterTable(AlterTable.class),
CreateIndex(CreateIndex.class),
+ DropIndex(DropIndex.class),
TruncateTable(TruncateTable.class),
// Insert or Update
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index 542214b..49151cd 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@ -431,26 +431,6 @@ public abstract class AbstractCatalogClient implements CatalogService {
}
@Override
- public boolean existIndexByColumn(final String databaseName, final String tableName, final String columnName) {
- try {
- return new ServerCallable<Boolean>(this.pool, catalogServerAddr, CatalogProtocol.class, false) {
- public Boolean call(NettyClientBase client) throws ServiceException {
-
- GetIndexByColumnRequest.Builder builder = GetIndexByColumnRequest.newBuilder();
- builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName));
- builder.setColumnName(columnName);
-
- CatalogProtocolService.BlockingInterface stub = getStub(client);
- return stub.existIndexByColumn(null, builder.build()).getValue();
- }
- }.withRetries();
- } catch (ServiceException e) {
- LOG.error(e.getMessage(), e);
- return false;
- }
- }
-
- @Override
public final IndexDesc getIndexByName(final String databaseName, final String indexName) {
try {
return new ServerCallable<IndexDesc>(this.pool, catalogServerAddr, CatalogProtocol.class, false) {
@@ -471,28 +451,6 @@ public abstract class AbstractCatalogClient implements CatalogService {
}
@Override
- public final IndexDesc getIndexByColumn(final String databaseName,
- final String tableName,
- final String columnName) {
- try {
- return new ServerCallable<IndexDesc>(this.pool, catalogServerAddr, CatalogProtocol.class, false) {
- public IndexDesc call(NettyClientBase client) throws ServiceException {
-
- GetIndexByColumnRequest.Builder builder = GetIndexByColumnRequest.newBuilder();
- builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName));
- builder.setColumnName(columnName);
-
- CatalogProtocolService.BlockingInterface stub = getStub(client);
- return new IndexDesc(stub.getIndexByColumn(null, builder.build()));
- }
- }.withRetries();
- } catch (ServiceException e) {
- LOG.error(e.getMessage(), e);
- return null;
- }
- }
-
- @Override
public boolean dropIndex(final String databaseName,
final String indexName) {
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
index 667ee88..76ab8ba 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
@@ -19,7 +19,6 @@
package org.apache.tajo.catalog;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
-import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes.DataType;
import java.util.Collection;
@@ -155,12 +154,8 @@ public interface CatalogService {
boolean existIndexByName(String databaseName, String indexName);
- boolean existIndexByColumn(String databaseName, String tableName, String columnName);
-
IndexDesc getIndexByName(String databaseName, String indexName);
- IndexDesc getIndexByColumn(String databaseName, String tableName, String columnName);
-
boolean dropIndex(String databaseName, String indexName);
boolean createFunction(FunctionDesc funcDesc);
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
index c5cb528..ee8dbc4 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
+++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
@@ -59,9 +59,7 @@ service CatalogProtocolService {
rpc createIndex(IndexDescProto) returns (BoolProto);
rpc dropIndex(IndexNameProto) returns (BoolProto);
rpc existIndexByName(IndexNameProto) returns (BoolProto);
- rpc existIndexByColumn(GetIndexByColumnRequest) returns (BoolProto);
rpc getIndexByName(IndexNameProto) returns (IndexDescProto);
- rpc getIndexByColumn(GetIndexByColumnRequest) returns (IndexDescProto);
rpc createFunction(FunctionDescProto) returns (BoolProto);
rpc dropFunction(UnregisterFunctionRequest) returns (BoolProto);
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
index aceb6f1..115ee6f 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
@@ -145,5 +145,4 @@ public class Column implements ProtoObject<ColumnProto>, GsonObject {
public String toJson() {
return CatalogGsonHelper.toJson(this, Column.class);
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
index cff2059..c8c6cd3 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
@@ -23,8 +23,12 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexKeyProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.common.ProtoObject;
+import org.apache.tajo.util.TUtil;
+
+import java.util.List;
public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
private IndexDescProto.Builder builder;
@@ -32,35 +36,42 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
private String indexName; // required
private String databaseName; // required
private String tableName; // required
- private Column column; // required
private IndexMethod indexMethod; // required
+ private List<IndexKey> keys;
private boolean isUnique = false; // optional [default = false]
private boolean isClustered = false; // optional [default = false]
- private boolean isAscending = false; // optional [default = false]
+ private String predicate; // optional
public IndexDesc() {
this.builder = IndexDescProto.newBuilder();
}
- public IndexDesc(String idxName, String databaseName, String tableName, Column column,
- IndexMethod type, boolean isUnique, boolean isClustered, boolean isAscending) {
+ public IndexDesc(String idxName, String databaseName, String tableName, IndexMethod type,
+ List<IndexKey> keys, boolean isUnique, boolean isClustered, String predicate) {
this();
this.indexName = idxName;
this.databaseName = databaseName;
this.tableName = tableName;
- this.column = column;
this.indexMethod = type;
+ this.keys = keys;
this.isUnique = isUnique;
this.isClustered = isClustered;
- this.isAscending = isAscending;
+ this.predicate = predicate;
}
public IndexDesc(IndexDescProto proto) {
- this(proto.getIndexName(),
- proto.getTableIdentifier().getDatabaseName(),
- proto.getTableIdentifier().getTableName(),
- new Column(proto.getColumn()),
- proto.getIndexMethod(), proto.getIsUnique(), proto.getIsClustered(), proto.getIsAscending());
+ this();
+ this.indexName = proto.getIndexName();
+ this.databaseName = proto.getTableIdentifier().getDatabaseName();
+ this.tableName = proto.getTableIdentifier().getTableName();
+ this.indexMethod = proto.getMethod();
+ this.keys = TUtil.newList();
+ for (IndexKeyProto eachProto : proto.getKeysList()) {
+ this.keys.add(new IndexKey(eachProto));
+ }
+ this.isUnique = proto.getIsUnique();
+ this.isClustered = proto.getIsClustered();
+ this.predicate = proto.hasPredicate() ? proto.getPredicate() : null;
}
public String getIndexName() {
@@ -71,10 +82,6 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
return tableName;
}
- public Column getColumn() {
- return column;
- }
-
public IndexMethod getIndexMethod() {
return this.indexMethod;
}
@@ -87,14 +94,24 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
return this.isUnique;
}
- public boolean isAscending() {
- return this.isAscending;
+ public List<IndexKey> getKeys() {
+ return keys;
+ }
+
+ public String getPredicate() {
+ return predicate;
+ }
+
+ public boolean hasPredicate() {
+ return predicate != null;
}
@Override
public IndexDescProto getProto() {
if (builder == null) {
builder = IndexDescProto.newBuilder();
+ } else {
+ builder.clear();
}
CatalogProtos.TableIdentifierProto.Builder tableIdentifierBuilder = CatalogProtos.TableIdentifierProto.newBuilder();
@@ -107,11 +124,15 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
builder.setTableIdentifier(tableIdentifierBuilder.build());
builder.setIndexName(this.indexName);
- builder.setColumn(this.column.getProto());
- builder.setIndexMethod(indexMethod);
+ builder.setMethod(indexMethod);
builder.setIsUnique(this.isUnique);
builder.setIsClustered(this.isClustered);
- builder.setIsAscending(this.isAscending);
+ for (IndexKey eachKey : keys) {
+ builder.addKeys(eachKey.getProto());
+ }
+ if (this.predicate != null) {
+ builder.setPredicate(this.predicate);
+ }
return builder.build();
}
@@ -121,30 +142,33 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
IndexDesc other = (IndexDesc) obj;
return getIndexName().equals(other.getIndexName())
&& getTableName().equals(other.getTableName())
- && getColumn().equals(other.getColumn())
&& getIndexMethod().equals(other.getIndexMethod())
&& isUnique() == other.isUnique()
&& isClustered() == other.isClustered()
- && isAscending() == other.isAscending();
+ && TUtil.checkEquals(getKeys(), other.getKeys())
+ && TUtil.checkEquals(getPredicate(), other.getPredicate());
} else {
return false;
}
}
public int hashCode() {
- return Objects.hashCode(getIndexName(), getTableName(), getColumn(),
- getIndexMethod(), isUnique(), isClustered(), isAscending());
+ return Objects.hashCode(getIndexName(), getTableName(), getIndexMethod(),
+ isUnique(), isClustered(), getKeys(), getPredicate());
}
public Object clone() throws CloneNotSupportedException {
IndexDesc desc = (IndexDesc) super.clone();
desc.indexName = indexName;
desc.tableName = tableName;
- desc.column = column;
desc.indexMethod = indexMethod;
desc.isUnique = isUnique;
desc.isClustered = isClustered;
- desc.isAscending = isAscending;
+ desc.keys = TUtil.newList();
+ for (IndexKey eachKey : keys) {
+ desc.keys.add((IndexKey) eachKey.clone());
+ }
+ desc.predicate = predicate;
return desc;
}
@@ -152,4 +176,74 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
return gson.toJson(this);
}
+
+ public static class IndexKey implements ProtoObject<IndexKeyProto>, Cloneable {
+ private final IndexKeyProto.Builder builder;
+ private String keyJson;
+ private boolean ascending = true;
+ private boolean nullFirst = false;
+
+ public IndexKey() {
+ builder = IndexKeyProto.newBuilder();
+ }
+
+ public IndexKey(final String keyJson, final boolean ascending, final boolean nullFirst) {
+ this();
+ this.keyJson = keyJson;
+ this.ascending = ascending;
+ this.nullFirst = nullFirst;
+ }
+
+ public IndexKey(IndexKeyProto proto) {
+ this(proto.getKeyJson(), proto.getAscending(), proto.getNullFirst());
+ }
+
+ @Override
+ public IndexKeyProto getProto() {
+ if (builder != null) {
+ builder.clear();
+ }
+ builder.setKeyJson(keyJson);
+ builder.setAscending(ascending);
+ builder.setNullFirst(nullFirst);
+ return builder.build();
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ IndexKey clone = (IndexKey) super.clone();
+ clone.keyJson = this.keyJson;
+ clone.ascending = this.ascending;
+ clone.nullFirst = this.nullFirst;
+ return clone;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof IndexKey) {
+ IndexKey other = (IndexKey) o;
+ return this.keyJson.equals(other.keyJson) &&
+ this.ascending == other.ascending &&
+ this.nullFirst == other.nullFirst;
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(keyJson, ascending, nullFirst);
+ }
+
+ public String getKeyJson() {
+ return keyJson;
+ }
+
+ public boolean isAscending() {
+ return ascending;
+ }
+
+ public boolean isNullFirst() {
+ return nullFirst;
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 367d0b8..90929a2 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -137,14 +137,21 @@ message FunctionDescProto {
optional string detail = 8;
}
+message IndexKeyProto {
+ required string keyJson = 1;
+ optional bool ascending = 2 [default = true];
+ optional bool nullFirst = 3 [default = false];
+}
+
message IndexDescProto {
- required TableIdentifierProto tableIdentifier = 1;
- required string indexName = 2;
- required ColumnProto column = 3;
- required IndexMethod indexMethod = 4;
+ required string indexName = 1;
+ required TableIdentifierProto tableIdentifier = 2;
+ required IndexMethod method = 3;
+ repeated IndexKeyProto keys = 4;
optional bool isUnique = 5 [default = false];
optional bool isClustered = 6 [default = false];
- optional bool isAscending = 7 [default = false];
+ optional string predicate = 7;
+ // optional string path = 8;
}
enum IndexMethod {
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java
index 3fa8f53..6f9f785 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java
@@ -18,34 +18,55 @@
package org.apache.tajo.catalog;
+import org.apache.tajo.catalog.IndexDesc.IndexKey;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
-import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.util.TUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.util.List;
+
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.*;
public class TestIndexDesc {
static IndexDesc desc1;
static IndexDesc desc2;
static IndexDesc desc3;
-
+ static List<IndexKey> keys1;
+ static List<IndexKey> keys2;
+ static List<IndexKey> keys3;
+ static String pred2;
+
static {
- desc1 = new IndexDesc(
- "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4),
- IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true);
-
- desc2 = new IndexDesc(
- "idx_test2", DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8),
- IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false);
-
- desc3 = new IndexDesc(
- "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4),
- IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true);
+ keys1 = TUtil.newList();
+ keys1.add(new IndexKey("{\"name\":\"id\",\"dataType\":{\"type\":\"INT4\"}}", true, true));
+ desc1 = new IndexDesc("idx_test", DEFAULT_DATABASE_NAME, "indexed",
+ IndexMethod.TWO_LEVEL_BIN_TREE, keys1, true, true, null);
+
+ keys2 = TUtil.newList();
+ keys2.add(new IndexKey("{\"name\":\"score\",\"dataType\":{\"type\":\"FLOAT8\"}}", false, false));
+ pred2 = "{\n" +
+ " \"LeftExpr\": {\n" +
+ " \"ColumnName\": \"score\",\n" +
+ " \"OpType\": \"Column\"\n" +
+ " },\n" +
+ " \"RightExpr\": {\n" +
+ " \"Value\": \"10\",\n" +
+ " \"ValueType\": \"Unsigned_Integer\",\n" +
+ " \"OpType\": \"Literal\"\n" +
+ " },\n" +
+ " \"OpType\": \"Equals\"\n" +
+ "}";
+ desc2 = new IndexDesc("idx_test2", DEFAULT_DATABASE_NAME, "indexed",
+ IndexMethod.TWO_LEVEL_BIN_TREE, keys2, false, false, pred2);
+
+ keys3 = TUtil.newList();
+ keys3.add(new IndexKey("{\"name\":\"id\",\"dataType\":{\"type\":\"INT4\"}}", true, true));
+ desc3 = new IndexDesc("idx_test", DEFAULT_DATABASE_NAME, "indexed",
+ IndexMethod.TWO_LEVEL_BIN_TREE, keys3, true, true, null);
}
@BeforeClass
@@ -67,19 +88,19 @@ public class TestIndexDesc {
public void testGetFields() {
assertEquals("idx_test", desc1.getIndexName());
assertEquals("indexed", desc1.getTableName());
- assertEquals(new Column("id", Type.INT4), desc1.getColumn());
assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc1.getIndexMethod());
+ assertTrue(keys1.equals(desc1.getKeys()));
assertEquals(true, desc1.isUnique());
assertEquals(true, desc1.isClustered());
- assertEquals(true, desc1.isAscending());
+ assertNull(desc1.getPredicate());
assertEquals("idx_test2", desc2.getIndexName());
assertEquals("indexed", desc2.getTableName());
- assertEquals(new Column("score", Type.FLOAT8), desc2.getColumn());
assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc2.getIndexMethod());
+ assertTrue(keys2.equals(desc2.getKeys()));
assertEquals(false, desc2.isUnique());
assertEquals(false, desc2.isClustered());
- assertEquals(false, desc2.isAscending());
+ assertEquals(pred2, desc2.getPredicate());
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index 4e391aa..55a4745 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -739,27 +739,6 @@ public class CatalogServer extends AbstractService {
}
@Override
- public BoolProto existIndexByColumn(RpcController controller, GetIndexByColumnRequest request)
- throws ServiceException {
-
- TableIdentifierProto identifier = request.getTableIdentifier();
- String databaseName = identifier.getDatabaseName();
- String tableName = identifier.getTableName();
- String columnName = request.getColumnName();
-
- rlock.lock();
- try {
- return store.existIndexByColumn(databaseName, tableName, columnName) ?
- ProtoUtil.TRUE : ProtoUtil.FALSE;
- } catch (Exception e) {
- LOG.error(e);
- return BoolProto.newBuilder().setValue(false).build();
- } finally {
- rlock.unlock();
- }
- }
-
- @Override
public IndexDescProto getIndexByName(RpcController controller, IndexNameProto request)
throws ServiceException {
@@ -781,29 +760,6 @@ public class CatalogServer extends AbstractService {
}
@Override
- public IndexDescProto getIndexByColumn(RpcController controller, GetIndexByColumnRequest request)
- throws ServiceException {
-
- TableIdentifierProto identifier = request.getTableIdentifier();
- String databaseName = identifier.getDatabaseName();
- String tableName = identifier.getTableName();
- String columnName = request.getColumnName();
-
- rlock.lock();
- try {
- if (!store.existIndexByColumn(databaseName, tableName, columnName)) {
- throw new NoSuchIndexException(databaseName, columnName);
- }
- return store.getIndexByColumn(databaseName, tableName, columnName);
- } catch (Exception e) {
- LOG.error("ERROR : cannot get index for " + tableName + "." + columnName, e);
- return null;
- } finally {
- rlock.unlock();
- }
- }
-
- @Override
public BoolProto dropIndex(RpcController controller, IndexNameProto request)
throws ServiceException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index bcf6774..449ec74 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -35,15 +35,11 @@ import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.exception.InternalException;
import org.apache.tajo.exception.UnimplementedException;
-import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.Pair;
import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
+import java.sql.*;
import java.util.*;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
@@ -683,20 +679,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt.close();
}
- String tidSql =
- "SELECT TID from " + TB_TABLES + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_NAME + "=?";
- pstmt = conn.prepareStatement(tidSql);
- pstmt.setInt(1, dbid);
- pstmt.setString(2, tableName);
- res = pstmt.executeQuery();
-
- if (!res.next()) {
- throw new CatalogException("ERROR: there is no TID matched to " + table.getTableName());
- }
-
- int tableId = res.getInt("TID");
- res.close();
- pstmt.close();
+ int tableId = getTableId(dbid, databaseName, tableName);
String colSql =
"INSERT INTO " + TB_COLUMNS + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) ";
@@ -1610,7 +1593,6 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
String databaseName = proto.getTableIdentifier().getDatabaseName();
String tableName = proto.getTableIdentifier().getTableName();
- String columnName = CatalogUtil.extractSimpleName(proto.getColumn().getName());
try {
int databaseId = getDatabaseId(databaseName);
@@ -1618,12 +1600,25 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
String sql = "INSERT INTO " + TB_INDEXES +
" (" + COL_DATABASES_PK + ", " + COL_TABLES_PK + ", INDEX_NAME, " +
- "COLUMN_NAME, DATA_TYPE, INDEX_TYPE, IS_UNIQUE, IS_CLUSTERED, IS_ASCENDING) VALUES (?,?,?,?,?,?,?,?,?)";
+ "METHOD, EXPR_NUM, EXPRS, ASC_ORDERS, NULL_ORDERS, IS_UNIQUE, IS_CLUSTERED, PRED) " +
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?)";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
}
+ StringBuilder exprsBuilder = new StringBuilder();
+ StringBuilder ascOrdersBuilder = new StringBuilder();
+ StringBuilder nullOrdersBuilder = new StringBuilder();
+ for (IndexKeyProto eachKey : proto.getKeysList()) {
+ exprsBuilder.append(eachKey.getKeyJson()).append(",");
+ ascOrdersBuilder.append(eachKey.getAscending()).append(",");
+ nullOrdersBuilder.append(eachKey.getNullFirst()).append(",");
+ }
+ exprsBuilder.deleteCharAt(exprsBuilder.length()-1);
+ ascOrdersBuilder.deleteCharAt(ascOrdersBuilder.length()-1);
+ nullOrdersBuilder.deleteCharAt(nullOrdersBuilder.length()-1);
+
conn = getConnection();
conn.setAutoCommit(false);
@@ -1631,12 +1626,18 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt.setInt(1, databaseId);
pstmt.setInt(2, tableId);
pstmt.setString(3, proto.getIndexName());
- pstmt.setString(4, columnName);
- pstmt.setString(5, proto.getColumn().getDataType().getType().name());
- pstmt.setString(6, proto.getIndexMethod().toString());
- pstmt.setBoolean(7, proto.hasIsUnique() && proto.getIsUnique());
- pstmt.setBoolean(8, proto.hasIsClustered() && proto.getIsClustered());
- pstmt.setBoolean(9, proto.hasIsAscending() && proto.getIsAscending());
+ pstmt.setString(4, proto.getMethod().name());
+ pstmt.setInt(5, proto.getKeysCount());
+ pstmt.setString(6, exprsBuilder.toString());
+ pstmt.setString(7, ascOrdersBuilder.toString());
+ pstmt.setString(8, nullOrdersBuilder.toString());
+ pstmt.setBoolean(9, proto.hasIsUnique() && proto.getIsUnique());
+ pstmt.setBoolean(10, proto.hasIsClustered() && proto.getIsClustered());
+ if (proto.hasPredicate()) {
+ pstmt.setString(11, proto.getPredicate());
+ } else {
+ pstmt.setNull(11, Types.VARCHAR, "PRED");
+ }
pstmt.executeUpdate();
conn.commit();
} catch (SQLException se) {
@@ -1649,23 +1650,38 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
@Override
public void dropIndex(String databaseName, final String indexName) throws CatalogException {
Connection conn = null;
- PreparedStatement pstmt = null;
try {
- int databaseId = getDatabaseId(databaseName);
- String sql = "DELETE FROM " + TB_INDEXES + " WHERE " + COL_DATABASES_PK + "=? AND INDEX_NAME=?";
+ conn = getConnection();
+ conn.setAutoCommit(false);
+ dropIndexInternal(conn, databaseName, indexName);
+ conn.commit();
+ } catch (SQLException se) {
+ try {
+ conn.rollback();
+ } catch (SQLException e) {
+ LOG.error(e);
+ }
+ } finally {
+ CatalogUtil.closeQuietly(conn);
+ }
+ }
+ private void dropIndexInternal(Connection conn, final String databaseName, final String indexName)
+ throws SQLException {
+ PreparedStatement pstmt = null;
+ try {
+ int databaseId = getDatabaseId(databaseName);
+ String sql = "DELETE FROM " + TB_INDEXES +
+ " WHERE " + COL_DATABASES_PK + "=? AND INDEX_NAME=?";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
}
- conn = getConnection();
pstmt = conn.prepareStatement(sql);
pstmt.setInt(1, databaseId);
pstmt.setString(2, indexName);
pstmt.executeUpdate();
- } catch (SQLException se) {
- throw new CatalogException(se);
} finally {
CatalogUtil.closeQuietly(pstmt);
}
@@ -1690,8 +1706,8 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
final static String GET_INDEXES_SQL =
- "SELECT " + COL_TABLES_PK + ", INDEX_NAME, COLUMN_NAME, DATA_TYPE, INDEX_TYPE, IS_UNIQUE, " +
- "IS_CLUSTERED, IS_ASCENDING FROM " + TB_INDEXES;
+ "SELECT " + COL_TABLES_PK + ", INDEX_NAME, EXPRS, METHOD, IS_UNIQUE, " +
+ "IS_CLUSTERED FROM " + TB_INDEXES;
@Override
@@ -1734,46 +1750,6 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
@Override
- public IndexDescProto getIndexByColumn(final String databaseName,
- final String tableName,
- final String columnName) throws CatalogException {
- Connection conn = null;
- ResultSet res = null;
- PreparedStatement pstmt = null;
- IndexDescProto proto = null;
-
- try {
- int databaseId = getDatabaseId(databaseName);
-
- String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND COLUMN_NAME=?";
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(sql);
- }
-
- conn = getConnection();
- pstmt = conn.prepareStatement(sql);
- pstmt.setInt(1, databaseId);
- ;
- pstmt.setString(2, columnName);
- res = pstmt.executeQuery();
- if (!res.next()) {
- throw new CatalogException("ERROR: there is no index matched to " + columnName);
- }
- IndexDescProto.Builder builder = IndexDescProto.newBuilder();
- resultToIndexDescProtoBuilder(builder, res);
- builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName));
- proto = builder.build();
- } catch (SQLException se) {
- throw new CatalogException(se);
- } finally {
- CatalogUtil.closeQuietly(pstmt, res);
- }
-
- return proto;
- }
-
- @Override
public boolean existIndexByName(String databaseName, final String indexName) throws CatalogException {
Connection conn = null;
ResultSet res = null;
@@ -1807,39 +1783,6 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
@Override
- public boolean existIndexByColumn(String databaseName, String tableName, String columnName)
- throws CatalogException {
- Connection conn = null;
- ResultSet res = null;
- PreparedStatement pstmt = null;
-
- boolean exist = false;
-
- try {
- int databaseId = getDatabaseId(databaseName);
-
- String sql =
- "SELECT INDEX_NAME FROM " + TB_INDEXES + " WHERE " + COL_DATABASES_PK + "=? AND COLUMN_NAME=?";
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(sql);
- }
-
- conn = getConnection();
- pstmt = conn.prepareStatement(sql);
- pstmt.setInt(1, databaseId);
- pstmt.setString(2, columnName);
- res = pstmt.executeQuery();
- exist = res.next();
- } catch (SQLException se) {
- throw new CatalogException(se);
- } finally {
- CatalogUtil.closeQuietly(pstmt, res);
- }
- return exist;
- }
-
- @Override
public IndexDescProto[] getIndexes(String databaseName, final String tableName)
throws CatalogException {
Connection conn = null;
@@ -1884,11 +1827,9 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
private void resultToIndexDescProtoBuilder(IndexDescProto.Builder builder,
final ResultSet res) throws SQLException {
builder.setIndexName(res.getString("index_name"));
- builder.setColumn(indexResultToColumnProto(res));
- builder.setIndexMethod(getIndexMethod(res.getString("index_type").trim()));
+ builder.setMethod(getIndexMethod(res.getString("method").trim()));
builder.setIsUnique(res.getBoolean("is_unique"));
builder.setIsClustered(res.getBoolean("is_clustered"));
- builder.setIsAscending(res.getBoolean("is_ascending"));
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
index 5de9633..ce4ea79 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
@@ -19,12 +19,11 @@
package org.apache.tajo.catalog.store;
import org.apache.tajo.catalog.FunctionDesc;
+import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.*;
import java.io.Closeable;
-import org.apache.tajo.catalog.exception.CatalogException;
-
import java.util.Collection;
import java.util.List;
@@ -106,14 +105,8 @@ public interface CatalogStore extends Closeable {
IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException;
- IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName)
- throws CatalogException;
-
boolean existIndexByName(String databaseName, String indexName) throws CatalogException;
- boolean existIndexByColumn(String databaseName, String tableName, String columnName)
- throws CatalogException;
-
IndexDescProto [] getIndexes(String databaseName, String tableName) throws CatalogException;
/************************** FUNCTION *****************************/
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java
index d6f9fc3..6a3b13c 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java
@@ -204,12 +204,6 @@ public class DerbyStore extends AbstractDBStore {
}
stmt.addBatch(sql);
- sql = "CREATE INDEX idx_indexes_columns ON " + TB_INDEXES + "(" + COL_DATABASES_PK + ",column_name)";
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(sql);
- }
- stmt.addBatch(sql);
stmt.executeBatch();
LOG.info("Table '" + TB_INDEXES + "' is created.");
baseTableMaps.put(TB_INDEXES, true);
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
index ca99160..db9b8f2 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
@@ -120,6 +120,8 @@ public class MemStore implements CatalogStore {
}
databases.put(databaseName, new HashMap<String, CatalogProtos.TableDescProto>());
+ indexes.put(databaseName, new HashMap<String, IndexDescProto>());
+ indexesByColumn.put(databaseName, new HashMap<String, IndexDescProto>());
}
@Override
@@ -368,9 +370,11 @@ public class MemStore implements CatalogStore {
throw new AlreadyExistsIndexException(proto.getIndexName());
}
- index.put(proto.getIndexName(), proto);
- indexByColumn.put(proto.getTableIdentifier().getTableName() + "."
- + CatalogUtil.extractSimpleName(proto.getColumn().getName()), proto);
+ String indexName = proto.getIndexName();
+ if (CatalogUtil.isFQTableName(indexName)) {
+ indexName = CatalogUtil.splitFQTableName(indexName)[1];
+ }
+ index.put(indexName, proto);
}
/* (non-Javadoc)
@@ -398,21 +402,6 @@ public class MemStore implements CatalogStore {
return index.get(indexName);
}
- /* (non-Javadoc)
- * @see CatalogStore#getIndexByName(java.lang.String, java.lang.String)
- */
- @Override
- public IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName)
- throws CatalogException {
-
- Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
- if (!indexByColumn.containsKey(columnName)) {
- throw new NoSuchIndexException(columnName);
- }
-
- return indexByColumn.get(columnName);
- }
-
@Override
public boolean existIndexByName(String databaseName, String indexName) throws CatalogException {
Map<String, IndexDescProto> index = checkAndGetDatabaseNS(indexes, databaseName);
@@ -420,13 +409,6 @@ public class MemStore implements CatalogStore {
}
@Override
- public boolean existIndexByColumn(String databaseName, String tableName, String columnName)
- throws CatalogException {
- Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
- return indexByColumn.containsKey(columnName);
- }
-
- @Override
public IndexDescProto[] getIndexes(String databaseName, String tableName) throws CatalogException {
List<IndexDescProto> protos = new ArrayList<IndexDescProto>();
Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql
index c4cfc25..45ed144 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql
@@ -1,12 +1,15 @@
CREATE TABLE INDEXES (
+-- INDEX_ID INT NOT NULL GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1),
DB_ID INT NOT NULL REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE,
INDEX_NAME VARCHAR(128) NOT NULL,
- COLUMN_NAME VARCHAR(128) NOT NULL,
- DATA_TYPE VARCHAR(128) NOT NULL,
- INDEX_TYPE CHAR(32) NOT NULL,
+ METHOD CHAR(32) NOT NULL,
+ EXPR_NUM INT NOT NULL,
+ EXPRS VARCHAR(1024) NOT NULL,
+ ASC_ORDERS VARCHAR(1024) NOT NULL,
+ NULL_ORDERS VARCHAR(1024) NOT NULL,
IS_UNIQUE BOOLEAN NOT NULL,
IS_CLUSTERED BOOLEAN NOT NULL,
- IS_ASCENDING BOOLEAN NOT NULL,
+ PRED VARCHAR(4096),
CONSTRAINT C_INDEXES_PK PRIMARY KEY (DB_ID, INDEX_NAME)
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql
index 62feb36..b375a48 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql
@@ -1,16 +1,17 @@
CREATE TABLE INDEXES (
+-- INDEX_ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
DB_ID INT NOT NULL,
TID INT NOT NULL,
INDEX_NAME VARCHAR(128) NOT NULL,
- COLUMN_NAME VARCHAR(128) NOT NULL,
- DATA_TYPE VARCHAR(128) NOT NULL,
- INDEX_TYPE CHAR(32) NOT NULL,
+ METHOD CHAR(32) NOT NULL,
+ EXPR_NUM INT NOT NULL,
+ EXPRS VARCHAR(1024) NOT NULL,
+ ASC_ORDERS VARCHAR(1024) NOT NULL,
+ NULL_ORDERS VARCHAR(1024) NOT NULL,
IS_UNIQUE BOOLEAN NOT NULL,
IS_CLUSTERED BOOLEAN NOT NULL,
- IS_ASCENDING BOOLEAN NOT NULL,
- PRIMARY KEY (DB_ID, INDEX_NAME),
+ PRED VARCHAR(4096),
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
UNIQUE INDEX IDX_DB_ID_NAME (DB_ID, INDEX_NAME),
- INDEX IDX_TID_COLUMN_NAME (TID, COLUMN_NAME)
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql
index 62feb36..b375a48 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql
@@ -1,16 +1,17 @@
CREATE TABLE INDEXES (
+-- INDEX_ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
DB_ID INT NOT NULL,
TID INT NOT NULL,
INDEX_NAME VARCHAR(128) NOT NULL,
- COLUMN_NAME VARCHAR(128) NOT NULL,
- DATA_TYPE VARCHAR(128) NOT NULL,
- INDEX_TYPE CHAR(32) NOT NULL,
+ METHOD CHAR(32) NOT NULL,
+ EXPR_NUM INT NOT NULL,
+ EXPRS VARCHAR(1024) NOT NULL,
+ ASC_ORDERS VARCHAR(1024) NOT NULL,
+ NULL_ORDERS VARCHAR(1024) NOT NULL,
IS_UNIQUE BOOLEAN NOT NULL,
IS_CLUSTERED BOOLEAN NOT NULL,
- IS_ASCENDING BOOLEAN NOT NULL,
- PRIMARY KEY (DB_ID, INDEX_NAME),
+ PRED VARCHAR(4096),
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
UNIQUE INDEX IDX_DB_ID_NAME (DB_ID, INDEX_NAME),
- INDEX IDX_TID_COLUMN_NAME (TID, COLUMN_NAME)
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 4039ff2..9f15aae 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -21,6 +21,7 @@ package org.apache.tajo.catalog;
import com.google.common.collect.Sets;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.TajoConstants;
+import org.apache.tajo.catalog.IndexDesc.IndexKey;
import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.catalog.exception.NoSuchFunctionException;
import org.apache.tajo.catalog.function.Function;
@@ -30,8 +31,8 @@ import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.store.DerbyStore;
-import org.apache.tajo.catalog.store.MySQLStore;
import org.apache.tajo.catalog.store.MariaDBStore;
+import org.apache.tajo.catalog.store.MySQLStore;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
@@ -51,7 +52,6 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation;
import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
public class TestCatalog {
static final String FieldName1="f1";
@@ -373,17 +373,20 @@ public class TestCatalog {
static IndexDesc desc3;
static {
- desc1 = new IndexDesc(
- "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4),
- IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true);
+ List<IndexKey> indexKeys = new ArrayList<IndexKey>();
+ indexKeys.add(new IndexKey(new Column("id", Type.INT4).toJson(), true, true));
+ desc1 = new IndexDesc("idx_test", DEFAULT_DATABASE_NAME, "indexed", IndexMethod.TWO_LEVEL_BIN_TREE,
+ indexKeys, true, true, null);
- desc2 = new IndexDesc(
- "idx_test2", DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8),
- IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false);
+ indexKeys = new ArrayList<IndexKey>();
+ indexKeys.add(new IndexKey(new Column("score", Type.FLOAT8).toJson(), false, false));
+ desc2 = new IndexDesc("idx_test2", DEFAULT_DATABASE_NAME, "indexed", IndexMethod.TWO_LEVEL_BIN_TREE,
+ indexKeys, false, false, null);
- desc3 = new IndexDesc(
- "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4),
- IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true);
+ indexKeys = new ArrayList<IndexKey>();
+ indexKeys.add(new IndexKey(new Column("id", Type.INT4).toJson(), true, true));
+ desc3 = new IndexDesc("idx_test", DEFAULT_DATABASE_NAME, "indexed", IndexMethod.TWO_LEVEL_BIN_TREE,
+ indexKeys, true, true, null);
}
public static TableDesc prepareTable() throws IOException {
@@ -439,18 +442,14 @@ public class TestCatalog {
assertTrue(catalog.createTable(desc));
assertFalse(catalog.existIndexByName("db1", desc1.getIndexName()));
- assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id"));
catalog.createIndex(desc1);
assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getIndexName()));
- assertTrue(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id"));
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getIndexName()));
- assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "score"));
catalog.createIndex(desc2);
assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getIndexName()));
- assertTrue(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "score"));
-
+
catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getIndexName());
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getIndexName()));
catalog.dropIndex(DEFAULT_DATABASE_NAME, desc2.getIndexName());
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
index 333c8d6..b466b96 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
@@ -29,7 +29,10 @@ import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.TajoProtos.QueryState;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.annotation.ThreadSafe;
-import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.cli.InvalidClientSessionException;
import org.apache.tajo.conf.TajoConf;
@@ -762,6 +765,50 @@ public class TajoClient implements Closeable {
}
+ /**
+ * Drop an index
+ *
+ * @param indexName The index name to be dropped. This name is case sensitive.
+ * @return True if the index is dropped successfully.
+ */
+ public boolean dropIndex(final String indexName) throws ServiceException {
+ return new ServerCallable<Boolean>(connPool, tajoMasterAddr,
+ TajoMasterClientProtocol.class, false, true) {
+ public Boolean call(NettyClientBase client) throws ServiceException {
+ checkSessionAndGet(client);
+
+ TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub();
+
+ SessionedStringProto.Builder builder = SessionedStringProto.newBuilder();
+ builder.setSessionId(sessionId);
+ builder.setValue(indexName);
+ return tajoMasterService.dropIndex(null, builder.build()).getValue();
+ }
+ }.withRetries();
+ }
+
+ /**
+ * Does the index exist?
+ *
+ * @param indexName The index name to be checked. This name is case sensitive.
+ * @return True if so.
+ */
+ public boolean existIndex(final String indexName) throws ServiceException {
+ return new ServerCallable<Boolean>(connPool, tajoMasterAddr,
+ TajoMasterClientProtocol.class, false, true) {
+ public Boolean call(NettyClientBase client) throws ServiceException {
+ checkSessionAndGet(client);
+
+ TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub();
+
+ SessionedStringProto.Builder builder = SessionedStringProto.newBuilder();
+ builder.setSessionId(sessionId);
+ builder.setValue(indexName);
+ return tajoMasterService.existIndex(null, builder.build()).getValue();
+ }
+ }.withRetries();
+ }
+
public List<BriefQueryInfo> getRunningQueryList() throws ServiceException {
return new ServerCallable<List<BriefQueryInfo>>(connPool, tajoMasterAddr,
TajoMasterClientProtocol.class, false, true) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-client/src/main/proto/ClientProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto
index c66b228..f56383e 100644
--- a/tajo-client/src/main/proto/ClientProtos.proto
+++ b/tajo-client/src/main/proto/ClientProtos.proto
@@ -224,3 +224,7 @@ message FunctionResponse {
repeated FunctionDescProto functions = 2;
optional string errorMessage = 3;
}
+
+message CreateIndexRequest {
+ required IndexDescProto desc = 1;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
index 9495fb1..79f7c27 100644
--- a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
+++ b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
@@ -65,4 +65,8 @@ service TajoMasterClientProtocolService {
rpc getTableList(GetTableListRequest) returns (GetTableListResponse);
rpc getTableDesc(GetTableDescRequest) returns (TableResponse);
rpc getFunctionList(SessionedStringProto) returns (FunctionResponse);
+
+ // Index Management APIs
+ rpc existIndex(SessionedStringProto) returns (BoolProto);
+ rpc dropIndex(SessionedStringProto) returns (BoolProto);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 706c973..438a255 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -68,10 +68,19 @@ schema_statement
;
index_statement
- : CREATE (u=UNIQUE)? INDEX identifier ON table_name (method_specifier)?
+ : create_index_statement
+ | drop_index_statement
+ ;
+
+create_index_statement
+ : CREATE (u=UNIQUE)? INDEX (if_not_exists)? identifier ON table_name (method_specifier)?
LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)?
;
+drop_index_statement
+ : DROP INDEX (if_exists)? index_name = identifier
+ ;
+
database_definition
: CREATE DATABASE (if_not_exists)? dbname = identifier
;
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index faf0212..08f7939 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -1142,7 +1142,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
}
@Override
- public Expr visitIndex_statement(SQLParser.Index_statementContext ctx) {
+ public Expr visitCreate_index_statement(@NotNull SQLParser.Create_index_statementContext ctx) {
String indexName = ctx.identifier().getText();
String tableName = ctx.table_name().getText();
Relation relation = new Relation(tableName);
@@ -1178,6 +1178,11 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
}
@Override
+ public Expr visitDrop_index_statement(@NotNull SQLParser.Drop_index_statementContext ctx) {
+ return new DropIndex(ctx.index_name.getText(), checkIfExist(ctx.if_exists()));
+ }
+
+ @Override
public Expr visitDatabase_definition(@NotNull SQLParser.Database_definitionContext ctx) {
return new CreateDatabase(ctx.identifier().getText(), null, checkIfExist(ctx.if_not_exists()));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java
index eb8a21f..0d73dfc 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java
@@ -49,6 +49,7 @@ public interface AlgebraVisitor<CONTEXT, RESULT> {
RESULT visitAlterTablespace(CONTEXT ctx, Stack<Expr> stack, AlterTablespace expr) throws PlanningException;
RESULT visitAlterTable(CONTEXT ctx, Stack<Expr> stack, AlterTable expr) throws PlanningException;
RESULT visitCreateIndex(CONTEXT ctx, Stack<Expr> stack, CreateIndex expr) throws PlanningException;
+ RESULT visitDropIndex(CONTEXT ctx, Stack<Expr> stack, DropIndex expr) throws PlanningException;
RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException;
// Insert or Update
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java
index b93541b..4c9932a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java
@@ -121,6 +121,9 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE
case CreateIndex:
current = visitCreateIndex(ctx, stack, (CreateIndex) expr);
break;
+ case DropIndex:
+ current = visitDropIndex(ctx, stack, (DropIndex) expr);
+ break;
case TruncateTable:
current = visitTruncateTable(ctx, stack, (TruncateTable)expr);
break;
@@ -479,6 +482,11 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE
}
@Override
+ public RESULT visitDropIndex(CONTEXT ctx, Stack<Expr> stack, DropIndex expr) throws PlanningException {
+ return null;
+ }
+
+ @Override
public RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException {
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java
index 59422f1..75cb3af 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java
@@ -132,6 +132,9 @@ public class BasicLogicalPlanVisitor<CONTEXT, RESULT> implements LogicalPlanVisi
case CREATE_INDEX:
current = visitCreateIndex(context, plan, block, (CreateIndexNode) node, stack);
break;
+ case DROP_INDEX:
+ current = visitDropIndex(context, plan, block, (DropIndexNode) node, stack);
+ break;
case TRUNCATE_TABLE:
current = visitTruncateTable(context, plan, block, (TruncateTableNode) node, stack);
break;
@@ -358,6 +361,12 @@ public class BasicLogicalPlanVisitor<CONTEXT, RESULT> implements LogicalPlanVisi
}
@Override
+ public RESULT visitDropIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, DropIndexNode node,
+ Stack<LogicalNode> stack) throws PlanningException {
+ return null;
+ }
+
+ @Override
public RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block,
TruncateTableNode node, Stack<LogicalNode> stack) throws PlanningException {
return null;
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
index 0ed3bdd..7932072 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
@@ -460,6 +460,13 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanPrepr
return createIndex;
}
+ @Override
+ public LogicalNode visitDropIndex(PreprocessContext ctx, Stack<Expr> stack, DropIndex expr)
+ throws PlanningException {
+ DropIndexNode dropIndexNode = ctx.plan.createNode(DropIndexNode.class);
+ return dropIndexNode;
+ }
+
public LogicalNode visitTruncateTable(PreprocessContext ctx, Stack<Expr> stack, TruncateTable expr)
throws PlanningException {
TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class);
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java
index 57f943e..7369ff7 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java
@@ -97,6 +97,9 @@ public interface LogicalPlanVisitor<CONTEXT, RESULT> {
RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, CreateIndexNode node,
Stack<LogicalNode> stack) throws PlanningException;
+ RESULT visitDropIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, DropIndexNode node,
+ Stack<LogicalNode> stack) throws PlanningException;
+
RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node,
Stack<LogicalNode> stack) throws PlanningException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index fc7259b..0e2b26c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -46,8 +46,8 @@ import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule;
import org.apache.tajo.engine.utils.SchemaUtil;
import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.StorageUtil;
-import org.apache.tajo.util.Pair;
import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -1888,11 +1888,19 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createIndexNode.setOptions(keyValueSet);
}
+ createIndexNode.setInSchema(child.getOutSchema());
createIndexNode.setChild(child);
return createIndexNode;
}
@Override
+ public LogicalNode visitDropIndex(PlanContext context, Stack<Expr> stack, DropIndex dropIndex) {
+ DropIndexNode dropIndexNode = context.queryBlock.getNodeFromExpr(dropIndex);
+ dropIndexNode.init(dropIndex.getIndexName(), dropIndex.isIfExists());
+ return dropIndexNode;
+ }
+
+ @Override
public LogicalNode visitTruncateTable(PlanContext context, Stack<Expr> stack, TruncateTable truncateTable)
throws PlanningException {
TruncateTableNode truncateTableNode = context.queryBlock.getNodeFromExpr(truncateTable);
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
index 55892ee..d80c945 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
@@ -39,6 +39,18 @@ import java.util.*;
public class PlannerUtil {
+ public static boolean checkIfCreateIndexPlan(LogicalNode node) {
+ LogicalNode baseNode = node;
+ if (node instanceof LogicalRootNode) {
+ baseNode = ((LogicalRootNode) node).getChild();
+ }
+
+ NodeType type = baseNode.getType();
+
+ return
+ baseNode.getType() == NodeType.CREATE_INDEX;
+ }
+
public static boolean checkIfDDLPlan(LogicalNode node) {
LogicalNode baseNode = node;
if (node instanceof LogicalRootNode) {
@@ -54,7 +66,9 @@ public class PlannerUtil {
baseNode.getType() == NodeType.DROP_TABLE ||
baseNode.getType() == NodeType.ALTER_TABLESPACE ||
baseNode.getType() == NodeType.ALTER_TABLE ||
- baseNode.getType() == NodeType.TRUNCATE_TABLE;
+ baseNode.getType() == NodeType.TRUNCATE_TABLE ||
+ baseNode.getType() == NodeType.CREATE_INDEX ||
+ baseNode.getType() == NodeType.DROP_INDEX;
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java
index 1dca608..4c33958 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java
@@ -29,10 +29,12 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
public class CreateIndexNode extends UnaryNode implements Cloneable {
@Expose private boolean isUnique;
+ @Expose private boolean isClustered;
@Expose private String indexName;
@Expose private SortSpec[] sortSpecs;
@Expose private IndexMethod indexType = IndexMethod.TWO_LEVEL_BIN_TREE;
@Expose private KeyValueSet options;
+ @Expose private boolean ifNotExists;
public CreateIndexNode(int pid) {
super(pid, NodeType.CREATE_INDEX);
@@ -78,6 +80,14 @@ public class CreateIndexNode extends UnaryNode implements Cloneable {
return this.options;
}
+ public boolean isIfNotExists() {
+ return ifNotExists;
+ }
+
+ public void setIfNotExists(boolean ifNotExists) {
+ this.ifNotExists = ifNotExists;
+ }
+
@Override
public int hashCode() {
return Objects.hashCode(isUnique, indexName, sortSpecs, indexType, options);
@@ -130,4 +140,12 @@ public class CreateIndexNode extends UnaryNode implements Cloneable {
public PlanString getPlanString() {
return new PlanString(this);
}
+
+ public boolean isClustered() {
+ return isClustered;
+ }
+
+ public void setClustered(boolean isClustered) {
+ this.isClustered = isClustered;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropIndexNode.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropIndexNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropIndexNode.java
new file mode 100644
index 0000000..e50c0b9
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropIndexNode.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.planner.logical;
+
+import com.google.common.base.Objects;
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.engine.planner.PlanString;
+import org.apache.tajo.util.TUtil;
+
+public class DropIndexNode extends LogicalNode implements Cloneable {
+ @Expose private String indexName;
+ @Expose private boolean ifExists;
+
+ public DropIndexNode(int pid) {
+ super(pid, NodeType.DROP_INDEX);
+ }
+
+ public void init(String indexName, boolean ifExists) {
+ this.indexName = indexName;
+ this.ifExists = ifExists;
+ }
+
+ @Override
+ public void preOrder(LogicalNodeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public void postOrder(LogicalNodeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public PlanString getPlanString() {
+ return new PlanString(this).appendTitle(ifExists ? " IF EXISTS" : "");
+ }
+
+ @Override
+ public String toString() {
+ return "DROP INDEX " + (ifExists ? "IF EXISTS " : "") + indexName;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(indexName, ifExists);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof DropIndexNode) {
+ DropIndexNode other = (DropIndexNode) o;
+ return TUtil.checkEquals(this.indexName, other.indexName) &&
+ this.ifExists == other.ifExists;
+ }
+ return false;
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ DropIndexNode clone = (DropIndexNode) super.clone();
+ clone.indexName = this.indexName;
+ clone.ifExists = this.ifExists;
+ return clone;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ public boolean isIfExists() {
+ return ifExists;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java
index 2e9e0da..b8ba248 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java
@@ -56,6 +56,7 @@ public enum NodeType {
ALTER_TABLESPACE (AlterTablespaceNode.class),
ALTER_TABLE (AlterTableNode.class),
CREATE_INDEX(CreateIndexNode.class),
+ DROP_INDEX(DropIndexNode.class),
TRUNCATE_TABLE (TruncateTableNode.class);
private final Class<? extends LogicalNode> baseClass;
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java
index 91b5322..5f1189a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java
@@ -26,13 +26,14 @@ import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.PlannerUtil;
import org.apache.tajo.engine.planner.logical.CreateIndexNode;
import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleComparator;
import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.storage.fragment.FileFragment;
+import org.apache.tajo.storage.fragment.FragmentConvertor;
import org.apache.tajo.storage.index.bst.BSTIndex;
import org.apache.tajo.storage.index.bst.BSTIndex.BSTIndexWriter;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -53,6 +54,14 @@ public class StoreIndexExec extends UnaryPhysicalExec {
this.logicalPlan = logicalPlan;
}
+ private Path getIndexPath(TajoConf conf, String dbName, String indexName) {
+ // assume that there is only one fragment for each task
+ String table = context.getInputTables().iterator().next();
+ FileFragment fragment = FragmentConvertor.convert(FileFragment.class, context.getTable(table));
+ return new Path(TajoConf.getWarehouseDir(conf), dbName + "/" + indexName + "/" +
+ fragment.getStartKey() + "_" + fragment.getEndKey());
+ }
+
@Override
public void init() throws IOException {
super.init();
@@ -70,9 +79,7 @@ public class StoreIndexExec extends UnaryPhysicalExec {
TajoConf conf = context.getConf();
String[] splits = logicalPlan.getIndexName().split("\\.");
- Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), splits[0] + "/" + splits[1] + "/" +
- context.getUniqueKeyFromFragments());
- System.out.println("exec: " + indexPath);
+ Path indexPath = getIndexPath(conf, splits[0], splits[1]);
// TODO: Create factory using reflection
BSTIndex bst = new BSTIndex(conf);
this.comparator = new TupleComparator(keySchema, sortSpecs);
[6/7] TAJO-837: Register index meta information at Catalog. (jihoon)
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 73f3cf5..aed2fe6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -35,12 +35,14 @@ import org.apache.tajo.algebra.Expr;
import org.apache.tajo.algebra.JsonHelper;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.IndexDesc.IndexKey;
import org.apache.tajo.catalog.exception.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.eval.EvalNode;
import org.apache.tajo.engine.exception.IllegalQueryStatusException;
@@ -59,6 +61,7 @@ import org.apache.tajo.master.querymaster.QueryJobManager;
import org.apache.tajo.master.querymaster.QueryMasterTask;
import org.apache.tajo.master.session.Session;
import org.apache.tajo.storage.*;
+import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
@@ -187,10 +190,17 @@ public class GlobalEngine extends AbstractService {
if (PlannerUtil.checkIfDDLPlan(rootNode)) {
context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
- updateQuery(session, rootNode.getChild());
- responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ if (PlannerUtil.checkIfCreateIndexPlan(rootNode)) {
+ return createIndex(session, (CreateIndexNode)rootNode.getChild(), queryContext,
+ plan, sql, jsonExpr, responseBuilder);
+ } else {
+ updateQuery(session, rootNode.getChild());
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+
+ return responseBuilder.build();
+ }
} else if (plan.isExplain()) { // explain query
String explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
Schema schema = new Schema();
@@ -216,8 +226,10 @@ public class GlobalEngine extends AbstractService {
responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- // Simple query indicates a form of 'select * from tb_name [LIMIT X];'.
+ return responseBuilder.build();
+
} else if (PlannerUtil.checkIfSimpleQuery(plan)) {
+ // Simple query indicates a form of 'select * from tb_name [LIMIT X];'.
ScanNode scanNode = plan.getRootBlock().getNode(NodeType.SCAN);
TableDesc desc = scanNode.getTableDesc();
if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
@@ -232,8 +244,10 @@ public class GlobalEngine extends AbstractService {
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
- // NonFromQuery indicates a form of 'select a, x+y;'
+ return responseBuilder.build();
+
} else if (PlannerUtil.checkIfNonFromQuery(plan)) {
+ // NonFromQuery indicates a form of 'select a, x+y;'
Target [] targets = plan.getRootBlock().getRawTargets();
if (targets == null) {
throw new PlanningException("No targets");
@@ -261,32 +275,40 @@ public class GlobalEngine extends AbstractService {
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
}
+ return responseBuilder.build();
+
} else { // it requires distributed execution. So, the query is forwarded to a query master.
context.getSystemMetrics().counter("Query", "numDMLQuery").inc();
- hookManager.doHooks(queryContext, plan);
+ return executeInCluster(queryContext, plan, session, sql, jsonExpr, responseBuilder);
+ }
+ }
- QueryJobManager queryJobManager = this.context.getQueryJobManager();
- QueryInfo queryInfo;
+ private SubmitQueryResponse executeInCluster(QueryContext queryContext, LogicalPlan plan, Session session, String sql, String jsonExpr,
+ SubmitQueryResponse.Builder responseBuilder)
+ throws Exception {
+ LogicalRootNode rootNode = plan.getRootBlock().getRoot();
+ hookManager.doHooks(queryContext, plan);
- queryInfo = queryJobManager.createNewQueryJob(session, queryContext, sql, jsonExpr, rootNode);
+ QueryJobManager queryJobManager = this.context.getQueryJobManager();
+ QueryInfo queryInfo;
- if(queryInfo == null) {
- responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR);
- responseBuilder.setErrorMessage("Fail starting QueryMaster.");
- } else {
- responseBuilder.setIsForwarded(true);
- responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
- if(queryInfo.getQueryMasterHost() != null) {
- responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
- }
- responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
- LOG.info("Query is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort());
+ queryInfo = queryJobManager.createNewQueryJob(session, queryContext, sql, jsonExpr, rootNode);
+
+ if(queryInfo == null) {
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR);
+ responseBuilder.setErrorMessage("Fail starting QueryMaster.");
+ } else {
+ responseBuilder.setIsForwarded(true);
+ responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ if(queryInfo.getQueryMasterHost() != null) {
+ responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
}
+ responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
+ LOG.info("Query is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort());
}
- SubmitQueryResponse response = responseBuilder.build();
- return response;
+ return responseBuilder.build();
}
private void insertNonFromQuery(QueryContext queryContext, InsertNode insertNode, SubmitQueryResponse.Builder responseBuilder)
@@ -448,6 +470,11 @@ public class GlobalEngine extends AbstractService {
TruncateTableNode truncateTable = (TruncateTableNode) root;
truncateTable(session, truncateTable);
return true;
+ case DROP_INDEX:
+ DropIndexNode dropIndexNode = (DropIndexNode) root;
+ dropIndex(session, dropIndexNode);
+ return true;
+
default:
throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson());
}
@@ -636,6 +663,98 @@ public class GlobalEngine extends AbstractService {
}
}
+ /**
+ * Create an index for a given table.
+ * @param session user session
+ * @param createIndexNode the root of logical plan
+ */
+ private SubmitQueryResponse createIndex(final Session session, final CreateIndexNode createIndexNode,
+ QueryContext queryContext, LogicalPlan plan,
+ String sql, String jsonExpr,
+ SubmitQueryResponse.Builder responseBuilder) throws Exception {
+ SubmitQueryResponse response = null;
+ final CatalogService catalog = context.getCatalog();
+ final String dbName = session.getCurrentDatabase();
+ String indexName = createIndexNode.getIndexName();
+ if (CatalogUtil.isFQTableName(indexName)) {
+ indexName = CatalogUtil.splitFQTableName(indexName)[1];
+ }
+
+ boolean exists = catalog.existIndexByName(dbName, indexName);
+ if (exists) {
+ if (createIndexNode.isIfNotExists()) {
+ LOG.info("index \"" + indexName + "\" already exists." );
+ } else {
+ throw new AlreadyExistsIndexException(createIndexNode.getIndexName());
+ }
+ } else {
+ response = executeInCluster(queryContext, plan, session, sql, jsonExpr, responseBuilder);
+
+ // get the table name and predicate from scan
+ try {
+ ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN);
+ String tableName;
+ if (CatalogUtil.isFQTableName(scanNode.getTableName())) {
+ tableName = CatalogUtil.splitFQTableName(scanNode.getTableName())[1];
+ } else {
+ tableName = scanNode.getTableName();
+ }
+ String predicate = scanNode.hasQual() ? scanNode.getQual().toJson() : null;
+ // extract index keys
+ List<IndexKey> indexKeys = TUtil.newList();
+ for (SortSpec eachKey : createIndexNode.getSortSpecs()) {
+ indexKeys.add(new IndexKey(eachKey.getSortKey().toJson(), eachKey.isAscending(), eachKey.isNullFirst()));
+ }
+
+ IndexDesc indexDesc = new IndexDesc(indexName, dbName, tableName, createIndexNode.getIndexType(),
+ indexKeys, createIndexNode.isUnique(), createIndexNode.isClustered(), predicate);
+ catalog.createIndex(indexDesc);
+ } catch (Exception e) {
+ // delete index
+ deleteIndexFiles(dbName, indexName);
+ }
+ }
+ return response;
+ }
+
+ /**
+ * Drop the specified index.
+ * @param session user session
+ * @param indexName index name
+ * @param ifExists if exists
+ */
+ public void dropIndex(final Session session, String indexName, boolean ifExists) {
+ final CatalogService catalog = context.getCatalog();
+ final String dbName = session.getCurrentDatabase();
+
+ boolean exists = catalog.existIndexByName(dbName, indexName);
+ LOG.info("index name exist: " + exists);
+ if (!exists) {
+ if (ifExists) {
+ LOG.info("index \"" + indexName + "\" does not exist." );
+ } else {
+ throw new NoSuchIndexException(indexName);
+ }
+ } else {
+ catalog.dropIndex(dbName, indexName);
+ deleteIndexFiles(dbName, indexName);
+ }
+ }
+
+ private void deleteIndexFiles(String dbName, String indexName) {
+ Path indexPath = new Path(context.getConf().getVar(ConfVars.WAREHOUSE_DIR), dbName + "/" + indexName);
+ try {
+ FileSystem fs = indexPath.getFileSystem(context.getConf());
+ fs.delete(indexPath, true);
+ } catch (IOException e) {
+ throw new InternalError(e.getMessage());
+ }
+ }
+
+ private void dropIndex(final Session session, final DropIndexNode dropIndexNode) {
+ dropIndex(session, dropIndexNode.getIndexName(), dropIndexNode.isIfExists());
+ }
+
private boolean existColumnName(String tableName, String columnName) {
final TableDesc tableDesc = catalog.getTableDesc(tableName);
return tableDesc.getSchema().containsByName(columnName) ? true : false;
@@ -692,7 +811,7 @@ public class GlobalEngine extends AbstractService {
if (exists) {
if (ifNotExists) {
- LOG.info("relation \"" + qualifiedName + "\" is already exists." );
+ LOG.info("relation \"" + qualifiedName + "\" already exists." );
return catalog.getTableDesc(databaseName, simpleTableName);
} else {
throw new AlreadyExistsTableException(CatalogUtil.buildFQName(databaseName, tableName));
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 97f59ef..1ac85e3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -27,7 +27,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
-import org.apache.tajo.*;
+import org.apache.tajo.QueryId;
+import org.apache.tajo.QueryIdFactory;
+import org.apache.tajo.TajoIdProtos;
+import org.apache.tajo.TajoProtos;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.exception.NoSuchDatabaseException;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
@@ -751,5 +754,42 @@ public class TajoMasterClientService extends AbstractService {
throw new ServiceException(t);
}
}
+
+ @Override
+ public BoolProto existIndex(RpcController controller, SessionedStringProto request) throws ServiceException {
+ try {
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String databaseName;
+ String indexName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ indexName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ indexName = request.getValue();
+ }
+
+ if (catalog.existIndexByName(databaseName, indexName)) {
+ return BOOL_TRUE;
+ } else {
+ return BOOL_FALSE;
+ }
+ } catch (Throwable e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public BoolProto dropIndex(RpcController controller, SessionedStringProto request) throws ServiceException {
+ try {
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+ context.getGlobalEngine().dropIndex(session, request.getValue(), true);
+ return BOOL_TRUE;
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
index 7ecf27a..2b67538 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
@@ -294,14 +294,6 @@ public class TaskAttemptContext {
return fragmentMap.get(id).toArray(new FragmentProto[fragmentMap.get(id).size()]);
}
- public long getUniqueKeyFromFragments() {
- List<FragmentProto> totalFragments = new ArrayList<FragmentProto>();
- for (List<FragmentProto> eachFragments : fragmentMap.values()) {
- totalFragments.addAll(eachFragments);
- }
- return Objects.hashCode(totalFragments.toArray(new FragmentProto[totalFragments.size()]));
- }
-
public int hashCode() {
return Objects.hashCode(queryId);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index b97eb30..7e87c29 100644
--- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -36,7 +36,9 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.FileUtil;
-import org.junit.*;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.rules.TestName;
import java.io.File;
@@ -46,7 +48,10 @@ import java.net.URL;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import static org.junit.Assert.*;
@@ -382,6 +387,16 @@ public class QueryTestCaseBase {
}
/**
+ * Assert that the index exists.
+ *
+ * @param indexName The index name to be checked. This name is case sensitive.
+ * @throws ServiceException
+ */
+ public void assertIndexExists(String indexName) throws ServiceException {
+ assertTrue(client.existIndex(indexName));
+ }
+
+ /**
* Assert that the table does not exist.
*
* @param tableName The table name to be checked. This name is case sensitive.
@@ -390,6 +405,10 @@ public class QueryTestCaseBase {
assertTrue(!client.existTable(tableName));
}
+ public void assertIndexNotExists(String indexName) throws ServiceException {
+ assertFalse(client.existIndex(indexName));
+ }
+
public void assertColumnExists(String tableName,String columnName) throws ServiceException {
TableDesc tableDesc = fetchTableMetaData(tableName);
assertTrue(tableDesc.getSchema().containsByName(columnName));
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index 2ca85cc..6a6d07c 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -54,7 +54,6 @@ import java.util.*;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
public class TestLogicalPlanner {
private static TajoTestingCluster util;
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 920c932..6c36040 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -34,7 +34,6 @@ import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
@@ -798,7 +797,7 @@ public class TestPhysicalPlanner {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex");
- Path indexPath = StorageUtil.concatPath(conf.getVar(ConfVars.ROOT_DIR), "default/idx_employee");
+ Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee");
if (sm.getFileSystem().exists(indexPath)) {
sm.getFileSystem().delete(indexPath, true);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c9b8f511/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
index 2a66909..4714bd6 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
@@ -18,6 +18,7 @@
package org.apache.tajo.engine.query;
+import com.google.protobuf.ServiceException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.IntegrationTest;
@@ -29,7 +30,9 @@ import org.junit.experimental.categories.Category;
import java.io.IOException;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
@Category(IntegrationTest.class)
public class TestCreateIndex extends QueryTestCaseBase {
@@ -38,35 +41,50 @@ public class TestCreateIndex extends QueryTestCaseBase {
super(TajoConstants.DEFAULT_DATABASE_NAME);
}
- private static void assertIndexExist(String indexName) throws IOException {
+ private void checkIndexExist(String indexName) throws IOException, ServiceException {
Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName);
FileSystem fs = indexPath.getFileSystem(conf);
assertTrue(fs.exists(indexPath));
assertEquals(2, fs.listStatus(indexPath).length);
- fs.deleteOnExit(indexPath);
+ assertIndexExists(indexName);
+ }
+
+ private void checkIndexNotExist(String indexName) throws IOException, ServiceException {
+ Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName);
+ FileSystem fs = indexPath.getFileSystem(conf);
+ assertFalse(fs.exists(indexPath));
+ assertIndexNotExists(indexName);
}
@Test
public final void testCreateIndex() throws Exception {
executeQuery();
- assertIndexExist("l_orderkey_idx");
+ checkIndexExist("l_orderkey_idx");
+ executeString("drop index l_orderkey_idx");
+ checkIndexNotExist("l_orderkey_idx");
}
@Test
public final void testCreateIndexOnMultiAttrs() throws Exception {
executeQuery();
- assertIndexExist("l_orderkey_partkey_idx");
+ checkIndexExist("l_orderkey_partkey_idx");
+ executeString("drop index l_orderkey_partkey_idx");
+ checkIndexNotExist("l_orderkey_partkey_idx");
}
@Test
public final void testCreateIndexWithCondition() throws Exception {
executeQuery();
- assertIndexExist("l_orderkey_partkey_lt10_idx");
+ checkIndexExist("l_orderkey_partkey_lt10_idx");
+ executeString("drop index l_orderkey_partkey_lt10_idx");
+ checkIndexNotExist("l_orderkey_partkey_lt10_idx");
}
@Test
public final void testCreateIndexOnExpression() throws Exception {
executeQuery();
- assertIndexExist("l_orderkey_100_lt10_idx");
+ checkIndexExist("l_orderkey_100_lt10_idx");
+ executeString("drop index l_orderkey_100_lt10_idx");
+ checkIndexNotExist("l_orderkey_100_lt10_idx");
}
}
[5/7] git commit: Merge branch 'master' of
http://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/7cd57069
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/7cd57069
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/7cd57069
Branch: refs/heads/index_support
Commit: 7cd570697ec02af3750a4a690387da9d19b55ccb
Parents: 334ea2e 2a6b38e
Author: Jihoon Son <ji...@apache.org>
Authored: Wed Jul 23 09:05:35 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Jul 23 09:05:35 2014 +0900
----------------------------------------------------------------------
CHANGES | 7 +
.../java/org/apache/tajo/catalog/Schema.java | 19 +-
.../main/java/org/apache/tajo/util/Bytes.java | 1184 ++++++++----------
.../java/org/apache/tajo/util/BytesUtils.java | 182 +++
.../java/org/apache/tajo/util/NumberUtil.java | 348 ++++-
.../java/org/apache/tajo/util/TestBytes.java | 111 --
.../org/apache/tajo/util/TestNumberUtil.java | 47 +-
.../org/apache/tajo/util/TestStringUtil.java | 40 +-
.../tajo/engine/planner/ExprAnnotator.java | 26 +-
.../tajo/engine/planner/ExprNormalizer.java | 13 +-
.../apache/tajo/engine/planner/LogicalPlan.java | 256 +---
.../engine/planner/LogicalPlanPreprocessor.java | 32 +-
.../tajo/engine/planner/LogicalPlanner.java | 49 +-
.../planner/logical/TableSubQueryNode.java | 1 -
.../planner/nameresolver/NameResolver.java | 291 +++++
.../planner/nameresolver/NameResolvingMode.java | 80 ++
.../planner/nameresolver/ResolverByLegacy.java | 126 ++
.../planner/nameresolver/ResolverByRels.java | 38 +
.../nameresolver/ResolverByRelsAndSubExprs.java | 42 +
.../nameresolver/ResolverBySubExprsAndRels.java | 42 +
.../apache/tajo/engine/eval/ExprTestBase.java | 6 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 4 +-
.../tajo/engine/query/TestCaseByCases.java | 7 +
.../tajo/engine/query/TestSelectQuery.java | 27 +
.../TestCaseByCases/testTAJO917Case1.sql | 13 +
.../TestSelectQuery/testNonQualifiedNames.sql | 1 +
...testSelectColumnAliasExistingInRelation1.sql | 1 +
...testSelectColumnAliasExistingInRelation2.sql | 1 +
.../TestCaseByCases/testTAJO917Case1.result | 7 +
.../testNonQualifiedNames.result | 7 +
...tSelectColumnAliasExistingInRelation1.result | 4 +
...tSelectColumnAliasExistingInRelation2.result | 7 +
.../java/org/apache/tajo/storage/CSVFile.java | 6 +-
.../java/org/apache/tajo/storage/RowFile.java | 3 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 40 +-
.../org/apache/tajo/storage/StorageUtil.java | 41 +
.../storage/TextSerializerDeserializer.java | 7 +-
.../apache/tajo/storage/index/bst/BSTIndex.java | 16 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 5 +-
.../sequencefile/SequenceFileAppender.java | 8 +-
.../sequencefile/SequenceFileScanner.java | 4 +-
.../apache/tajo/storage/v2/CSVFileScanner.java | 8 +-
.../java/org/apache/tajo/storage/v2/RCFile.java | 6 +-
.../org/apache/tajo/storage/TestLazyTuple.java | 6 +-
44 files changed, 2093 insertions(+), 1076 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/7cd57069/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/7cd57069/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
[4/7] git commit: TAJO-666: java.nio.BufferOverflowException occurs
when the query includes an order by clause on a TEXT column. (Mai Hai Thanh
via jihoon)
Posted by ji...@apache.org.
TAJO-666: java.nio.BufferOverflowException occurs when the query includes an order by clause on a TEXT column. (Mai Hai Thanh via jihoon)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/2a6b38e8
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/2a6b38e8
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/2a6b38e8
Branch: refs/heads/index_support
Commit: 2a6b38e8460a048ce09d18aefb8786627473fbf7
Parents: 862435b
Author: Jihoon Son <ji...@apache.org>
Authored: Tue Jul 22 14:25:03 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Tue Jul 22 14:25:03 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../org/apache/tajo/storage/RowStoreUtil.java | 40 +++++++++++++++++++-
2 files changed, 42 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2a6b38e8/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8208fd7..8a1aae6 100644
--- a/CHANGES
+++ b/CHANGES
@@ -97,6 +97,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-666: java.nio.BufferOverflowException occurs when the query includes an order by
+ clause on a TEXT column. (Mai Hai Thanh via jihoon)
+
TAJO-939: Refactoring the column resolver in LogicalPlan. (hyunsik)
TAJO-965: Upgrade Bytes class and move some methods to others. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/2a6b38e8/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 5140a63..dc453a9 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -174,7 +174,7 @@ public class RowStoreUtil {
}
public byte [] toBytes(Tuple tuple) {
nullFlags.clear();
- int size = 4096; // 4kb
+ int size = estimateTupleDataSize(tuple);
ByteBuffer bb = ByteBuffer.allocate(size + headerSize);
bb.position(headerSize);
Column col;
@@ -236,6 +236,44 @@ public class RowStoreUtil {
return buf;
}
+ // Note that, NULL values are treated separately
+ private int estimateTupleDataSize(Tuple tuple) {
+ int size = 0;
+ Column col;
+
+ for (int i = 0; i < schema.size(); i++) {
+ if (tuple.isNull(i)) {
+ continue;
+ }
+
+ col = schema.getColumn(i);
+ switch (col.getDataType().getType()) {
+ case BOOLEAN:
+ case BIT:
+ case CHAR: size += 1; break;
+ case INT2: size += 2; break;
+ case DATE:
+ case INT4:
+ case FLOAT4: size += 4; break;
+ case TIME:
+ case TIMESTAMP:
+ case INT8:
+ case FLOAT8: size += 8; break;
+ case INTERVAL: size += 12; break;
+ case TEXT:
+ case BLOB: size += (4 + tuple.get(i).asByteArray().length); break;
+ case INET4:
+ case INET6: size += tuple.get(i).asByteArray().length; break;
+ default:
+ size += 4;
+ }
+ }
+
+ size += 100; // optimistic reservation
+
+ return size;
+ }
+
public Schema getSchema() {
return schema;
}