You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ji...@apache.org on 2015/10/29 05:44:57 UTC
[06/15] incubator-asterixdb git commit: ASTERIXDB-1102: VarSize
Encoding to store length of String and ByteArray
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryBase64Printer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryBase64Printer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryBase64Printer.java
index fd1d595..01c2202 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryBase64Printer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryBase64Printer.java
@@ -19,133 +19,35 @@
package org.apache.asterix.dataflow.data.nontagged.printers.adm;
-import org.apache.asterix.dataflow.data.nontagged.serde.ABinarySerializerDeserializer;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.data.IPrinter;
-
import java.io.IOException;
import java.io.PrintStream;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.data.IPrinter;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.util.bytes.Base64Printer;
+
public class ABinaryBase64Printer implements IPrinter {
private ABinaryBase64Printer() {
}
public static final ABinaryBase64Printer INSTANCE = new ABinaryBase64Printer();
- @Override public void init() throws AlgebricksException {
-
+ @Override
+ public void init() throws AlgebricksException {
}
- @Override public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
- int validLength = ABinarySerializerDeserializer.getLength(b, s + 1);
- int start = s + 1 + ABinarySerializerDeserializer.SIZE_OF_LENGTH;
+ @Override
+ public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+ int validLength = ByteArrayPointable.getContentLength(b, s + 1);
+ int start = s + 1 + ByteArrayPointable.getNumberBytesToStoreMeta(validLength);
try {
ps.print("base64(\"");
- printBase64Binary(b, start, validLength, ps);
+ Base64Printer.printBase64Binary(b, start, validLength, ps);
ps.print("\")");
} catch (IOException e) {
throw new AlgebricksException(e);
}
}
- /**
- * Encodes a byte array into a {@code Appendable} stream by doing base64 encoding.
- *
- * @return the same stream in purpose of chained process.
- */
- public static Appendable printBase64Binary(byte[] input, int offset, int len, Appendable appendable)
- throws IOException {
- // encode elements until only 1 or 2 elements are left to encode
- int remaining = len;
- int i;
- for (i = offset; remaining >= 3; remaining -= 3, i += 3) {
- appendable.append(encode(input[i] >> 2));
- appendable.append(encode(
- ((input[i] & 0x3) << 4)
- | ((input[i + 1] >> 4) & 0xF)));
- appendable.append(encode(
- ((input[i + 1] & 0xF) << 2)
- | ((input[i + 2] >> 6) & 0x3)));
- appendable.append(encode(input[i + 2] & 0x3F));
- }
- // encode when exactly 1 element (left) to encode
- if (remaining == 1) {
- appendable.append(encode(input[i] >> 2));
- appendable.append(encode(((input[i]) & 0x3) << 4));
- appendable.append('=');
- appendable.append('=');
- }
- // encode when exactly 2 elements (left) to encode
- if (remaining == 2) {
- appendable.append(encode(input[i] >> 2));
- appendable.append(encode(((input[i] & 0x3) << 4)
- | ((input[i + 1] >> 4) & 0xF)));
- appendable.append(encode((input[i + 1] & 0xF) << 2));
- appendable.append('=');
- }
- return appendable;
- }
-
- /**
- * Encodes a byte array into a char array by doing base64 encoding.
- * The caller must supply a big enough buffer.
- *
- * @return the value of {@code ptr+((len+2)/3)*4}, which is the new offset
- * in the output buffer where the further bytes should be placed.
- */
- public static int printBase64Binary(byte[] input, int offset, int len, char[] buf, int ptr) {
- // encode elements until only 1 or 2 elements are left to encode
- int remaining = len;
- int i;
- for (i = offset; remaining >= 3; remaining -= 3, i += 3) {
- buf[ptr++] = encode(input[i] >> 2);
- buf[ptr++] = encode(
- ((input[i] & 0x3) << 4)
- | ((input[i + 1] >> 4) & 0xF));
- buf[ptr++] = encode(
- ((input[i + 1] & 0xF) << 2)
- | ((input[i + 2] >> 6) & 0x3));
- buf[ptr++] = encode(input[i + 2] & 0x3F);
- }
- // encode when exactly 1 element (left) to encode
- if (remaining == 1) {
- buf[ptr++] = encode(input[i] >> 2);
- buf[ptr++] = encode(((input[i]) & 0x3) << 4);
- buf[ptr++] = '=';
- buf[ptr++] = '=';
- }
- // encode when exactly 2 elements (left) to encode
- if (remaining == 2) {
- buf[ptr++] = encode(input[i] >> 2);
- buf[ptr++] = encode(((input[i] & 0x3) << 4)
- | ((input[i + 1] >> 4) & 0xF));
- buf[ptr++] = encode((input[i + 1] & 0xF) << 2);
- buf[ptr++] = '=';
- }
- return ptr;
- }
-
- private static final char[] encodeMap = initEncodeMap();
-
- private static char[] initEncodeMap() {
- char[] map = new char[64];
- int i;
- for (i = 0; i < 26; i++) {
- map[i] = (char) ('A' + i);
- }
- for (i = 26; i < 52; i++) {
- map[i] = (char) ('a' + (i - 26));
- }
- for (i = 52; i < 62; i++) {
- map[i] = (char) ('0' + (i - 52));
- }
- map[62] = '+';
- map[63] = '/';
-
- return map;
- }
-
- public static char encode(int i) {
- return encodeMap[i & 0x3F];
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinter.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinter.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinter.java
index b9f9750..a997be4 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinter.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinter.java
@@ -19,33 +19,35 @@
package org.apache.asterix.dataflow.data.nontagged.printers.adm;
-import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools;
-import org.apache.asterix.dataflow.data.nontagged.serde.ABinarySerializerDeserializer;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.data.IPrinter;
-
import java.io.IOException;
import java.io.PrintStream;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.data.IPrinter;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.util.bytes.HexPrinter;
+
public class ABinaryHexPrinter implements IPrinter {
private ABinaryHexPrinter() {
}
public static final ABinaryHexPrinter INSTANCE = new ABinaryHexPrinter();
- @Override public void init() throws AlgebricksException {
-
+ @Override
+ public void init() throws AlgebricksException {
}
- @Override public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
- int validLength = ABinarySerializerDeserializer.getLength(b, s + 1);
- int start = s + 1 + ABinarySerializerDeserializer.SIZE_OF_LENGTH;
+ @Override
+ public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+ int validLength = ByteArrayPointable.getContentLength(b, s + 1);
+ int start = s + 1 + ByteArrayPointable.getNumberBytesToStoreMeta(validLength);
try {
ps.print("hex(\"");
- PrintTools.printHexString(b, start, validLength, ps);
+ HexPrinter.printHexString(b, start, validLength, ps);
ps.print("\")");
} catch (IOException e) {
throw new AlgebricksException(e);
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16Printer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16Printer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16Printer.java
index 9076750..6ac3dd1 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16Printer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16Printer.java
@@ -32,20 +32,6 @@ import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
public class AInt16Printer implements IPrinter {
private static final String SUFFIX_STRING = "i16";
- private static byte[] _suffix;
- private static int _suffix_count;
- static {
- ByteArrayAccessibleOutputStream interm = new ByteArrayAccessibleOutputStream();
- DataOutput dout = new DataOutputStream(interm);
- try {
- dout.writeUTF(SUFFIX_STRING);
- interm.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- _suffix = interm.getByteArray();
- _suffix_count = interm.size();
- }
public static final AInt16Printer INSTANCE = new AInt16Printer();
@@ -59,7 +45,7 @@ public class AInt16Printer implements IPrinter {
short i = AInt16SerializerDeserializer.getShort(b, s + 1);
try {
WriteValueTools.writeInt(i, ps);
- WriteValueTools.writeUTF8StringNoQuotes(_suffix, 0, _suffix_count, ps);
+ WriteValueTools.writeUTF8StringNoQuotes(SUFFIX_STRING, ps);
} catch (IOException e) {
throw new AlgebricksException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt32Printer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt32Printer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt32Printer.java
index 036d16d..240f68c 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt32Printer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt32Printer.java
@@ -32,20 +32,6 @@ import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
public class AInt32Printer implements IPrinter {
private static final String SUFFIX_STRING = "i32";
- private static byte[] _suffix;
- private static int _suffix_count;
- static {
- ByteArrayAccessibleOutputStream interm = new ByteArrayAccessibleOutputStream();
- DataOutput dout = new DataOutputStream(interm);
- try {
- dout.writeUTF(SUFFIX_STRING);
- interm.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- _suffix = interm.getByteArray();
- _suffix_count = interm.size();
- }
public static final AInt32Printer INSTANCE = new AInt32Printer();
@@ -58,7 +44,7 @@ public class AInt32Printer implements IPrinter {
int d = AInt32SerializerDeserializer.getInt(b, s + 1);
try {
WriteValueTools.writeInt(d, ps);
- WriteValueTools.writeUTF8StringNoQuotes(_suffix, 0, _suffix_count, ps);
+ WriteValueTools.writeUTF8StringNoQuotes(SUFFIX_STRING, ps);
} catch (IOException e) {
throw new AlgebricksException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt8Printer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt8Printer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt8Printer.java
index 38eb91e..b74a177 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt8Printer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt8Printer.java
@@ -32,20 +32,6 @@ import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
public class AInt8Printer implements IPrinter {
private static final String SUFFIX_STRING = "i8";
- private static byte[] _suffix;
- private static int _suffix_count;
- static {
- ByteArrayAccessibleOutputStream interm = new ByteArrayAccessibleOutputStream();
- DataOutput dout = new DataOutputStream(interm);
- try {
- dout.writeUTF(SUFFIX_STRING);
- interm.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- _suffix = interm.getByteArray();
- _suffix_count = interm.size();
- }
public static final AInt8Printer INSTANCE = new AInt8Printer();
@@ -59,7 +45,7 @@ public class AInt8Printer implements IPrinter {
byte o = AInt8SerializerDeserializer.getByte(b, s + 1);
try {
WriteValueTools.writeInt(o, ps);
- WriteValueTools.writeUTF8StringNoQuotes(_suffix, 0, _suffix_count, ps);
+ WriteValueTools.writeUTF8StringNoQuotes(SUFFIX_STRING, ps);
} catch (IOException e) {
throw new AlgebricksException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ABinaryHexPrinter.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ABinaryHexPrinter.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ABinaryHexPrinter.java
index 8cf5e3e..f298af6 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ABinaryHexPrinter.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/csv/ABinaryHexPrinter.java
@@ -23,6 +23,8 @@ import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools;
import org.apache.asterix.dataflow.data.nontagged.serde.ABinarySerializerDeserializer;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.data.IPrinter;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.util.bytes.HexPrinter;
import java.io.IOException;
import java.io.PrintStream;
@@ -38,10 +40,10 @@ public class ABinaryHexPrinter implements IPrinter {
}
@Override public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
- int validLength = ABinarySerializerDeserializer.getLength(b, s + 1);
- int start = s + 1 + ABinarySerializerDeserializer.SIZE_OF_LENGTH;
+ int validLength = ByteArrayPointable.getContentLength(b, s + 1);
+ int start = s + 1 + ByteArrayPointable.getNumberBytesToStoreMeta(validLength);
try {
- PrintTools.printHexString(b, start, validLength, ps);
+ HexPrinter.printHexString(b, start, validLength, ps);
} catch (IOException e) {
throw new AlgebricksException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/clean/ABinaryHexPrinter.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/clean/ABinaryHexPrinter.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/clean/ABinaryHexPrinter.java
index 97f5203..01647c6 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/clean/ABinaryHexPrinter.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/clean/ABinaryHexPrinter.java
@@ -23,6 +23,8 @@ import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools;
import org.apache.asterix.dataflow.data.nontagged.serde.ABinarySerializerDeserializer;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.data.IPrinter;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.util.bytes.HexPrinter;
import java.io.IOException;
import java.io.PrintStream;
@@ -38,11 +40,12 @@ public class ABinaryHexPrinter implements IPrinter {
}
@Override public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
- int validLength = ABinarySerializerDeserializer.getLength(b, s + 1);
- int start = s + 1 + ABinarySerializerDeserializer.SIZE_OF_LENGTH;
+
+ int validLength = ByteArrayPointable.getContentLength(b, s + 1);
+ int start = s + 1 + ByteArrayPointable.getNumberBytesToStoreMeta(validLength);
try {
ps.print("\"");
- PrintTools.printHexString(b, start, validLength, ps);
+ HexPrinter.printHexString(b, start, validLength, ps);
ps.print("\"");
} catch (IOException e) {
throw new AlgebricksException(e);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/lossless/ABinaryHexPrinter.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/lossless/ABinaryHexPrinter.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/lossless/ABinaryHexPrinter.java
index af84be7..4698c49 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/lossless/ABinaryHexPrinter.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/json/lossless/ABinaryHexPrinter.java
@@ -23,6 +23,8 @@ import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools;
import org.apache.asterix.dataflow.data.nontagged.serde.ABinarySerializerDeserializer;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.data.IPrinter;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.util.bytes.HexPrinter;
import java.io.IOException;
import java.io.PrintStream;
@@ -38,11 +40,11 @@ public class ABinaryHexPrinter implements IPrinter {
}
@Override public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
- int validLength = ABinarySerializerDeserializer.getLength(b, s + 1);
- int start = s + 1 + ABinarySerializerDeserializer.SIZE_OF_LENGTH;
+ int validLength = ByteArrayPointable.getContentLength(b, s + 1);
+ int start = s + 1 + ByteArrayPointable.getNumberBytesToStoreMeta(validLength);
try {
ps.print("\"");
- PrintTools.printHexString(b, start, validLength, ps);
+ HexPrinter.printHexString(b, start, validLength, ps);
ps.print("\"");
} catch (IOException e) {
throw new AlgebricksException(e);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ABinarySerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ABinarySerializerDeserializer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ABinarySerializerDeserializer.java
index 4014b6e..3bde3b3 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ABinarySerializerDeserializer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ABinarySerializerDeserializer.java
@@ -19,41 +19,30 @@
package org.apache.asterix.dataflow.data.nontagged.serde;
+import java.io.DataInput;
+import java.io.DataOutput;
+
import org.apache.asterix.om.base.ABinary;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
import org.apache.hyracks.dataflow.common.data.marshalling.ByteArraySerializerDeserializer;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-public class ABinarySerializerDeserializer implements ISerializerDeserializer<ABinary>{
+public class ABinarySerializerDeserializer implements ISerializerDeserializer<ABinary> {
private static final long serialVersionUID = 1L;
- public static final int SIZE_OF_LENGTH = ByteArrayPointable.SIZE_OF_LENGTH;
- public static final ABinarySerializerDeserializer INSTANCE = new ABinarySerializerDeserializer();
+ public ABinarySerializerDeserializer() {}
- private ABinarySerializerDeserializer(){}
+ private ByteArraySerializerDeserializer byteArrayPtrSerDer = new ByteArraySerializerDeserializer();
@Override
public ABinary deserialize(DataInput in) throws HyracksDataException {
- return new ABinary(ByteArraySerializerDeserializer.INSTANCE.deserialize(in));
+ return new ABinary(byteArrayPtrSerDer.deserialize(in));
}
@Override
public void serialize(ABinary binary, DataOutput out) throws HyracksDataException {
- ByteArraySerializerDeserializer.INSTANCE.serialize(binary.getBytes(), binary.getStart(),
- binary.getLength() , out);
- }
-
- public static int getLength(byte [] bytes, int offset){
- return ByteArrayPointable.getLength(bytes,offset);
+ byteArrayPtrSerDer.serialize(binary.getBytes(), binary.getStart(), binary.getLength(), out);
}
- public static void putLength(int length, byte [] bytes, int offset){
- ByteArrayPointable.putLength(length, bytes, offset);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AObjectSerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AObjectSerializerDeserializer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AObjectSerializerDeserializer.java
index 80ac99c..ca2557a 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AObjectSerializerDeserializer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AObjectSerializerDeserializer.java
@@ -57,9 +57,18 @@ public class AObjectSerializerDeserializer implements ISerializerDeserializer<IA
private static final long serialVersionUID = 1L;
- public static final AObjectSerializerDeserializer INSTANCE = new AObjectSerializerDeserializer();
+ private final AStringSerializerDeserializer aStringSerDer = new AStringSerializerDeserializer();
+ private final ABinarySerializerDeserializer aBinarySerDer = new ABinarySerializerDeserializer();
+ private ARecordSerializerDeserializer aRecordSerDer = null;
- private AObjectSerializerDeserializer() {
+ private ARecordSerializerDeserializer getRecordSerDer() {
+ if (aRecordSerDer == null) {
+ aRecordSerDer = ARecordSerializerDeserializer.CREATE_SCHEMALESS_INSTANCE();
+ }
+ return aRecordSerDer;
+ }
+
+ public AObjectSerializerDeserializer() {
}
@Override
@@ -91,10 +100,10 @@ public class AObjectSerializerDeserializer implements ISerializerDeserializer<IA
return ADoubleSerializerDeserializer.INSTANCE.deserialize(in);
}
case STRING: {
- return AStringSerializerDeserializer.INSTANCE.deserialize(in);
+ return aStringSerDer.deserialize(in);
}
case BINARY: {
- return ABinarySerializerDeserializer.INSTANCE.deserialize(in);
+ return aBinarySerDer.deserialize(in);
}
case DATE: {
return ADateSerializerDeserializer.INSTANCE.deserialize(in);
@@ -136,7 +145,7 @@ public class AObjectSerializerDeserializer implements ISerializerDeserializer<IA
return ACircleSerializerDeserializer.INSTANCE.deserialize(in);
}
case RECORD: {
- return ARecordSerializerDeserializer.SCHEMALESS_INSTANCE.deserialize(in);
+ return getRecordSerDer().deserialize(in);
}
case ORDEREDLIST: {
return AOrderedListSerializerDeserializer.SCHEMALESS_INSTANCE.deserialize(in);
@@ -196,11 +205,11 @@ public class AObjectSerializerDeserializer implements ISerializerDeserializer<IA
break;
}
case STRING: {
- AStringSerializerDeserializer.INSTANCE.serialize((AString) instance, out);
+ aStringSerDer.serialize((AString) instance, out);
break;
}
case BINARY: {
- ABinarySerializerDeserializer.INSTANCE.serialize((ABinary) instance, out);
+ aBinarySerDer.serialize((ABinary) instance, out);
}
case DATE: {
ADateSerializerDeserializer.INSTANCE.serialize((ADate) instance, out);
@@ -247,7 +256,7 @@ public class AObjectSerializerDeserializer implements ISerializerDeserializer<IA
break;
}
case RECORD: {
- ARecordSerializerDeserializer.SCHEMALESS_INSTANCE.serialize((ARecord) instance, out);
+ getRecordSerDer().serialize((ARecord) instance, out);
break;
}
case ORDEREDLIST: {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
index a0de782..801907c 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
@@ -42,13 +42,24 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.string.UTF8StringUtil;
public class ARecordSerializerDeserializer implements ISerializerDeserializer<ARecord> {
private static final long serialVersionUID = 1L;
- public static final ARecordSerializerDeserializer SCHEMALESS_INSTANCE = new ARecordSerializerDeserializer();
+ public static final ARecordSerializerDeserializer CREATE_SCHEMALESS_INSTANCE() {
+ return new ARecordSerializerDeserializer();
+ }
+
+ private final AStringSerializerDeserializer aStringSerDer = new AStringSerializerDeserializer();
+ private AObjectSerializerDeserializer aObjSerDer = null;
+ private AObjectSerializerDeserializer getObjSerDer() {
+ if (aObjSerDer == null){
+ aObjSerDer = new AObjectSerializerDeserializer();
+ }
+ return aObjSerDer;
+ }
private ARecordType recordType;
private int numberOfSchemaFields = 0;
@@ -139,8 +150,8 @@ public class ARecordSerializerDeserializer implements ISerializerDeserializer<AR
in.readInt();
}
for (int i = 0; i < numberOfOpenFields; i++) {
- fieldNames[i] = AStringSerializerDeserializer.INSTANCE.deserialize(in).getStringValue();
- openFields[i] = AObjectSerializerDeserializer.INSTANCE.deserialize(in);
+ fieldNames[i] = aStringSerDer.deserialize(in).getStringValue();
+ openFields[i] = getObjSerDer().deserialize(in);
fieldTypes[i] = openFields[i].getType();
}
ARecordType openPartRecType = new ARecordType(null, fieldNames, fieldTypes, true);
@@ -288,7 +299,8 @@ public class ARecordSerializerDeserializer implements ISerializerDeserializer<AR
return -1; // this record does not have an open part
int numberOfOpenField = AInt32SerializerDeserializer.getInt(serRecord, openPartOffset);
- int utflength = UTF8StringPointable.getUTFLength(fieldName, 1);
+ int fieldUtflength = UTF8StringUtil.getUTFLength(fieldName, 1);
+ int fieldUtfMetaLen = UTF8StringUtil.getNumBytesToStoreLength(fieldUtflength);
IBinaryHashFunction utf8HashFunction = AqlBinaryHashFunctionFactoryProvider.UTF8STRING_POINTABLE_INSTANCE
.createBinaryHashFunction();
@@ -296,34 +308,33 @@ public class ARecordSerializerDeserializer implements ISerializerDeserializer<AR
IBinaryComparator utf8BinaryComparator = AqlBinaryComparatorFactoryProvider.UTF8STRING_POINTABLE_INSTANCE
.createBinaryComparator();
- int fieldNameHashCode = utf8HashFunction.hash(fieldName, 1, utflength);
+ int fieldNameHashCode = utf8HashFunction.hash(fieldName, 1, fieldUtflength + fieldUtfMetaLen);
int offset = openPartOffset + 4;
int fieldOffset = -1;
- short recordFieldNameLength = 0;
int mid = 0;
int high = numberOfOpenField - 1;
int low = 0;
while (low <= high) {
mid = (high + low) / 2;
- // 8 = hash code (4) + offset to the (name + tag + value ) of the
- // field (4).
+ // 8 = hash code (4) + offset to the (name + tag + value ) of the field (4).
int h = AInt32SerializerDeserializer.getInt(serRecord, offset + (8 * mid));
if (h == fieldNameHashCode) {
fieldOffset = AInt32SerializerDeserializer.getInt(serRecord, offset + (8 * mid) + 4);
- recordFieldNameLength = AInt16SerializerDeserializer.getShort(serRecord, fieldOffset);
+ // the utf8 comparator do not require to put the precise length, we can just pass a estimated limit.
if (utf8BinaryComparator
- .compare(serRecord, fieldOffset, recordFieldNameLength, fieldName, 1, utflength) == 0)
- return fieldOffset + 2 + utflength;
+ .compare(serRecord, fieldOffset, serRecord.length,
+ fieldName, 1, fieldUtflength + fieldUtfMetaLen) == 0)
+ // since they are equal, we can directly use the meta length and the utf length.
+ return fieldOffset + fieldUtfMetaLen + fieldUtflength;
else { // this else part has not been tested yet
for (int j = mid + 1; j < numberOfOpenField; j++) {
h = AInt32SerializerDeserializer.getInt(serRecord, offset + (8 * j));
if (h == fieldNameHashCode) {
fieldOffset = AInt32SerializerDeserializer.getInt(serRecord, offset + (8 * j) + 4);
- recordFieldNameLength = AInt16SerializerDeserializer.getShort(serRecord, fieldOffset);
- if (utf8BinaryComparator.compare(serRecord, fieldOffset, recordFieldNameLength, fieldName,
- 1, utflength) == 0)
- return fieldOffset + 2 + utflength;
+ if (utf8BinaryComparator.compare(serRecord, fieldOffset, serRecord.length, fieldName,
+ 1, fieldUtflength) == 0)
+ return fieldOffset + fieldUtfMetaLen + fieldUtflength;
} else
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
index 53b3f28..272b36f 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AStringSerializerDeserializer.java
@@ -31,15 +31,14 @@ public class AStringSerializerDeserializer implements ISerializerDeserializer<AS
private static final long serialVersionUID = 1L;
- public static final AStringSerializerDeserializer INSTANCE = new AStringSerializerDeserializer();
+ private final UTF8StringSerializerDeserializer utf8SerDer = new UTF8StringSerializerDeserializer();
- private AStringSerializerDeserializer() {
- }
+ public AStringSerializerDeserializer() {}
@Override
public AString deserialize(DataInput in) throws HyracksDataException {
try {
- return new AString(UTF8StringSerializerDeserializer.INSTANCE.deserialize(in));
+ return new AString(utf8SerDer.deserialize(in));
} catch (IOException e) {
throw new HyracksDataException(e);
}
@@ -48,7 +47,7 @@ public class AStringSerializerDeserializer implements ISerializerDeserializer<AS
@Override
public void serialize(AString instance, DataOutput out) throws HyracksDataException {
try {
- UTF8StringSerializerDeserializer.INSTANCE.serialize(instance.getStringValue(), out);
+ utf8SerDer.serialize(instance.getStringValue(), out);
} catch (IOException e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUUIDStringSerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUUIDStringSerializerDeserializer.java b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUUIDStringSerializerDeserializer.java
index 3c4df08..63c29a8 100644
--- a/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUUIDStringSerializerDeserializer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/AUUIDStringSerializerDeserializer.java
@@ -42,7 +42,7 @@ public class AUUIDStringSerializerDeserializer implements ISerializerDeserialize
@Override
public AUUID deserialize(DataInput in) throws HyracksDataException {
try {
- return AUUID.fromString(new String(UTF8StringSerializerDeserializer.INSTANCE.deserialize(in)));
+ return AUUID.fromString(new String(new UTF8StringSerializerDeserializer().deserialize(in)));
} catch (IOException e) {
throw new HyracksDataException(e);
}
@@ -52,7 +52,7 @@ public class AUUIDStringSerializerDeserializer implements ISerializerDeserialize
@Override
public void serialize(AUUID instance, DataOutput out) throws HyracksDataException {
try {
- UTF8StringSerializerDeserializer.INSTANCE.serialize(instance.toStringLiteralOnly(), out);
+ new UTF8StringSerializerDeserializer().serialize(instance.toStringLiteralOnly(), out);
} catch (IOException e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java
index 8e1b2b3..8f43a17 100644
--- a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java
+++ b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java
@@ -49,6 +49,7 @@ import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.data.std.primitive.LongPointable;
import org.apache.hyracks.data.std.primitive.RawUTF8StringPointable;
import org.apache.hyracks.data.std.primitive.ShortPointable;
+import org.apache.hyracks.data.std.primitive.UTF8StringLowercasePointable;
public class AqlBinaryComparatorFactoryProvider implements IBinaryComparatorFactoryProvider, Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java
index 054047c..8cfe51e 100644
--- a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java
+++ b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.data.std.primitive.DoublePointable;
import org.apache.hyracks.data.std.primitive.FloatPointable;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.data.std.primitive.RawUTF8StringPointable;
+import org.apache.hyracks.data.std.primitive.UTF8StringLowercasePointable;
public class AqlBinaryHashFunctionFactoryProvider implements IBinaryHashFunctionFactoryProvider, Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryTokenizerFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryTokenizerFactoryProvider.java b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryTokenizerFactoryProvider.java
index 60e9101..e73f769 100644
--- a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryTokenizerFactoryProvider.java
+++ b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlBinaryTokenizerFactoryProvider.java
@@ -26,6 +26,7 @@ import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizerFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8WordTokenFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizerFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8NGramTokenFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8WordTokenFactory;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
index c3996b4..910f633 100644
--- a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
+++ b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/AqlSerializerDeserializerProvider.java
@@ -84,7 +84,7 @@ public class AqlSerializerDeserializerProvider implements ISerializerDeserialize
switch (aqlType.getTypeTag()) {
case ANY:
case UNION: { // we could do smth better for nullable fields
- return AObjectSerializerDeserializer.INSTANCE;
+ return new AObjectSerializerDeserializer();
}
default: {
return addTag(getNonTaggedSerializerDeserializer(aqlType), aqlType.getTypeTag());
@@ -132,10 +132,10 @@ public class AqlSerializerDeserializerProvider implements ISerializerDeserialize
return ANullSerializerDeserializer.INSTANCE;
}
case STRING: {
- return AStringSerializerDeserializer.INSTANCE;
+ return new AStringSerializerDeserializer();
}
case BINARY: {
- return ABinarySerializerDeserializer.INSTANCE;
+ return new ABinarySerializerDeserializer();
}
case TIME: {
return ATimeSerializerDeserializer.INSTANCE;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/NGramUTF8StringBinaryTokenizerFactory.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/NGramUTF8StringBinaryTokenizerFactory.java b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/NGramUTF8StringBinaryTokenizerFactory.java
deleted file mode 100644
index 968ff1b..0000000
--- a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/NGramUTF8StringBinaryTokenizerFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.formats.nontagged;
-
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
-
-// TODO: Should be moved into Hyracks with the rest of the tokenizer code.
-public class NGramUTF8StringBinaryTokenizerFactory implements IBinaryTokenizerFactory {
-
- private static final long serialVersionUID = 1L;
- private final int gramLength;
- private final boolean usePrePost;
- private final boolean ignoreTokenCount;
- private final boolean sourceHasTypeTag;
- private final ITokenFactory tokenFactory;
-
- public NGramUTF8StringBinaryTokenizerFactory(int gramLength, boolean usePrePost, boolean ignoreTokenCount,
- boolean sourceHasTypeTag, ITokenFactory tokenFactory) {
- this.gramLength = gramLength;
- this.usePrePost = usePrePost;
- this.ignoreTokenCount = ignoreTokenCount;
- this.sourceHasTypeTag = sourceHasTypeTag;
- this.tokenFactory = tokenFactory;
- }
-
- @Override
- public IBinaryTokenizer createTokenizer() {
- return new NGramUTF8StringBinaryTokenizer(gramLength, usePrePost, ignoreTokenCount, sourceHasTypeTag,
- tokenFactory);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/UTF8StringLowercasePointable.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/UTF8StringLowercasePointable.java b/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/UTF8StringLowercasePointable.java
deleted file mode 100644
index fa84f99..0000000
--- a/asterix-om/src/main/java/org/apache/asterix/formats/nontagged/UTF8StringLowercasePointable.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.formats.nontagged;
-
-import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.data.std.api.AbstractPointable;
-import org.apache.hyracks.data.std.api.IComparable;
-import org.apache.hyracks.data.std.api.IHashable;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.api.IPointableFactory;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-
-// TODO: Perhaps this class should go into hyracks.
-public final class UTF8StringLowercasePointable extends AbstractPointable implements IHashable, IComparable {
- public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public boolean isFixedLength() {
- return false;
- }
-
- @Override
- public int getFixedLength() {
- return 0;
- }
- };
-
- public static final IPointableFactory FACTORY = new IPointableFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IPointable createPointable() {
- return new UTF8StringLowercasePointable();
- }
-
- @Override
- public ITypeTraits getTypeTraits() {
- return TYPE_TRAITS;
- }
- };
-
- public static int getUTFLen(byte[] b, int s) {
- return ((b[s] & 0xff) << 8) + ((b[s + 1] & 0xff) << 0);
- }
-
- @Override
- public int compareTo(IPointable pointer) {
- return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
- }
-
- @Override
- public int compareTo(byte[] bytes, int start, int length) {
- int utflen1 = getUTFLen(this.bytes, this.start);
- int utflen2 = getUTFLen(bytes, start);
-
- int c1 = 0;
- int c2 = 0;
-
- int s1Start = this.start + 2;
- int s2Start = start + 2;
-
- while (c1 < utflen1 && c2 < utflen2) {
- char ch1 = Character.toLowerCase(UTF8StringPointable.charAt(this.bytes, s1Start + c1));
- char ch2 = Character.toLowerCase(UTF8StringPointable.charAt(bytes, s2Start + c2));
-
- if (ch1 != ch2) {
- return ch1 - ch2;
- }
- c1 += UTF8StringPointable.charSize(this.bytes, s1Start + c1);
- c2 += UTF8StringPointable.charSize(bytes, s2Start + c2);
- }
- return utflen1 - utflen2;
- }
-
- @Override
- public int hash() {
- int h = 0;
- int utflen = getUTFLen(bytes, start);
- int sStart = start + 2;
- int c = 0;
-
- while (c < utflen) {
- char ch = Character.toLowerCase(UTF8StringPointable.charAt(bytes, sStart + c));
- h = 31 * h + ch;
- c += UTF8StringPointable.charSize(bytes, sStart + c);
- }
- return h;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/om/base/ABinary.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/base/ABinary.java b/asterix-om/src/main/java/org/apache/asterix/om/base/ABinary.java
index f3e3864..b3ee50e 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/base/ABinary.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/base/ABinary.java
@@ -19,11 +19,9 @@
package org.apache.asterix.om.base;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.dataflow.data.nontagged.serde.ABinarySerializerDeserializer;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.visitors.IOMVisitor;
-import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -57,15 +55,9 @@ public class ABinary implements IAObject {
}
public int getLength() {
- return getByteArrayContentLength() + SIZE_OF_LEADING_LENGTH_FIELD;
+ return length;
}
- public int getByteArrayContentLength() {
- return ABinarySerializerDeserializer.getLength(bytes, start);
- }
-
- public static final int SIZE_OF_LEADING_LENGTH_FIELD = ByteArrayPointable.SIZE_OF_LENGTH;
-
@Override
public void accept(IOMVisitor visitor) throws AsterixException {
visitor.visitABinary(this);
@@ -98,7 +90,7 @@ public class ABinary implements IAObject {
@Override
public int hash() {
- int m = HASH_PREFIX <= getLength() ? HASH_PREFIX : getLength();
+ int m = Math.min(HASH_PREFIX, getLength());
int h = 0;
for (int i = 0; i < m; i++) {
h += 31 * h + bytes[start + i];
@@ -109,10 +101,9 @@ public class ABinary implements IAObject {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- int validLength = getByteArrayContentLength();
- int start = getStart() + SIZE_OF_LEADING_LENGTH_FIELD;
+ int start = getStart();
sb.append("ABinary: [ ");
- for (int i = 0; i < validLength; i++) {
+ for (int i = 0; i < getLength(); i++) {
if (i > 0) {
sb.append(", ");
}
@@ -127,10 +118,9 @@ public class ABinary implements IAObject {
public JSONObject toJSON() throws JSONException {
JSONObject json = new JSONObject();
- int validLength = getByteArrayContentLength();
- int start = getStart() + SIZE_OF_LEADING_LENGTH_FIELD;
+ int start = getStart();
JSONArray byteArray = new JSONArray();
- for (int i = 0; i < validLength; i++) {
+ for (int i = 0; i < getLength(); i++) {
byteArray.put(bytes[start + i]);
}
json.put("ABinary", byteArray);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
index 5f257e0..fdf22c1 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
@@ -24,8 +24,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang3.mutable.Mutable;
-
import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
@@ -112,6 +110,7 @@ import org.apache.asterix.om.typecomputer.impl.UnaryStringInt64OrNullTypeCompute
import org.apache.asterix.om.typecomputer.impl.UnaryStringOrNullTypeComputer;
import org.apache.asterix.om.typecomputer.impl.UnorderedListConstructorResultType;
import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
@@ -238,10 +237,6 @@ public class AsterixBuiltinFunctions {
// String funcitons
public final static FunctionIdentifier STRING_EQUAL = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
"string-equal", 2);
- public final static FunctionIdentifier STRING_START_WITH = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
- "start-with", 2);
- public final static FunctionIdentifier STRING_END_WITH = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
- "end-with", 2);
public final static FunctionIdentifier STRING_MATCHES = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
"matches", 2);
public final static FunctionIdentifier STRING_MATCHES_WITH_FLAG = new FunctionIdentifier(
@@ -256,6 +251,17 @@ public class AsterixBuiltinFunctions {
FunctionConstants.ASTERIX_NS, "replace", 4);
public final static FunctionIdentifier STRING_LENGTH = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
"string-length", 1);
+ public final static FunctionIdentifier STRING_LIKE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "like",
+ 2);
+ public final static FunctionIdentifier STRING_CONTAINS = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
+ "contains",
+ 2);
+ public final static FunctionIdentifier STRING_STARTS_WITH = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
+ "starts-with", 2);
+ public final static FunctionIdentifier STRING_ENDS_WITH = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
+ "ends-with", 2);
+ public final static FunctionIdentifier SUBSTRING = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
+ "substring", 3);
public final static FunctionIdentifier SUBSTRING2 = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
"substring", 2);
public final static FunctionIdentifier SUBSTRING_BEFORE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
@@ -287,16 +293,6 @@ public class AsterixBuiltinFunctions {
public final static FunctionIdentifier MAKE_FIELD_NAME_HANDLE = new FunctionIdentifier(
FunctionConstants.ASTERIX_NS, "make-field-name-handle", 1);
- public final static FunctionIdentifier SUBSTRING = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
- "substring", 3);
- public final static FunctionIdentifier LIKE = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "like", 2);
- public final static FunctionIdentifier CONTAINS = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "contains",
- 2);
- public final static FunctionIdentifier STARTS_WITH = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
- "starts-with", 2);
- public final static FunctionIdentifier ENDS_WITH = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
- "ends-with", 2);
-
public final static FunctionIdentifier AVG = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-avg", 1);
public final static FunctionIdentifier COUNT = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-count", 1);
public final static FunctionIdentifier SUM = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "agg-sum", 1);
@@ -755,7 +751,6 @@ public class AsterixBuiltinFunctions {
addFunction(CIRCLE_CONSTRUCTOR, OptionalACircleTypeComputer.INSTANCE, true);
addPrivateFunction(CONCAT_NON_NULL, ConcatNonNullTypeComputer.INSTANCE, true);
- addFunction(CONTAINS, ABooleanTypeComputer.INSTANCE, true);
addFunction(COUNT, AInt64TypeComputer.INSTANCE, true);
addPrivateFunction(COUNTHASHED_GRAM_TOKENS, OrderedListOfAInt32TypeComputer.INSTANCE, true);
addPrivateFunction(COUNTHASHED_WORD_TOKENS, OrderedListOfAInt32TypeComputer.INSTANCE, true);
@@ -780,7 +775,6 @@ public class AsterixBuiltinFunctions {
addPrivateFunction(EDIT_DISTANCE_LIST_IS_FILTERABLE, ABooleanTypeComputer.INSTANCE, true);
addPrivateFunction(EMBED_TYPE, AnyTypeComputer.INSTANCE, true);
addPrivateFunction(EMPTY_STREAM, ABooleanTypeComputer.INSTANCE, true);
- addFunction(ENDS_WITH, ABooleanTypeComputer.INSTANCE, true);
// add(FIELD_ACCESS, NonTaggedFieldAccessByNameResultType.INSTANCE);
addFunction(FLOAT_CONSTRUCTOR, OptionalAFloatTypeComputer.INSTANCE, true);
@@ -798,7 +792,6 @@ public class AsterixBuiltinFunctions {
addFunction(INT32_CONSTRUCTOR, OptionalAInt32TypeComputer.INSTANCE, true);
addFunction(INT64_CONSTRUCTOR, OptionalAInt64TypeComputer.INSTANCE, true);
addFunction(LEN, OptionalAInt64TypeComputer.INSTANCE, true);
- addFunction(LIKE, BinaryBooleanOrNullFunctionTypeComputer.INSTANCE, true);
addFunction(LINE_CONSTRUCTOR, OptionalALineTypeComputer.INSTANCE, true);
addPrivateFunction(LISTIFY, OrderedListConstructorResultType.INSTANCE, true);
addPrivateFunction(LOCAL_AVG, NonTaggedLocalAvgTypeComputer.INSTANCE, true);
@@ -833,6 +826,8 @@ public class AsterixBuiltinFunctions {
addFunction(FIND_BINARY, OptionalAInt64TypeComputer.INSTANCE, true);
addFunction(FIND_BINARY_FROM, OptionalAInt64TypeComputer.INSTANCE, true);
+ addFunction(STRING_LIKE, BinaryBooleanOrNullFunctionTypeComputer.INSTANCE, true);
+ addFunction(STRING_CONTAINS, ABooleanTypeComputer.INSTANCE, true);
addFunction(STRING_TO_CODEPOINT, OrderedListOfAInt64TypeComputer.INSTANCE, true);
addFunction(CODEPOINT_TO_STRING, AStringTypeComputer.INSTANCE, true);
addFunction(STRING_CONCAT, OptionalAStringTypeComputer.INSTANCE, true);
@@ -840,8 +835,8 @@ public class AsterixBuiltinFunctions {
addFunction(STRING_LENGTH, UnaryStringInt64OrNullTypeComputer.INSTANCE, true);
addFunction(STRING_LOWERCASE, UnaryStringOrNullTypeComputer.INSTANCE, true);
addFunction(STRING_UPPERCASE, UnaryStringOrNullTypeComputer.INSTANCE, true);
- addFunction(STRING_START_WITH, BinaryStringBoolOrNullTypeComputer.INSTANCE, true);
- addFunction(STRING_END_WITH, BinaryStringBoolOrNullTypeComputer.INSTANCE, true);
+ addFunction(STRING_STARTS_WITH, BinaryStringBoolOrNullTypeComputer.INSTANCE, true);
+ addFunction(STRING_ENDS_WITH, BinaryStringBoolOrNullTypeComputer.INSTANCE, true);
addFunction(STRING_MATCHES, BinaryStringBoolOrNullTypeComputer.INSTANCE, true);
addFunction(STRING_MATCHES_WITH_FLAG, TripleStringBoolOrNullTypeComputer.INSTANCE, true);
addFunction(STRING_REPLACE, TripleStringStringOrNullTypeComputer.INSTANCE, true);
@@ -917,7 +912,6 @@ public class AsterixBuiltinFunctions {
addFunction(GET_CIRCLE_RADIUS_ACCESSOR, ADoubleTypeComputer.INSTANCE, true);
addFunction(GET_CIRCLE_CENTER_ACCESSOR, APointTypeComputer.INSTANCE, true);
addFunction(GET_POINTS_LINE_RECTANGLE_POLYGON_ACCESSOR, OrderedListOfAPointTypeComputer.INSTANCE, true);
- addFunction(STARTS_WITH, ABooleanTypeComputer.INSTANCE, true);
addFunction(STRING_CONSTRUCTOR, OptionalAStringTypeComputer.INSTANCE, true);
addFunction(BINARY_HEX_CONSTRUCTOR, OptionalABinaryTypeComputer.INSTANCE, true);
addFunction(BINARY_BASE64_CONSTRUCTOR, OptionalABinaryTypeComputer.INSTANCE, true);
@@ -1229,7 +1223,8 @@ public class AsterixBuiltinFunctions {
|| (includePrivateFunctions && builtinPrivateFunctionsSet.keySet().contains(finfo))) {
return true;
}
- fi = new FunctionIdentifier(AlgebricksBuiltinFunctions.ALGEBRICKS_NS, signature.getName(), signature.getArity());
+ fi = new FunctionIdentifier(AlgebricksBuiltinFunctions.ALGEBRICKS_NS, signature.getName(),
+ signature.getArity());
finfo = getAsterixFunctionInfo(fi);
if (builtinPublicFunctionsSet.keySet().contains(finfo)
|| (includePrivateFunctions && builtinPrivateFunctionsSet.keySet().contains(finfo))) {
@@ -1342,7 +1337,8 @@ public class AsterixBuiltinFunctions {
registeredFunctionsDomain.put(functionInfo, funcDomain);
}
- public static void addPrivateFunction(FunctionIdentifier fi, IResultTypeComputer typeComputer, boolean isFunctional) {
+ public static void addPrivateFunction(FunctionIdentifier fi, IResultTypeComputer typeComputer,
+ boolean isFunctional) {
IFunctionInfo functionInfo = new AsterixFunctionInfo(fi, isFunctional);
builtinPrivateFunctionsSet.put(functionInfo, functionInfo);
funTypeComputer.put(functionInfo, typeComputer);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java b/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java
index 5759c72..823ea3e 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/pointables/ARecordVisitablePointable.java
@@ -38,6 +38,7 @@ import org.apache.asterix.om.util.NonTaggedFormatUtil;
import org.apache.asterix.om.util.ResettableByteArrayOutputStream;
import org.apache.asterix.om.util.container.IObjectFactory;
import org.apache.hyracks.api.dataflow.value.INullWriter;
+import org.apache.hyracks.util.string.UTF8StringWriter;
/**
* This class interprets the binary data representation of a record. One can
@@ -66,6 +67,7 @@ public class ARecordVisitablePointable extends AbstractVisitablePointable {
private final ResettableByteArrayOutputStream typeBos = new ResettableByteArrayOutputStream();
private final DataOutputStream typeDos = new DataOutputStream(typeBos);
+ private final UTF8StringWriter utf8Writer = new UTF8StringWriter();
private final ResettableByteArrayOutputStream dataBos = new ResettableByteArrayOutputStream();
private final DataOutputStream dataDos = new DataOutputStream(dataBos);
@@ -82,7 +84,7 @@ public class ARecordVisitablePointable extends AbstractVisitablePointable {
/**
* private constructor, to prevent constructing it arbitrarily
- *
+ *
* @param inputType
*/
public ARecordVisitablePointable(ARecordType inputType) {
@@ -113,7 +115,7 @@ public class ARecordVisitablePointable extends AbstractVisitablePointable {
// add type name Reference (including a astring type tag)
int nameStart = typeBos.size();
typeDos.writeByte(ATypeTag.STRING.serialize());
- typeDos.writeUTF(fieldNameStrs[i]);
+ utf8Writer.writeUTF8(fieldNameStrs[i], typeDos);
int nameEnd = typeBos.size();
IVisitablePointable typeNameReference = AFlatValuePointable.FACTORY.create(null);
typeNameReference.set(typeBos.getByteArray(), nameStart, nameEnd - nameStart);
@@ -183,7 +185,7 @@ public class ARecordVisitablePointable extends AbstractVisitablePointable {
nullBitMapOffset = s;
offsetArrayOffset = s
+ (this.numberOfSchemaFields % 8 == 0 ? numberOfSchemaFields / 8
- : numberOfSchemaFields / 8 + 1);
+ : numberOfSchemaFields / 8 + 1);
} else {
offsetArrayOffset = s;
}
@@ -280,7 +282,7 @@ public class ARecordVisitablePointable extends AbstractVisitablePointable {
public ARecordType getInputRecordType() {
return inputRecType;
}
-
+
@Override
public <R, T> R accept(IVisitablePointableVisitor<R, T> vistor, T tag) throws AsterixException {
return vistor.visit(this, tag);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java b/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java
index ab80f4f..7e41f3b 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/pointables/cast/ARecordCaster.java
@@ -53,6 +53,7 @@ import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
+import org.apache.hyracks.util.string.UTF8StringWriter;
/**
* This class is to do the runtime type cast for a record. It is ONLY visible to
@@ -95,6 +96,8 @@ class ARecordCaster {
private int[] fieldNamesSortedIndex;
private int[] reqFieldNamesSortedIndex;
+ private final UTF8StringWriter utf8Writer = new UTF8StringWriter();
+
public ARecordCaster() {
try {
bos.reset();
@@ -182,7 +185,7 @@ class ARecordCaster {
// add type name pointable (including a string type tag)
int nameStart = bos.size();
dos.write(ATypeTag.STRING.serialize());
- dos.writeUTF(fname);
+ utf8Writer.writeUTF8(fname, dos);
int nameEnd = bos.size();
IVisitablePointable typeNamePointable = allocator.allocateEmpty();
typeNamePointable.set(bos.getByteArray(), nameStart, nameEnd - nameStart);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java b/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java
index 59c3195..ee91aab 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java
@@ -38,7 +38,8 @@ import org.apache.hyracks.data.std.api.IPointableFactory;
import org.apache.hyracks.data.std.primitive.BooleanPointable;
import org.apache.hyracks.data.std.primitive.BytePointable;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+import org.apache.hyracks.util.string.UTF8StringWriter;
/*
* This class interprets the binary data representation of a record.
@@ -115,12 +116,12 @@ public class ARecordPointable extends AbstractPointable {
private static final int OPEN_FIELD_OFFSET_SIZE = 4;
private static final int OPEN_FIELD_HEADER = OPEN_FIELD_HASH_SIZE + OPEN_FIELD_OFFSET_SIZE;
- private static final int STRING_LENGTH_SIZE = 2;
-
private static boolean isOpen(ARecordType recordType) {
return recordType == null || recordType.isOpen();
}
+ private final UTF8StringWriter utf8Writer = new UTF8StringWriter();
+
public int getSchemeFieldCount(ARecordType recordType) {
return recordType.getFieldNames().length;
}
@@ -231,7 +232,7 @@ public class ARecordPointable extends AbstractPointable {
public void getClosedFieldName(ARecordType recordType, int fieldId, DataOutput dOut) throws IOException {
dOut.writeByte(ATypeTag.STRING.serialize());
- dOut.writeUTF(getClosedFieldName(recordType, fieldId));
+ utf8Writer.writeUTF8(getClosedFieldName(recordType, fieldId), dOut);
}
public byte getClosedFieldTag(ARecordType recordType, int fieldId) {
@@ -301,8 +302,8 @@ public class ARecordPointable extends AbstractPointable {
}
public int getOpenFieldNameSize(ARecordType recordType, int fieldId) {
- return UTF8StringPointable.getUTFLength(bytes, getOpenFieldNameOffset(recordType, fieldId))
- + STRING_LENGTH_SIZE;
+ int utfleng = UTF8StringUtil.getUTFLength(bytes, getOpenFieldNameOffset(recordType, fieldId));
+ return utfleng + UTF8StringUtil.getNumBytesToStoreLength(utfleng);
}
public int getOpenFieldNameOffset(ARecordType recordType, int fieldId) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java b/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
index d7f231b..a1e0739 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
@@ -26,6 +26,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+import org.apache.hyracks.util.string.UTF8StringWriter;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -61,6 +63,8 @@ public class ARecordType extends AbstractComplexType {
private final int serializedFieldNameOffsets[];
private final long hashCodeIndexPairs[];
+ private final UTF8StringSerializerDeserializer utf8SerDer = new UTF8StringSerializerDeserializer();
+
/**
* @param typeName
* the name of the type
@@ -88,6 +92,7 @@ public class ARecordType extends AbstractComplexType {
.createBinaryHashFunction();
ByteArrayAccessibleOutputStream baaos = new ByteArrayAccessibleOutputStream();
DataOutputStream dos = new DataOutputStream(baaos);
+ UTF8StringWriter writer = new UTF8StringWriter();
serializedFieldNameOffsets = new int[fieldNames.length];
hashCodeIndexPairs = new long[fieldNames.length];
@@ -95,7 +100,7 @@ public class ARecordType extends AbstractComplexType {
for (int i = 0; i < fieldNames.length; i++) {
serializedFieldNameOffsets[i] = baaos.size();
try {
- dos.writeUTF(fieldNames[i]);
+ writer.writeUTF8(fieldNames[i], dos);
} catch (IOException e) {
throw new AsterixException(e);
}
@@ -116,7 +121,7 @@ public class ARecordType extends AbstractComplexType {
int j;
for (int i = 0; i < fieldNames.length; i++) {
j = findFieldPosition(serializedFieldNames, serializedFieldNameOffsets[i],
- UTF8StringPointable.getStringLength(serializedFieldNames, serializedFieldNameOffsets[i]));
+ UTF8StringUtil.getStringLength(serializedFieldNames, serializedFieldNameOffsets[i]));
if (j != i) {
throw new AsterixException("Closed fields " + j + " and " + i + " have the same field name \""
+ fieldNames[i] + "\"");
@@ -156,7 +161,7 @@ public class ARecordType extends AbstractComplexType {
while (i < hashCodeIndexPairs.length && (int) (hashCodeIndexPairs[i] >>> 32) == probeFieldHash) {
fIndex = (int) hashCodeIndexPairs[i];
- int cFieldLength = UTF8StringPointable.getStringLength(serializedFieldNames,
+ int cFieldLength = UTF8StringUtil.getStringLength(serializedFieldNames,
serializedFieldNameOffsets[fIndex]);
if (fieldNameComparator.compare(serializedFieldNames, serializedFieldNameOffsets[fIndex], cFieldLength,
bytes, start, length) == 0) {
@@ -221,7 +226,7 @@ public class ARecordType extends AbstractComplexType {
public int findFieldPosition(String fieldName) throws IOException {
ByteArrayAccessibleOutputStream baaos = new ByteArrayAccessibleOutputStream();
DataOutputStream dos = new DataOutputStream(baaos);
- UTF8StringSerializerDeserializer.INSTANCE.serialize(fieldName, dos);
+ utf8SerDer.serialize(fieldName, dos);
return findFieldPosition(baaos.getByteArray(), 0, baaos.getByteArray().length);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/main/java/org/apache/asterix/om/util/NonTaggedFormatUtil.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/util/NonTaggedFormatUtil.java b/asterix-om/src/main/java/org/apache/asterix/om/util/NonTaggedFormatUtil.java
index 9c8f883..6377bcc 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/util/NonTaggedFormatUtil.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/util/NonTaggedFormatUtil.java
@@ -40,7 +40,9 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import org.apache.hyracks.util.string.UTF8StringUtil;
public final class NonTaggedFormatUtil {
@@ -141,11 +143,21 @@ public final class NonTaggedFormatUtil {
else
return AInt16SerializerDeserializer.getShort(serNonTaggedAObject, offset) * 16 + 2;
case STRING:
+ if (tagged) {
+ int len = UTF8StringUtil.getUTFLength(serNonTaggedAObject, offset + 1);
+ return len + UTF8StringUtil.getNumBytesToStoreLength(len);
+ } else {
+ int len = UTF8StringUtil.getUTFLength(serNonTaggedAObject, offset);
+ return len + UTF8StringUtil.getNumBytesToStoreLength(len);
+ }
case BINARY:
- if (tagged)
- return AInt16SerializerDeserializer.getUnsignedShort(serNonTaggedAObject, offset + 1) + 2;
- else
- return AInt16SerializerDeserializer.getUnsignedShort(serNonTaggedAObject, offset) + 2;
+ if (tagged) {
+ int len = ByteArrayPointable.getContentLength(serNonTaggedAObject, offset + 1);
+ return len + ByteArrayPointable.getNumberBytesToStoreMeta(len);
+ } else {
+ int len = ByteArrayPointable.getContentLength(serNonTaggedAObject, offset);
+ return len + ByteArrayPointable.getNumberBytesToStoreMeta(len);
+ }
case RECORD:
if (tagged)
return ARecordSerializerDeserializer.getRecordLength(serNonTaggedAObject, offset + 1) - 1;
@@ -158,11 +170,13 @@ public final class NonTaggedFormatUtil {
return AOrderedListSerializerDeserializer.getOrderedListLength(serNonTaggedAObject, offset) - 1;
case UNORDEREDLIST:
if (tagged)
- return AUnorderedListSerializerDeserializer.getUnorderedListLength(serNonTaggedAObject, offset + 1) - 1;
+ return AUnorderedListSerializerDeserializer.getUnorderedListLength(serNonTaggedAObject, offset + 1)
+ - 1;
else
return AUnorderedListSerializerDeserializer.getUnorderedListLength(serNonTaggedAObject, offset) - 1;
default:
- throw new NotImplementedException("No getLength implemented for a value of this type " + typeTag + " .");
+ throw new NotImplementedException(
+ "No getLength implemented for a value of this type " + typeTag + " .");
}
}
@@ -177,7 +191,8 @@ public final class NonTaggedFormatUtil {
case POINT3D:
return 3;
default:
- throw new NotImplementedException("getNumDimensions is not implemented for this type " + typeTag + " .");
+ throw new NotImplementedException(
+ "getNumDimensions is not implemented for this type " + typeTag + " .");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/printers/ABinaryPrinterTest.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/printers/ABinaryPrinterTest.java b/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/printers/ABinaryPrinterTest.java
index f43890c..881ad50 100644
--- a/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/printers/ABinaryPrinterTest.java
+++ b/asterix-om/src/test/java/org/apache/asterix/dataflow/data/nontagged/printers/ABinaryPrinterTest.java
@@ -19,25 +19,27 @@
package org.apache.asterix.dataflow.data.nontagged.printers;
-import org.apache.asterix.dataflow.data.nontagged.printers.adm.ABinaryHexPrinter;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
-import org.junit.Test;
+import static org.junit.Assert.assertTrue;
-import javax.xml.bind.DatatypeConverter;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.util.Arrays;
-import static org.junit.Assert.*;
+import javax.xml.bind.DatatypeConverter;
+
+import org.apache.asterix.dataflow.data.nontagged.printers.adm.ABinaryHexPrinter;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.junit.Test;
public class ABinaryPrinterTest {
- public static byte[] generateABinaryBytesByStringContent(String content){
- String padding = "000000" + content;
- byte [] bytes = DatatypeConverter.parseHexBinary(padding);
- ByteArrayPointable.putLength(content.length() / 2, bytes, 1);
- return bytes;
+ public static byte[] generateABinaryBytesByStringContent(String content) {
+ byte[] bytes = DatatypeConverter.parseHexBinary(content);
+ ByteArrayPointable ptr = ByteArrayPointable.generatePointableFromPureBytes(bytes);
+ byte[] ret = new byte[ptr.getLength() + 1];
+ System.arraycopy(ptr.getByteArray(), ptr.getStartOffset(), ret, 1, ptr.getLength());
+ return ret;
}
public static void testOneInputString(String input) throws AlgebricksException {
@@ -45,13 +47,13 @@ public class ABinaryPrinterTest {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
- byte [] bytes = generateABinaryBytesByStringContent(input);
+ byte[] bytes = generateABinaryBytesByStringContent(input);
ABinaryHexPrinter.INSTANCE.print(bytes, 0, bytes.length, ps);
String pureHex = baos.toString();
assertTrue(pureHex.startsWith("hex("));
assertTrue(pureHex.endsWith("\")"));
- assertTrue(pureHex.substring(5,pureHex.length()-2).equalsIgnoreCase(input));
+ assertTrue(pureHex.substring(5, pureHex.length() - 2).equalsIgnoreCase(input));
}
@Test
@@ -62,10 +64,11 @@ public class ABinaryPrinterTest {
}
@Test
- public void testSpecialString() throws Exception{
+ public void testLongString() throws Exception {
testOneInputString("");
- char [] chars = new char[(ByteArrayPointable.MAX_LENGTH ) * 2];
- Arrays.fill(chars, 'F');
- testOneInputString(new String(chars));
+ int aVeryLongLength = 65540;
+ char[] aVeryLongHexString = new char[aVeryLongLength * 2];
+ Arrays.fill(aVeryLongHexString, 'F');
+ testOneInputString(new String(aVeryLongHexString));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/742aba85/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/accessors/TemporalYearAccessor.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/accessors/TemporalYearAccessor.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/accessors/TemporalYearAccessor.java
index 9b932ec..0358751 100644
--- a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/accessors/TemporalYearAccessor.java
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/accessors/TemporalYearAccessor.java
@@ -44,6 +44,7 @@ import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.data.std.api.IDataOutputProvider;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.data.std.util.UTF8StringCharacterIterator;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
public class TemporalYearAccessor extends AbstractScalarFunctionDynamicDescriptor {
@@ -86,6 +87,9 @@ public class TemporalYearAccessor extends AbstractScalarFunctionDynamicDescripto
private final GregorianCalendarSystem calSystem = GregorianCalendarSystem.getInstance();
+ private final UTF8StringPointable strExprPtr = new UTF8StringPointable();
+ private final UTF8StringCharacterIterator strIter = new UTF8StringCharacterIterator();
+
// for output: type integer
@SuppressWarnings("unchecked")
private final ISerializerDeserializer<AInt64> intSerde = AqlSerializerDeserializerProvider.INSTANCE
@@ -129,18 +133,20 @@ public class TemporalYearAccessor extends AbstractScalarFunctionDynamicDescripto
return;
} else if (bytes[0] == SER_STRING_TYPE_TAG) {
int year;
- if (UTF8StringPointable.charAt(bytes, 3) == '-') {
+ strExprPtr.set(bytes, 1, bytes.length);
+ strIter.reset(strExprPtr);
+ char firstChar = strIter.next();
+ if (firstChar == '-') {
// in case of a negative year
year = -1
- * ((UTF8StringPointable.charAt(bytes, 4) - '0') * 1000
- + (UTF8StringPointable.charAt(bytes, 5) - '0') * 100
- + (UTF8StringPointable.charAt(bytes, 6) - '0') * 10 + (UTF8StringPointable
- .charAt(bytes, 7) - '0'));
+ * ((strIter.next() - '0') * 1000
+ + (strIter.next() - '0') * 100
+ + (strIter.next() - '0') * 10 + (strIter.next() - '0'));
} else {
- year = (UTF8StringPointable.charAt(bytes, 3) - '0') * 1000
- + (UTF8StringPointable.charAt(bytes, 4) - '0') * 100
- + (UTF8StringPointable.charAt(bytes, 5) - '0') * 10
- + (UTF8StringPointable.charAt(bytes, 6) - '0');
+ year = (firstChar - '0') * 1000
+ + (strIter.next() - '0') * 100
+ + (strIter.next() - '0') * 10
+ + (strIter.next() - '0');
}
aMutableInt64.setValue(year);
intSerde.serialize(aMutableInt64, out);