You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2016/01/28 15:30:08 UTC
drill git commit: DRILL-4291: Fix Missing classes when trying to
query varchar[] using JDBC
Repository: drill
Updated Branches:
refs/heads/master d70cf36ea -> a5157190b
DRILL-4291: Fix Missing classes when trying to query varchar[] using JDBC
- Create a new simplified version of the Hadoop Text class that doesn't include massive dependencies.
- Update Vectors to use new Text class.
- Update the jdbc-all module to have a test which also includes complex types.
- Clean up exclusions in Jdbc jar file to reduce file size.
- Add an enforcer rule that ensures the adbc-all jar exclusions are maintained in the future.
This closes #336.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a5157190
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a5157190
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a5157190
Branch: refs/heads/master
Commit: a5157190b7f634923046d91e7afbaa7e293dc4ac
Parents: d70cf36
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Jan 24 09:59:12 2016 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Jan 28 06:29:27 2016 -0800
----------------------------------------------------------------------
.../src/main/codegen/includes/vv_imports.ftl | 2 +-
.../Decimal/DecimalAggrTypeFunctions1.java | 10 +-
.../java/org/apache/drill/DrillTestWrapper.java | 29 +-
.../test/java/org/apache/drill/TestBuilder.java | 20 +-
.../exec/fn/interp/TestConstantFolding.java | 16 +-
.../exec/planner/TestDirectoryExplorerUDFs.java | 9 +-
.../store/parquet/ParquetRecordReaderTest.java | 15 +-
exec/jdbc-all/pom.xml | 128 ++--
.../org/apache/drill/jdbc/ITTestShadedJar.java | 24 +-
exec/jdbc-all/src/test/resources/types.json | 25 +
.../src/main/codegen/includes/vv_imports.ftl | 2 +-
.../codegen/templates/HolderReaderImpl.java | 1 -
.../drill/exec/util/JsonStringArrayList.java | 6 -
.../drill/exec/util/JsonStringHashMap.java | 6 -
.../java/org/apache/drill/exec/util/Text.java | 621 +++++++++++++++++++
.../apache/drill/exec/util/TextSerializer.java | 39 --
16 files changed, 768 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 5865ece..fadcab8 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -71,7 +71,7 @@ import java.math.BigInteger;
import org.joda.time.DateTime;
import org.joda.time.Period;
-import org.apache.hadoop.io.Text;
+import org.apache.drill.exec.util.Text;
import org.apache.drill.exec.vector.accessor.sql.TimePrintMillis;
import javax.inject.Inject;
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
index 974db9b..22339da 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java
@@ -85,7 +85,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
value = new ObjectHolder();
//${type.runningType}Holder tmp = new ${type.runningType}Holder();
byte[] byteArray = new byte[${type.runningType}Holder.WIDTH];
- org.apache.hadoop.io.Text tmp = new org.apache.hadoop.io.Text(byteArray);
+ org.apache.drill.exec.util.Text tmp = new org.apache.drill.exec.util.Text(byteArray);
value.obj = tmp;
<#if aggrtype.funcName == "max">
for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
@@ -127,7 +127,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#elseif aggrtype.funcName == "max">
<#if type.outputType.endsWith("Sparse")>
//${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
- org.apache.hadoop.io.Text tmp = (org.apache.hadoop.io.Text) value.obj;
+ org.apache.drill.exec.util.Text tmp = (org.apache.drill.exec.util.Text) value.obj;
int cmp = org.apache.drill.exec.util.DecimalUtility.compareSparseSamePrecScale(in.buffer, in.start, tmp.getBytes(), tmp.getLength());
if (cmp == 1) {
//in.buffer.getBytes(in.start, tmp.getBytes(), 0, ${type.runningType}Holder.WIDTH);
@@ -144,7 +144,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
<#elseif aggrtype.funcName == "min">
<#if type.outputType.endsWith("Sparse")>
//${type.runningType}Holder tmp = (${type.runningType}Holder) value.obj;
- org.apache.hadoop.io.Text tmp = (org.apache.hadoop.io.Text) value.obj;
+ org.apache.drill.exec.util.Text tmp = (org.apache.drill.exec.util.Text) value.obj;
int cmp = org.apache.drill.exec.util.DecimalUtility.compareSparseSamePrecScale(in.buffer, in.start, tmp.getBytes(), tmp.getLength());
if (cmp == -1) {
//in.buffer.getBytes(in.start, tmp.getBytes(), 0, ${type.runningType}Holder.WIDTH);
@@ -187,7 +187,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
org.apache.drill.exec.util.DecimalUtility.getSparseFromBigDecimal((java.math.BigDecimal) value.obj, out.buffer, out.start, out.scale, out.precision, out.nDecimalDigits);
<#else>
<#if type.outputType.endsWith("Sparse")>
- org.apache.hadoop.io.Text tmp = (org.apache.hadoop.io.Text) value.obj;
+ org.apache.drill.exec.util.Text tmp = (org.apache.drill.exec.util.Text) value.obj;
buffer = buffer.reallocIfNeeded(tmp.getLength());
//buffer.setBytes(0, tmp.getBytes(), 0, tmp.getLength());
for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
@@ -213,7 +213,7 @@ public static class ${type.inputType}${aggrtype.className} implements DrillAggFu
value.value = 0;
<#elseif aggrtype.funcName == "max" || aggrtype.funcName == "min">
<#if type.outputType.endsWith("Sparse")>
- org.apache.hadoop.io.Text tmp = (org.apache.hadoop.io.Text) value.obj;
+ org.apache.drill.exec.util.Text tmp = (org.apache.drill.exec.util.Text) value.obj;
for (int i = 0; i < ${type.runningType}Holder.nDecimalDigits; i++) {
org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.setInteger(tmp.getBytes(), i, 0xFFFFFFFF);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
index 6f78f8c..df2cfb0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java
@@ -17,6 +17,19 @@
******************************************************************************/
package org.apache.drill;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.HyperVectorValueIterator;
@@ -30,22 +43,8 @@ import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.apache.drill.exec.util.Text;
import org.apache.drill.exec.vector.ValueVector;
-import org.apache.hadoop.io.Text;
-import org.codehaus.jackson.node.BinaryNode;
-
-import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
/**
* An object to encapsulate the options for a Drill unit test, as well as the execution methods to perform the tests and
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
index 330ff12..9115a6c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestBuilder.java
@@ -17,9 +17,14 @@
******************************************************************************/
package org.apache.drill;
-import com.google.common.base.Joiner;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-import com.google.common.base.Preconditions;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
@@ -32,15 +37,10 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.util.JsonStringHashMap;
-import org.apache.hadoop.io.Text;
+import org.apache.drill.exec.util.Text;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
public class TestBuilder {
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
index fdc63bb..c7d2f0f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
@@ -17,23 +17,19 @@
******************************************************************************/
package org.apache.drill.exec.fn.interp;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
+import java.io.File;
+import java.io.PrintWriter;
+import java.util.List;
+
import org.apache.drill.PlanTestBase;
import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.hadoop.io.Text;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import java.io.File;
-import java.io.PrintWriter;
-import java.util.List;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
public class TestConstantFolding extends PlanTestBase {
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
index c2d4136..516f975 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
@@ -17,17 +17,18 @@
******************************************************************************/
package org.apache.drill.exec.planner;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
+import java.util.List;
+
import org.apache.drill.PlanTestBase;
import org.apache.drill.exec.fn.interp.TestConstantFolding;
import org.apache.drill.exec.util.JsonStringArrayList;
-import org.apache.hadoop.io.Text;
+import org.apache.drill.exec.util.Text;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
public class TestDirectoryExplorerUDFs extends PlanTestBase {
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index d5f7352..fdfcf42 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -56,26 +56,25 @@ import org.apache.drill.exec.store.CachedSingleFileSystem;
import org.apache.drill.exec.store.TestOutputMutator;
import org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader;
import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.exec.util.Text;
import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.NullableBigIntVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.parquet.hadoop.CodecFactory;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.column.page.DataPageV1;
import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.hadoop.CodecFactory;
import org.apache.parquet.hadoop.Footer;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.MessageType;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.base.Stopwatch;
@@ -433,8 +432,8 @@ public class ParquetRecordReaderTest extends BaseTestQuery {
testParquetFullEngineEventBased(false, "/parquet/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props);
HashMap<String, FieldInfo> fields2 = new HashMap<>();
// pass strings instead of byte arrays
- Object[] textVals = { new org.apache.hadoop.io.Text("b"), new org.apache.hadoop.io.Text("b2"),
- new org.apache.hadoop.io.Text("b3")};
+ Object[] textVals = { new org.apache.drill.exec.util.Text("b"), new org.apache.drill.exec.util.Text("b2"),
+ new org.apache.drill.exec.util.Text("b3") };
ParquetTestProperties props2 = new ParquetTestProperties(1, 30000, DEFAULT_BYTES_PER_PAGE, fields2);
props2.fields.put("a", new FieldInfo("boolean", "a", 1, textVals, TypeProtos.MinorType.BIT, props2));
testParquetFullEngineEventBased(false, "/parquet/parquet_scan_screen_read_entry_replace.json",
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/jdbc-all/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index a937a32..a9059b4 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -61,6 +61,14 @@
<artifactId>hbase</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ </exclusion>
+ <exclusion>
<artifactId>jersey-container-jetty-servlet</artifactId>
<groupId>org.glassfish.jersey.containers</groupId>
</exclusion>
@@ -77,6 +85,10 @@
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-mapred</artifactId>
+ </exclusion>
+ <exclusion>
<artifactId>jersey-media-multipart</artifactId>
<groupId>org.glassfish.jersey.media</groupId>
</exclusion>
@@ -90,11 +102,11 @@
</exclusion>
<exclusion>
<artifactId>parquet-hadoop</artifactId>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
</exclusion>
<exclusion>
<artifactId>parquet-column</artifactId>
- <groupId>com.twitter</groupId>
+ <groupId>org.apache.parquet</groupId>
</exclusion>
<exclusion>
<artifactId>infinispan-core</artifactId>
@@ -156,28 +168,6 @@
<version>2.6.1</version>
<scope>test</scope>
</dependency>
- <!-- Specify xalan and xerces versions to avoid setXIncludeAware error. -->
- <dependency>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- <exclusions>
- <!-- <exclusion> -->
- <!-- <groupId>xml-apis</groupId> -->
- <!-- <artifactId>xml-apis</artifactId> -->
- <!-- </exclusion> -->
- </exclusions>
- </dependency>
- <dependency>
- <groupId>xalan</groupId>
- <artifactId>xalan</artifactId>
- <exclusions>
- <!-- <exclusion> -->
- <!-- <groupId>xml-apis</groupId> -->
- <!-- <artifactId>xml-apis</artifactId> -->
- <!-- </exclusion> -->
- </exclusions>
- </dependency>
-
<!-- Test Dependencies -->
<dependency>
@@ -283,6 +273,10 @@
<exclude>org.apache.calcite:*</exclude>
<exclude>org.pentaho:*</exclude>
<exclude>org.msgpack:*</exclude>
+ <exclude>xerces:*</exclude>
+ <exclude>xalan:*</exclude>
+ <exclude>org.apache.avro:*</exclude>
+ <exclude>org.mongodb:*</exclude>
<exclude>com.googlecode.json-simple:*</exclude>
<exclude>dom4j:*</exclude>
<exclude>org.hibernate:*</exclude>
@@ -292,6 +286,7 @@
<exclude>com.univocity:*</exclude>
<exclude>net.sf.jpam:*</exclude>
<exclude>com.twitter:*</exclude>
+ <exclude>org.apache.parquet:*</exclude>
<exclude>javax.inject:*</exclude>
<exclude>com.beust:*</exclude>
<exclude>jline:*</exclude>
@@ -391,6 +386,9 @@
<exclude>javax/**</exclude>
<exclude>rest/**</exclude>
<exclude>*.tokens</exclude>
+ <exclude>codegen/**</exclude>
+ <exclude>bootstrap-storage-plugins.json</exclude>
+ <exclude>org/apache/parquet</exclude>
<exclude>com/google/common/math</exclude>
<exclude>com/google/common/net</exclude>
<exclude>com/google/common/primitives</exclude>
@@ -414,6 +412,39 @@
</filters>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-jdbc-jar-compactness</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <phase>verify</phase>
+ <configuration>
+ <rules>
+ <requireFilesSize>
+ <message>
+
+ The file drill-jdbc-all-${project.version}.jar is outside the expected size range.
+
+ This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users.
+
+ </message>
+ <maxsize>20000000</maxsize>
+ <minsize>15000000</minsize>
+ <files>
+ <file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file>
+ </files>
+ </requireFilesSize>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -493,55 +524,6 @@
</plugins>
</build>
</profile>
- <!-- mondrian data includes 10s of MBs of JSON file if you want to include
- them run maven with -Pwith-mondrian-data -->
- <profile>
- <id>with-mondrian-data</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <dependencies>
- <dependency>
- <groupId>net.hydromatic</groupId>
- <artifactId>foodmart-queries</artifactId>
- <version>0.4.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>net.hydromatic</groupId>
- <artifactId>foodmart-data-json</artifactId>
- <version>0.4</version>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>default-hadoop</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <properties>
- <projectVersion>${project.version}</projectVersion>
- </properties>
- </profile>
- <profile>
- <id>mapr</id>
- <properties>
- <projectVersion>${project.version}-mapr</projectVersion>
- </properties>
- </profile>
- <profile>
- <id>cdh</id>
- <properties>
- <projectVersion>${project.version}-mapr</projectVersion>
- </properties>
- </profile>
- <profile>
- <id>hdp</id>
- <properties>
- <projectVersion>${project.version}-hdp</projectVersion>
- </properties>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
index 97a0984..b335c0f 100644
--- a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
+++ b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java
@@ -22,8 +22,10 @@ import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Statement;
import java.util.Vector;
@@ -60,12 +62,9 @@ public class ITTestShadedJar {
Class<?> clazz = loader.loadClass("org.apache.drill.jdbc.Driver");
try {
Driver driver = (Driver) clazz.newInstance();
- try (Connection c = driver.connect("jdbc:drill:drillbit=localhost:31010", null);
- Statement s = c.createStatement();
- ResultSet result = s.executeQuery("select * from (VALUES 1)");) {
- while (result.next()) {
- System.out.println(result.getObject(1));
- }
+ try (Connection c = driver.connect("jdbc:drill:drillbit=localhost:31010", null)) {
+ String path = Paths.get("").toAbsolutePath().toString() + "/../src/test/resources/types.json";
+ printQuery(c, "select * from dfs.`" + path + "`");
}
} catch (Exception ex) {
throw ex;
@@ -73,6 +72,19 @@ public class ITTestShadedJar {
}
+ private static void printQuery(Connection c, String query) throws SQLException {
+ try (Statement s = c.createStatement(); ResultSet result = s.executeQuery(query)) {
+ while (result.next()) {
+ final int columnCount = result.getMetaData().getColumnCount();
+ for(int i = 1; i < columnCount+1; i++){
+ System.out.print(result.getObject(i));
+ System.out.print('\t');
+ }
+ System.out.println(result.getObject(1));
+ }
+ }
+ }
+
@BeforeClass
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/jdbc-all/src/test/resources/types.json
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/src/test/resources/types.json b/exec/jdbc-all/src/test/resources/types.json
new file mode 100644
index 0000000..1e554ea
--- /dev/null
+++ b/exec/jdbc-all/src/test/resources/types.json
@@ -0,0 +1,25 @@
+{
+ "bool": true,
+ "double": 1.3,
+ "str": "my string",
+ "array": ["hello", "goodbye"],
+ "bin" : {
+ "$binary" : "ZHJpbGw="
+ },
+ "drill_date" : {
+ "$dateDay" : "1997-07-16"
+ },
+ "drill_timestamp" : {
+ "$date" : "2009-02-23T08:00:00.000Z"
+ },
+ "time" : {
+ "$time" : "19:20:30.450Z"
+ },
+ "interval" : {
+ "$interval" : "PT26.400S"
+ },
+ "integer" : {
+ "$numberLong" : 4
+ },
+ "map" : { x:5, y:10}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/vector/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/includes/vv_imports.ftl b/exec/vector/src/main/codegen/includes/vv_imports.ftl
index e6dc973..3b1ec64 100644
--- a/exec/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/vector/src/main/codegen/includes/vv_imports.ftl
@@ -65,7 +65,7 @@ import java.math.BigInteger;
import org.joda.time.DateTime;
import org.joda.time.Period;
-import org.apache.hadoop.io.Text;
+import org.apache.drill.exec.util.Text;
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/vector/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/HolderReaderImpl.java b/exec/vector/src/main/codegen/templates/HolderReaderImpl.java
index d0f5c4a..ccff618 100644
--- a/exec/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/exec/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -43,7 +43,6 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import org.apache.drill.exec.expr.holders.*;
-import org.apache.hadoop.io.Text;
import org.joda.time.Period;
// Source code generated using FreeMarker template ${.template_name}
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
index dd5145d..ea5e3ad 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
@@ -20,11 +20,8 @@ package org.apache.drill.exec.util;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hadoop.io.Text;
-
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
public class JsonStringArrayList<E> extends ArrayList<E> {
@@ -32,9 +29,6 @@ public class JsonStringArrayList<E> extends ArrayList<E> {
static {
mapper = new ObjectMapper();
- SimpleModule serializer = new SimpleModule("TextSerializer")
- .addSerializer(Text.class, new TextSerializer());
- mapper.registerModule(serializer);
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
index ac980eb..22c927c 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
@@ -20,11 +20,8 @@ package org.apache.drill.exec.util;
import java.util.LinkedHashMap;
import java.util.Map;
-import org.apache.hadoop.io.Text;
-
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
/*
* Simple class that extends the regular java.util.HashMap but overrides the
@@ -36,9 +33,6 @@ public class JsonStringHashMap<K, V> extends LinkedHashMap<K, V> {
static {
mapper = new ObjectMapper();
- SimpleModule serializer = new SimpleModule("TextSerializer")
- .addSerializer(Text.class, new TextSerializer());
- mapper.registerModule(serializer);
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/vector/src/main/java/org/apache/drill/exec/util/Text.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/Text.java b/exec/vector/src/main/java/org/apache/drill/exec/util/Text.java
new file mode 100644
index 0000000..1074623
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/Text.java
@@ -0,0 +1,621 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.MalformedInputException;
+import java.text.CharacterIterator;
+import java.text.StringCharacterIterator;
+import java.util.Arrays;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+
+/**
+ * A simplified byte wrapper similar to Hadoop's Text class without all the dependencies. Lifted from Hadoop 2.7.1
+ */
+@JsonSerialize(using = Text.TextSerializer.class)
+public class Text {
+
+ private static ThreadLocal<CharsetEncoder> ENCODER_FACTORY =
+ new ThreadLocal<CharsetEncoder>() {
+ @Override
+ protected CharsetEncoder initialValue() {
+ return Charset.forName("UTF-8").newEncoder().
+ onMalformedInput(CodingErrorAction.REPORT).
+ onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+ };
+
+ private static ThreadLocal<CharsetDecoder> DECODER_FACTORY =
+ new ThreadLocal<CharsetDecoder>() {
+ @Override
+ protected CharsetDecoder initialValue() {
+ return Charset.forName("UTF-8").newDecoder().
+ onMalformedInput(CodingErrorAction.REPORT).
+ onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+ };
+
+ private static final byte[] EMPTY_BYTES = new byte[0];
+
+ private byte[] bytes;
+ private int length;
+
+ public Text() {
+ bytes = EMPTY_BYTES;
+ }
+
+ /**
+ * Construct from a string.
+ */
+ public Text(String string) {
+ set(string);
+ }
+
+ /** Construct from another text. */
+ public Text(Text utf8) {
+ set(utf8);
+ }
+
+ /**
+ * Construct from a byte array.
+ */
+ public Text(byte[] utf8) {
+ set(utf8);
+ }
+
+ /**
+ * Get a copy of the bytes that is exactly the length of the data. See {@link #getBytes()} for faster access to the
+ * underlying array.
+ */
+ public byte[] copyBytes() {
+ byte[] result = new byte[length];
+ System.arraycopy(bytes, 0, result, 0, length);
+ return result;
+ }
+
+ /**
+ * Returns the raw bytes; however, only data up to {@link #getLength()} is valid. Please use {@link #copyBytes()} if
+ * you need the returned array to be precisely the length of the data.
+ */
+ public byte[] getBytes() {
+ return bytes;
+ }
+
+ /** Returns the number of bytes in the byte array */
+ public int getLength() {
+ return length;
+ }
+
+ /**
+ * Returns the Unicode Scalar Value (32-bit integer value) for the character at <code>position</code>. Note that this
+ * method avoids using the converter or doing String instantiation
+ *
+ * @return the Unicode scalar value at position or -1 if the position is invalid or points to a trailing byte
+ */
+ public int charAt(int position) {
+ if (position > this.length)
+ {
+ return -1; // too long
+ }
+ if (position < 0)
+ {
+ return -1; // duh.
+ }
+
+ ByteBuffer bb = (ByteBuffer) ByteBuffer.wrap(bytes).position(position);
+ return bytesToCodePoint(bb.slice());
+ }
+
+ public int find(String what) {
+ return find(what, 0);
+ }
+
+ /**
+ * Finds any occurence of <code>what</code> in the backing buffer, starting as position <code>start</code>. The
+ * starting position is measured in bytes and the return value is in terms of byte position in the buffer. The backing
+ * buffer is not converted to a string for this operation.
+ *
+ * @return byte position of the first occurence of the search string in the UTF-8 buffer or -1 if not found
+ */
+ public int find(String what, int start) {
+ try {
+ ByteBuffer src = ByteBuffer.wrap(this.bytes, 0, this.length);
+ ByteBuffer tgt = encode(what);
+ byte b = tgt.get();
+ src.position(start);
+
+ while (src.hasRemaining()) {
+ if (b == src.get()) { // matching first byte
+ src.mark(); // save position in loop
+ tgt.mark(); // save position in target
+ boolean found = true;
+ int pos = src.position() - 1;
+ while (tgt.hasRemaining()) {
+ if (!src.hasRemaining()) { // src expired first
+ tgt.reset();
+ src.reset();
+ found = false;
+ break;
+ }
+ if (!(tgt.get() == src.get())) {
+ tgt.reset();
+ src.reset();
+ found = false;
+ break; // no match
+ }
+ }
+ if (found) {
+ return pos;
+ }
+ }
+ }
+ return -1; // not found
+ } catch (CharacterCodingException e) {
+ // can't get here
+ e.printStackTrace();
+ return -1;
+ }
+ }
+
+ /**
+ * Set to contain the contents of a string.
+ */
+ public void set(String string) {
+ try {
+ ByteBuffer bb = encode(string, true);
+ bytes = bb.array();
+ length = bb.limit();
+ } catch (CharacterCodingException e) {
+ throw new RuntimeException("Should not have happened ", e);
+ }
+ }
+
+ /**
+ * Set to a utf8 byte array
+ */
+ public void set(byte[] utf8) {
+ set(utf8, 0, utf8.length);
+ }
+
+ /** copy a text. */
+ public void set(Text other) {
+ set(other.getBytes(), 0, other.getLength());
+ }
+
+ /**
+ * Set the Text to range of bytes
+ *
+ * @param utf8
+ * the data to copy from
+ * @param start
+ * the first position of the new string
+ * @param len
+ * the number of bytes of the new string
+ */
+ public void set(byte[] utf8, int start, int len) {
+ setCapacity(len, false);
+ System.arraycopy(utf8, start, bytes, 0, len);
+ this.length = len;
+ }
+
+ /**
+ * Append a range of bytes to the end of the given text
+ *
+ * @param utf8
+ * the data to copy from
+ * @param start
+ * the first position to append from utf8
+ * @param len
+ * the number of bytes to append
+ */
+ public void append(byte[] utf8, int start, int len) {
+ setCapacity(length + len, true);
+ System.arraycopy(utf8, start, bytes, length, len);
+ length += len;
+ }
+
+ /**
+ * Clear the string to empty.
+ *
+ * <em>Note</em>: For performance reasons, this call does not clear the underlying byte array that is retrievable via
+ * {@link #getBytes()}. In order to free the byte-array memory, call {@link #set(byte[])} with an empty byte array
+ * (For example, <code>new byte[0]</code>).
+ */
+ public void clear() {
+ length = 0;
+ }
+
+ /*
+ * Sets the capacity of this Text object to <em>at least</em> <code>len</code> bytes. If the current buffer is longer,
+ * then the capacity and existing content of the buffer are unchanged. If <code>len</code> is larger than the current
+ * capacity, the Text object's capacity is increased to match.
+ *
+ * @param len the number of bytes we need
+ *
+ * @param keepData should the old data be kept
+ */
+ private void setCapacity(int len, boolean keepData) {
+ if (bytes == null || bytes.length < len) {
+ if (bytes != null && keepData) {
+ bytes = Arrays.copyOf(bytes, Math.max(len, length << 1));
+ } else {
+ bytes = new byte[len];
+ }
+ }
+ }
+
+ /**
+ * Convert text back to string
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ try {
+ return decode(bytes, 0, length);
+ } catch (CharacterCodingException e) {
+ throw new RuntimeException("Should not have happened ", e);
+ }
+ }
+
+ /**
+ * Read a Text object whose length is already known. This allows creating Text from a stream which uses a different
+ * serialization format.
+ */
+ public void readWithKnownLength(DataInput in, int len) throws IOException {
+ setCapacity(len, false);
+ in.readFully(bytes, 0, len);
+ length = len;
+ }
+
+ /** Returns true iff <code>o</code> is a Text with the same contents. */
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Text)) {
+ return false;
+ }
+
+ final Text that = (Text) o;
+ if (this.getLength() != that.getLength()) {
+ return false;
+ }
+
+ byte[] thisBytes = Arrays.copyOf(this.getBytes(), getLength());
+ byte[] thatBytes = Arrays.copyOf(that.getBytes(), getLength());
+ return Arrays.equals(thisBytes, thatBytes);
+
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ // / STATIC UTILITIES FROM HERE DOWN
+ /**
+ * Converts the provided byte array to a String using the UTF-8 encoding. If the input is malformed, replace by a
+ * default value.
+ */
+ public static String decode(byte[] utf8) throws CharacterCodingException {
+ return decode(ByteBuffer.wrap(utf8), true);
+ }
+
+ public static String decode(byte[] utf8, int start, int length)
+ throws CharacterCodingException {
+ return decode(ByteBuffer.wrap(utf8, start, length), true);
+ }
+
+ /**
+ * Converts the provided byte array to a String using the UTF-8 encoding. If <code>replace</code> is true, then
+ * malformed input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a
+ * MalformedInputException.
+ */
+ public static String decode(byte[] utf8, int start, int length, boolean replace)
+ throws CharacterCodingException {
+ return decode(ByteBuffer.wrap(utf8, start, length), replace);
+ }
+
+ private static String decode(ByteBuffer utf8, boolean replace)
+ throws CharacterCodingException {
+ CharsetDecoder decoder = DECODER_FACTORY.get();
+ if (replace) {
+ decoder.onMalformedInput(
+ java.nio.charset.CodingErrorAction.REPLACE);
+ decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
+ }
+ String str = decoder.decode(utf8).toString();
+ // set decoder back to its default value: REPORT
+ if (replace) {
+ decoder.onMalformedInput(CodingErrorAction.REPORT);
+ decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+ return str;
+ }
+
+ /**
+ * Converts the provided String to bytes using the UTF-8 encoding. If the input is malformed, invalid chars are
+ * replaced by a default value.
+ *
+ * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit()
+ */
+
+ public static ByteBuffer encode(String string)
+ throws CharacterCodingException {
+ return encode(string, true);
+ }
+
+ /**
+ * Converts the provided String to bytes using the UTF-8 encoding. If <code>replace</code> is true, then malformed
+ * input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a
+ * MalformedInputException.
+ *
+ * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit()
+ */
+ public static ByteBuffer encode(String string, boolean replace)
+ throws CharacterCodingException {
+ CharsetEncoder encoder = ENCODER_FACTORY.get();
+ if (replace) {
+ encoder.onMalformedInput(CodingErrorAction.REPLACE);
+ encoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
+ }
+ ByteBuffer bytes =
+ encoder.encode(CharBuffer.wrap(string.toCharArray()));
+ if (replace) {
+ encoder.onMalformedInput(CodingErrorAction.REPORT);
+ encoder.onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+ return bytes;
+ }
+
+ static final public int DEFAULT_MAX_LEN = 1024 * 1024;
+
+ // //// states for validateUTF8
+
+ private static final int LEAD_BYTE = 0;
+
+ private static final int TRAIL_BYTE_1 = 1;
+
+ private static final int TRAIL_BYTE = 2;
+
+ /**
+ * Check if a byte array contains valid utf-8
+ *
+ * @param utf8
+ * byte array
+ * @throws MalformedInputException
+ * if the byte array contains invalid utf-8
+ */
+ public static void validateUTF8(byte[] utf8) throws MalformedInputException {
+ validateUTF8(utf8, 0, utf8.length);
+ }
+
+ /**
+ * Check to see if a byte array is valid utf-8
+ *
+ * @param utf8
+ * the array of bytes
+ * @param start
+ * the offset of the first byte in the array
+ * @param len
+ * the length of the byte sequence
+ * @throws MalformedInputException
+ * if the byte array contains invalid bytes
+ */
+ public static void validateUTF8(byte[] utf8, int start, int len)
+ throws MalformedInputException {
+ int count = start;
+ int leadByte = 0;
+ int length = 0;
+ int state = LEAD_BYTE;
+ while (count < start + len) {
+ int aByte = utf8[count] & 0xFF;
+
+ switch (state) {
+ case LEAD_BYTE:
+ leadByte = aByte;
+ length = bytesFromUTF8[aByte];
+
+ switch (length) {
+ case 0: // check for ASCII
+ if (leadByte > 0x7F) {
+ throw new MalformedInputException(count);
+ }
+ break;
+ case 1:
+ if (leadByte < 0xC2 || leadByte > 0xDF) {
+ throw new MalformedInputException(count);
+ }
+ state = TRAIL_BYTE_1;
+ break;
+ case 2:
+ if (leadByte < 0xE0 || leadByte > 0xEF) {
+ throw new MalformedInputException(count);
+ }
+ state = TRAIL_BYTE_1;
+ break;
+ case 3:
+ if (leadByte < 0xF0 || leadByte > 0xF4) {
+ throw new MalformedInputException(count);
+ }
+ state = TRAIL_BYTE_1;
+ break;
+ default:
+ // too long! Longest valid UTF-8 is 4 bytes (lead + three)
+ // or if < 0 we got a trail byte in the lead byte position
+ throw new MalformedInputException(count);
+ } // switch (length)
+ break;
+
+ case TRAIL_BYTE_1:
+ if (leadByte == 0xF0 && aByte < 0x90) {
+ throw new MalformedInputException(count);
+ }
+ if (leadByte == 0xF4 && aByte > 0x8F) {
+ throw new MalformedInputException(count);
+ }
+ if (leadByte == 0xE0 && aByte < 0xA0) {
+ throw new MalformedInputException(count);
+ }
+ if (leadByte == 0xED && aByte > 0x9F) {
+ throw new MalformedInputException(count);
+ }
+ // falls through to regular trail-byte test!!
+ case TRAIL_BYTE:
+ if (aByte < 0x80 || aByte > 0xBF) {
+ throw new MalformedInputException(count);
+ }
+ if (--length == 0) {
+ state = LEAD_BYTE;
+ } else {
+ state = TRAIL_BYTE;
+ }
+ break;
+ default:
+ break;
+ } // switch (state)
+ count++;
+ }
+ }
+
+ /**
+ * Magic numbers for UTF-8. These are the number of bytes that <em>follow</em> a given lead byte. Trailing bytes have
+ * the value -1. The values 4 and 5 are presented in this table, even though valid UTF-8 cannot include the five and
+ * six byte sequences.
+ */
+ static final int[] bytesFromUTF8 =
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0,
+ // trail bytes
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
+ 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5 };
+
+ /**
+ * Returns the next code point at the current position in the buffer. The buffer's position will be incremented. Any
+ * mark set on this buffer will be changed by this method!
+ */
+ public static int bytesToCodePoint(ByteBuffer bytes) {
+ bytes.mark();
+ byte b = bytes.get();
+ bytes.reset();
+ int extraBytesToRead = bytesFromUTF8[(b & 0xFF)];
+ if (extraBytesToRead < 0)
+ {
+ return -1; // trailing byte!
+ }
+ int ch = 0;
+
+ switch (extraBytesToRead) {
+ case 5:
+ ch += (bytes.get() & 0xFF);
+ ch <<= 6; /* remember, illegal UTF-8 */
+ case 4:
+ ch += (bytes.get() & 0xFF);
+ ch <<= 6; /* remember, illegal UTF-8 */
+ case 3:
+ ch += (bytes.get() & 0xFF);
+ ch <<= 6;
+ case 2:
+ ch += (bytes.get() & 0xFF);
+ ch <<= 6;
+ case 1:
+ ch += (bytes.get() & 0xFF);
+ ch <<= 6;
+ case 0:
+ ch += (bytes.get() & 0xFF);
+ }
+ ch -= offsetsFromUTF8[extraBytesToRead];
+
+ return ch;
+ }
+
+ static final int offsetsFromUTF8[] =
+ { 0x00000000, 0x00003080,
+ 0x000E2080, 0x03C82080, 0xFA082080, 0x82082080 };
+
+ /**
+ * For the given string, returns the number of UTF-8 bytes required to encode the string.
+ *
+ * @param string
+ * text to encode
+ * @return number of UTF-8 bytes required to encode
+ */
+ public static int utf8Length(String string) {
+ CharacterIterator iter = new StringCharacterIterator(string);
+ char ch = iter.first();
+ int size = 0;
+ while (ch != CharacterIterator.DONE) {
+ if ((ch >= 0xD800) && (ch < 0xDC00)) {
+ // surrogate pair?
+ char trail = iter.next();
+ if ((trail > 0xDBFF) && (trail < 0xE000)) {
+ // valid pair
+ size += 4;
+ } else {
+ // invalid pair
+ size += 3;
+ iter.previous(); // rewind one
+ }
+ } else if (ch < 0x80) {
+ size++;
+ } else if (ch < 0x800) {
+ size += 2;
+ } else {
+ // ch < 0x10000, that is, the largest char value
+ size += 3;
+ }
+ ch = iter.next();
+ }
+ return size;
+ }
+
+ public static class TextSerializer extends StdSerializer<Text> {
+
+ public TextSerializer() {
+ super(Text.class);
+ }
+
+ @Override
+ public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
+ throws IOException, JsonGenerationException {
+ jsonGenerator.writeString(text.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/a5157190/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
deleted file mode 100644
index 87df102..0000000
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-
-public class TextSerializer extends StdSerializer<Text> {
-
- public TextSerializer() {
- super(Text.class);
- }
-
- @Override
- public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
- jsonGenerator.writeString(text.toString());
- }
-}