You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2017/06/03 04:46:07 UTC
[12/12] drill git commit: DRILL-5229: Update kudu-client to 1.3.0
DRILL-5229: Update kudu-client to 1.3.0
closes #828
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dd2692ec
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dd2692ec
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dd2692ec
Branch: refs/heads/master
Commit: dd2692ecd57a180f33e51147afe063627552979e
Parents: 676ea88
Author: eskabetxe <bo...@boto.pro>
Authored: Sat May 6 13:41:36 2017 +0200
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Fri Jun 2 21:43:14 2017 -0700
----------------------------------------------------------------------
contrib/storage-kudu/pom.xml | 5 ++-
.../codegen/templates/KuduRecordWriter.java | 4 +--
.../drill/exec/store/kudu/DrillKuduTable.java | 14 ++++----
.../drill/exec/store/kudu/KuduGroupScan.java | 4 +--
.../drill/exec/store/kudu/KuduRecordReader.java | 30 ++++++++--------
.../exec/store/kudu/KuduRecordWriterImpl.java | 37 ++++++++++----------
.../exec/store/kudu/KuduSchemaFactory.java | 6 ++--
.../exec/store/kudu/KuduStoragePlugin.java | 2 +-
.../drill/store/kudu/TestKuduConnect.java | 30 ++++++++--------
9 files changed, 67 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/pom.xml b/contrib/storage-kudu/pom.xml
index 2ba9cac..74e6eb8 100644
--- a/contrib/storage-kudu/pom.xml
+++ b/contrib/storage-kudu/pom.xml
@@ -18,7 +18,6 @@
</parent>
<artifactId>drill-kudu-storage</artifactId>
-
<name>contrib/kudu-storage-plugin</name>
@@ -47,9 +46,9 @@
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
- <version>0.6.0</version>
+ <version>1.3.0</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java b/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java
index 01c7c28..2b76cac 100644
--- a/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java
+++ b/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java
@@ -88,7 +88,7 @@ import java.lang.UnsupportedOperationException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import org.kududb.client.*;
+import org.apache.kudu.client.*;
import org.apache.drill.exec.store.*;
public abstract class KuduRecordWriter extends AbstractRecordWriter implements RecordWriter {
@@ -157,7 +157,7 @@ public abstract class KuduRecordWriter extends AbstractRecordWriter implements R
<#elseif minor.class == "VarChar" >
byte[] bytes = new byte[holder.end - holder.start];
holder.buffer.getBytes(holder.start, bytes);
- row.addStringUtf8(fieldId, bytes);
+ row.addString(fieldId, new String(bytes));
<#elseif minor.class == "VarBinary">
byte[] bytes = new byte[holder.end - holder.start];
holder.buffer.getBytes(holder.start, bytes);
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
index 3fc69c6..8404aac 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
@@ -23,9 +23,9 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.drill.exec.planner.logical.DynamicDrillTable;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
import com.google.common.collect.Lists;
@@ -56,8 +56,6 @@ public class DrillKuduTable extends DynamicDrillTable {
private RelDataType getSqlTypeFromKuduType(RelDataTypeFactory typeFactory, Type type) {
switch (type) {
- case BINARY:
- return typeFactory.createSqlType(SqlTypeName.VARBINARY, Integer.MAX_VALUE);
case BOOL:
return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
case DOUBLE:
@@ -70,9 +68,11 @@ public class DrillKuduTable extends DynamicDrillTable {
case INT8:
return typeFactory.createSqlType(SqlTypeName.INTEGER);
case STRING:
- return typeFactory.createSqlType(SqlTypeName.VARCHAR, Integer.MAX_VALUE);
- case TIMESTAMP:
+ return typeFactory.createSqlType(SqlTypeName.VARCHAR);
+ case UNIXTIME_MICROS:
return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
+ case BINARY:
+ return typeFactory.createSqlType(SqlTypeName.VARBINARY, Integer.MAX_VALUE);
default:
throw new UnsupportedOperationException("Unsupported type.");
}
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
index 873f216..dfc3c44 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
@@ -51,8 +51,8 @@ import org.apache.drill.exec.store.schedule.AssignmentCreator;
import org.apache.drill.exec.store.schedule.CompleteWork;
import org.apache.drill.exec.store.schedule.EndpointByteMap;
import org.apache.drill.exec.store.schedule.EndpointByteMapImpl;
-import org.kududb.client.LocatedTablet;
-import org.kududb.client.LocatedTablet.Replica;
+import org.apache.kudu.client.LocatedTablet;
+import org.apache.kudu.client.LocatedTablet.Replica;
@JsonTypeName("kudu-scan")
public class KuduGroupScan extends AbstractGroupScan {
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
index 541daa4..ef7efcf 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
@@ -52,16 +52,16 @@ import org.apache.drill.exec.vector.TimeStampVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduScanner;
-import org.kududb.client.KuduScanner.KuduScannerBuilder;
-import org.kududb.client.KuduTable;
-import org.kududb.client.RowResult;
-import org.kududb.client.RowResultIterator;
-import org.kududb.client.shaded.com.google.common.collect.ImmutableMap;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduScanner;
+import org.apache.kudu.client.KuduScanner.KuduScannerBuilder;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.RowResult;
+import org.apache.kudu.client.RowResultIterator;
+import org.apache.kudu.client.shaded.com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -114,8 +114,8 @@ public class KuduRecordReader extends AbstractRecordReader {
context.getStats().startWait();
try {
scanner = builder
- .lowerBoundPartitionKeyRaw(scanSpec.getStartKey())
- .exclusiveUpperBoundPartitionKeyRaw(scanSpec.getEndKey())
+ .lowerBoundRaw(scanSpec.getStartKey())
+ .exclusiveUpperBoundRaw(scanSpec.getEndKey())
.build();
} finally {
context.getStats().stopWait();
@@ -138,7 +138,7 @@ public class KuduRecordReader extends AbstractRecordReader {
.put(Type.INT32, MinorType.INT)
.put(Type.INT64, MinorType.BIGINT)
.put(Type.STRING, MinorType.VARCHAR)
- .put(Type.TIMESTAMP, MinorType.TIMESTAMP)
+ .put(Type.UNIXTIME_MICROS, MinorType.TIMESTAMP)
.build();
}
@@ -236,7 +236,7 @@ public class KuduRecordReader extends AbstractRecordReader {
break;
}
case STRING: {
- ByteBuffer value = result.getBinary(pci.index);
+ ByteBuffer value = ByteBuffer.wrap(result.getString(pci.index).getBytes());
if (pci.kuduColumn.isNullable()) {
((NullableVarCharVector.Mutator) pci.vv.getMutator())
.setSafe(rowIndex, value, 0, value.remaining());
@@ -309,7 +309,7 @@ public class KuduRecordReader extends AbstractRecordReader {
.setSafe(rowIndex, result.getLong(pci.index));
}
break;
- case TIMESTAMP:
+ case UNIXTIME_MICROS:
if (pci.kuduColumn.isNullable()) {
((NullableTimeStampVector.Mutator) pci.vv.getMutator())
.setSafe(rowIndex, result.getLong(pci.index) / 1000);
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
index 6b39cc5..2e40acf 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
@@ -17,11 +17,6 @@
*/
package org.apache.drill.exec.store.kudu;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
@@ -29,15 +24,21 @@ import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.VectorAccessible;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.Insert;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduSession;
-import org.kududb.client.KuduTable;
-import org.kududb.client.OperationResponse;
-import org.kududb.client.SessionConfiguration.FlushMode;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.Insert;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduSession;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.CreateTableOptions;
+import org.apache.kudu.client.OperationResponse;
+import org.apache.kudu.client.SessionConfiguration.FlushMode;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
public class KuduRecordWriterImpl extends KuduRecordWriter {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(KuduRecordWriterImpl.class);
@@ -81,7 +82,7 @@ public class KuduRecordWriterImpl extends KuduRecordWriter {
i++;
}
Schema kuduSchema = new Schema(columns);
- table = client.createTable(name, kuduSchema);
+ table = client.createTable(name, kuduSchema, new CreateTableOptions());
}
} catch (Exception e) {
throw new IOException(e);
@@ -113,11 +114,11 @@ public class KuduRecordWriterImpl extends KuduRecordWriter {
case INT:
return Type.INT32;
case TIMESTAMP:
- return Type.TIMESTAMP;
- case VARBINARY:
- return Type.BINARY;
+ return Type.UNIXTIME_MICROS;
case VARCHAR:
return Type.STRING;
+ case VARBINARY:
+ return Type.BINARY;
default:
throw UserException
.dataWriteError()
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
index 34e5b2a..4d9caf3 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
@@ -31,9 +31,9 @@ import org.apache.drill.exec.planner.logical.CreateTableEntry;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.SchemaConfig;
import org.apache.drill.exec.store.SchemaFactory;
-import org.kududb.Schema;
-import org.kududb.client.KuduTable;
-import org.kududb.client.ListTablesResponse;
+import org.apache.kudu.Schema;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.ListTablesResponse;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
index 15aa469..0d98755 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
@@ -24,7 +24,7 @@ import org.apache.drill.common.JSONOptions;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.AbstractStoragePlugin;
import org.apache.drill.exec.store.SchemaConfig;
-import org.kududb.client.KuduClient;
+import org.apache.kudu.client.KuduClient;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java b/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
index 0ee0134..2391fc9 100644
--- a/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
+++ b/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
@@ -18,24 +18,25 @@
package org.apache.drill.store.kudu;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.junit.Ignore;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.CreateTableOptions;
-import org.kududb.client.Insert;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduScanner;
-import org.kududb.client.KuduSession;
-import org.kududb.client.KuduTable;
-import org.kududb.client.ListTablesResponse;
-import org.kududb.client.PartialRow;
-import org.kududb.client.RowResult;
-import org.kududb.client.RowResultIterator;
-import org.kududb.client.SessionConfiguration;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.CreateTableOptions;
+import org.apache.kudu.client.Insert;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduScanner;
+import org.apache.kudu.client.KuduSession;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.ListTablesResponse;
+import org.apache.kudu.client.PartialRow;
+import org.apache.kudu.client.RowResult;
+import org.apache.kudu.client.RowResultIterator;
+import org.apache.kudu.client.SessionConfiguration;
@Ignore("requires remote kudu server")
public class TestKuduConnect {
@@ -63,6 +64,7 @@ public class TestKuduConnect {
CreateTableOptions builder = new CreateTableOptions();
builder.setNumReplicas(replicas);
+ builder.setRangePartitionColumns(Arrays.asList("key"));
for (int i = 1; i < tablets; i++) {
PartialRow splitRow = schema.newPartialRow();
splitRow.addInt("key", i*1000);