You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by lu...@apache.org on 2021/01/27 05:52:13 UTC
[drill] branch master updated: DRILL-7733: Use streaming for REST
JSON queries
This is an automated email from the ASF dual-hosted git repository.
luoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 077f2a8 DRILL-7733: Use streaming for REST JSON queries
077f2a8 is described below
commit 077f2a81d4583455593024536ee93bc20d080e48
Author: Paul Rogers <pa...@gmail.com>
AuthorDate: Wed Jan 20 18:12:12 2021 -0800
DRILL-7733: Use streaming for REST JSON queries
---
.../drill/exec/store/esri/ShpBatchReader.java | 25 +--
.../drill/exec/store/esri/ShpFormatConfig.java | 4 -
.../drill/exec/store/esri/ShpFormatPlugin.java | 11 +-
.../drill/exec/store/excel/ExcelBatchReader.java | 10 +-
.../drill/exec/store/hdf5/HDF5BatchReader.java | 6 +-
.../hdf5/writers/HDF5TimestampDataWriter.java | 5 +-
.../drill/exec/store/httpd/HttpdLogRecord.java | 15 +-
.../drill/exec/store/httpd/TestHTTPDLogReader.java | 52 ++---
.../exec/store/image/ImageDirectoryProcessor.java | 2 +-
.../exec/store/image/TestImageRecordReader.java | 2 +-
.../drill/exec/store/syslog/SyslogBatchReader.java | 4 +-
exec/java-exec/pom.xml | 7 +
.../apache/drill/exec/ops/SendingAccountor.java | 10 +-
.../physical/impl/protocol/OperatorDriver.java | 2 +-
.../impl/scan/convert/ConvertDateToString.java | 11 +-
.../impl/scan/convert/ConvertStringToDate.java | 6 +-
.../impl/scan/convert/ConvertStringToTime.java | 6 +-
.../scan/convert/ConvertStringToTimeStamp.java | 10 +-
.../scan/convert/ConvertTimeStampToString.java | 17 +-
.../impl/scan/convert/ConvertTimeToString.java | 13 +-
.../impl/scan/convert/DirectConverter.java | 6 +-
.../exec/physical/resultSet/util/JsonWriter.java | 156 +++++++++++++
.../drill/exec/server/options/TypeValidators.java | 4 +-
.../drill/exec/server/rest/BaseQueryRunner.java | 123 +++++++++++
.../exec/server/rest/BaseWebUserConnection.java | 24 ++
.../drill/exec/server/rest/QueryResources.java | 70 ++++--
.../drill/exec/server/rest/RestQueryRunner.java | 107 +--------
.../drill/exec/server/rest/WebUserConnection.java | 21 +-
.../drill/exec/server/rest/stream/QueryRunner.java | 75 +++++++
.../rest/stream/StreamingHttpConnection.java | 230 +++++++++++++++++++
.../store/easy/json/values/TimeValueListener.java | 2 +-
.../exec/store/easy/text/reader/HeaderBuilder.java | 3 -
.../drill/exec/store/pcap/PcapBatchReader.java | 4 +-
.../drill/exec/store/pcap/decoder/TcpSession.java | 6 +-
.../apache/drill/exec/store/pcapng/PcapColumn.java | 2 +-
.../exec/vector/complex/fn/BasicJsonOutput.java | 34 +--
.../exec/vector/complex/fn/ExtendedJsonOutput.java | 5 +-
.../drill/exec/vector/complex/fn/JsonOutput.java | 7 +-
.../drill/exec/vector/complex/fn/JsonWriter.java | 20 +-
.../java/org/apache/drill/TestSchemaCommands.java | 2 +-
.../TestMetastoreWithEasyFormatPlugin.java | 2 +-
.../impl/scan/convert/TestDirectConverter.java | 41 ++--
.../exec/physical/impl/xsort/TestExternalSort.java | 8 +-
.../exec/physical/rowSet/TestFillEmpties.java | 15 +-
.../exec/physical/rowSet/TestScalarAccessors.java | 84 ++++---
.../drill/exec/server/rest/FileVerifier.java | 149 +++++++++++++
.../exec/server/rest/TestResponseHeaders.java | 1 +
.../drill/exec/server/rest/TestRestJson.java | 246 +++++++++++++++++++++
.../exec/server/rest/WebSessionResourcesTest.java | 4 -
.../store/easy/json/loader/TestExtendedArrays.java | 6 +-
.../store/easy/json/loader/TestExtendedTypes.java | 8 +-
.../easy/json/loader/TestExtendedWithSchema.java | 6 +-
.../easy/text/compliant/TestCsvWithSchema.java | 71 +++---
.../apache/drill/exec/store/log/TestLogReader.java | 2 +-
.../exec/store/pcapng/TestPcapngRecordReader.java | 2 +-
.../apache/drill/test/rowSet/RowSetUtilities.java | 10 +-
.../drill/test/rowSet/file/JsonFileBuilder.java | 139 ++----------
exec/java-exec/src/test/resources/rest/cust20.json | 28 +++
exec/java-exec/src/test/resources/rest/failed.json | 3 +
exec/java-exec/src/test/resources/rest/small.json | 18 ++
.../main/codegen/templates/ColumnAccessors.java | 47 ++--
.../record/metadata/AbstractColumnMetadata.java | 2 +-
.../drill/exec/record/metadata/ColumnMetadata.java | 3 +-
.../record/metadata/PrimitiveColumnMetadata.java | 26 +--
.../apache/drill/exec/vector/DateUtilities.java | 24 +-
.../drill/exec/vector/accessor/ScalarReader.java | 13 +-
.../drill/exec/vector/accessor/ValueWriter.java | 6 +-
.../accessor/reader/AbstractScalarReader.java | 6 +-
.../accessor/writer/AbstractScalarWriter.java | 6 +-
.../vector/accessor/writer/BaseScalarWriter.java | 6 +-
.../accessor/writer/NullableScalarWriter.java | 6 +-
.../accessor/writer/dummy/DummyScalarWriter.java | 6 +-
.../metadata/schema/parser/TestSchemaParser.java | 4 +-
73 files changed, 1502 insertions(+), 615 deletions(-)
diff --git a/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpBatchReader.java b/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpBatchReader.java
index 1b49667..d499a3f 100644
--- a/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpBatchReader.java
+++ b/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpBatchReader.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileSplit;
import org.jamel.dbf.DbfReader;
import org.jamel.dbf.structure.DbfField;
-import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,6 +44,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
+import java.time.Instant;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -59,7 +59,6 @@ public class ShpBatchReader implements ManagedReader<FileSchemaNegotiator> {
private FileSplit split;
private ResultSetLoader loader;
- private final ShpReaderConfig readerConfig;
private Path hadoopShp;
private Path hadoopDbf;
private Path hadoopPrj;
@@ -77,16 +76,7 @@ public class ShpBatchReader implements ManagedReader<FileSchemaNegotiator> {
private SpatialReference spatialReference;
private final int maxRecords;
- public static class ShpReaderConfig {
- protected final ShpFormatPlugin plugin;
-
- public ShpReaderConfig(ShpFormatPlugin plugin) {
- this.plugin = plugin;
- }
- }
-
- public ShpBatchReader(ShpReaderConfig readerConfig, int maxRecords) {
- this.readerConfig = readerConfig;
+ public ShpBatchReader(int maxRecords) {
this.maxRecords = maxRecords;
}
@@ -172,15 +162,6 @@ public class ShpBatchReader implements ManagedReader<FileSchemaNegotiator> {
}
}
- private String byteArrayToString(byte[] in) {
- char out[] = new char[in.length * 2];
- for (int i = 0; i < in.length; i++) {
- out[i * 2] = "0123456789ABCDEF".charAt((in[i] >> 4) & 15);
- out[i * 2 + 1] = "0123456789ABCDEF".charAt(in[i] & 15);
- }
- return new String(out);
- }
-
private void processShapefileSet(RowSetLoader rowWriter, final int gid, final Geometry geom, final Object[] dbfRow) {
if (rowWriter.limitReached(maxRecords)) {
return;
@@ -288,7 +269,7 @@ public class ShpBatchReader implements ManagedReader<FileSchemaNegotiator> {
private void writeTimeColumn(TupleWriter rowWriter, String name, long value) {
int index = rowWriter.tupleSchema().index(name);
- Instant instant = new Instant(value);
+ Instant instant = Instant.ofEpochMilli(value);
if (index == -1) {
ColumnMetadata colSchema = MetadataUtils.newScalar(name, TypeProtos.MinorType.INT, TypeProtos.DataMode.OPTIONAL);
index = rowWriter.addColumn(colSchema);
diff --git a/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpFormatConfig.java b/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpFormatConfig.java
index fd59eb3..4921045 100644
--- a/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpFormatConfig.java
+++ b/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpFormatConfig.java
@@ -51,10 +51,6 @@ public class ShpFormatConfig implements FormatPluginConfig {
return extensions;
}
- public ShpBatchReader.ShpReaderConfig getReaderConfig(ShpFormatPlugin plugin) {
- return new ShpBatchReader.ShpReaderConfig(plugin);
- }
-
@Override
public int hashCode() {
return Objects.hash(extensions);
diff --git a/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpFormatPlugin.java b/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpFormatPlugin.java
index 20bb704..d93243c 100644
--- a/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpFormatPlugin.java
+++ b/contrib/format-esri/src/main/java/org/apache/drill/exec/store/esri/ShpFormatPlugin.java
@@ -30,7 +30,6 @@ import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.store.dfs.easy.EasyFormatPlugin;
import org.apache.drill.exec.store.dfs.easy.EasySubScan;
-import org.apache.drill.exec.store.esri.ShpBatchReader.ShpReaderConfig;
import org.apache.hadoop.conf.Configuration;
public class ShpFormatPlugin extends EasyFormatPlugin<ShpFormatConfig> {
@@ -38,18 +37,16 @@ public class ShpFormatPlugin extends EasyFormatPlugin<ShpFormatConfig> {
public static final String PLUGIN_NAME = "shp";
public static class ShpReaderFactory extends FileReaderFactory {
- private final ShpReaderConfig readerConfig;
private final int maxRecords;
- public ShpReaderFactory(ShpReaderConfig config, int maxRecords) {
- readerConfig = config;
+ public ShpReaderFactory(int maxRecords) {
this.maxRecords = maxRecords;
}
@Override
public ManagedReader<? extends FileScanFramework.FileSchemaNegotiator> newReader() {
- return new ShpBatchReader(readerConfig, maxRecords);
+ return new ShpBatchReader(maxRecords);
}
}
@@ -59,13 +56,13 @@ public class ShpFormatPlugin extends EasyFormatPlugin<ShpFormatConfig> {
@Override
public ManagedReader<? extends FileSchemaNegotiator> newBatchReader(EasySubScan scan, OptionManager options) throws ExecutionSetupException {
- return new ShpBatchReader(formatConfig.getReaderConfig(this), scan.getMaxRecords());
+ return new ShpBatchReader(scan.getMaxRecords());
}
@Override
protected FileScanFramework.FileScanBuilder frameworkBuilder(OptionManager options, EasySubScan scan) {
FileScanFramework.FileScanBuilder builder = new FileScanFramework.FileScanBuilder();
- builder.setReaderFactory(new ShpReaderFactory(new ShpReaderConfig(this), scan.getMaxRecords()));
+ builder.setReaderFactory(new ShpReaderFactory(scan.getMaxRecords()));
initScanBuilder(builder, scan);
builder.nullType(Types.optional(TypeProtos.MinorType.VARCHAR));
return builder;
diff --git a/contrib/format-excel/src/main/java/org/apache/drill/exec/store/excel/ExcelBatchReader.java b/contrib/format-excel/src/main/java/org/apache/drill/exec/store/excel/ExcelBatchReader.java
index 82901f1..d30f859 100644
--- a/contrib/format-excel/src/main/java/org/apache/drill/exec/store/excel/ExcelBatchReader.java
+++ b/contrib/format-excel/src/main/java/org/apache/drill/exec/store/excel/ExcelBatchReader.java
@@ -42,12 +42,12 @@ import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
-import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -563,7 +563,7 @@ public class ExcelBatchReader implements ManagedReader<FileSchemaNegotiator> {
if (timeValue == null) {
metadataColumnWriters.get(index).setNull();
} else {
- metadataColumnWriters.get(index).setTimestamp(new Instant(timeValue));
+ metadataColumnWriters.get(index).setTimestamp(Instant.ofEpochMilli(timeValue.getTime()));
}
}
}
@@ -604,6 +604,7 @@ public class ExcelBatchReader implements ManagedReader<FileSchemaNegotiator> {
super(columnWriter);
}
+ @Override
public void load(Cell cell) {
if (cell == null) {
columnWriter.setNull();
@@ -622,6 +623,7 @@ public class ExcelBatchReader implements ManagedReader<FileSchemaNegotiator> {
super(columnWriter);
}
+ @Override
public void load(Cell cell) {
if (cell == null) {
columnWriter.setNull();
@@ -637,6 +639,7 @@ public class ExcelBatchReader implements ManagedReader<FileSchemaNegotiator> {
super(columnWriter);
}
+ @Override
public void load(Cell cell) {
if (cell == null) {
columnWriter.setNull();
@@ -652,13 +655,14 @@ public class ExcelBatchReader implements ManagedReader<FileSchemaNegotiator> {
super(columnWriter);
}
+ @Override
public void load(Cell cell) {
if (cell == null) {
columnWriter.setNull();
} else {
logger.debug("Cell value: {}", cell.getNumericCellValue());
Date dt = DateUtil.getJavaDate(cell.getNumericCellValue(), TimeZone.getTimeZone("UTC"));
- Instant timeStamp = new Instant(dt.toInstant().getEpochSecond() * 1000);
+ Instant timeStamp = Instant.ofEpochMilli(dt.toInstant().getEpochSecond() * 1000);
columnWriter.setTimestamp(timeStamp);
}
}
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/HDF5BatchReader.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/HDF5BatchReader.java
index 828061f..c560660 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/HDF5BatchReader.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/HDF5BatchReader.java
@@ -52,7 +52,6 @@ import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.TupleWriter;
import org.apache.hadoop.mapred.FileSplit;
-import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,6 +61,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.StandardCopyOption;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -970,7 +970,7 @@ public class HDF5BatchReader implements ManagedReader<FileSchemaNegotiator> {
}
private void writeTimestampColumn(TupleWriter rowWriter, String name, long timestamp) {
- Instant ts = new Instant(timestamp);
+ Instant ts = Instant.ofEpochMilli(timestamp);
ScalarWriter colWriter = getColWriter(rowWriter, name, TypeProtos.MinorType.TIMESTAMP);
colWriter.setTimestamp(ts);
}
@@ -984,7 +984,7 @@ public class HDF5BatchReader implements ManagedReader<FileSchemaNegotiator> {
ScalarWriter arrayWriter = rowWriter.column(index).array().scalar();
for (long l : list) {
- arrayWriter.setTimestamp(new Instant(l));
+ arrayWriter.setTimestamp(Instant.ofEpochMilli(l));
}
}
diff --git a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5TimestampDataWriter.java b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5TimestampDataWriter.java
index 808e89f..881a762 100644
--- a/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5TimestampDataWriter.java
+++ b/contrib/format-hdf5/src/main/java/org/apache/drill/exec/store/hdf5/writers/HDF5TimestampDataWriter.java
@@ -18,10 +18,11 @@
package org.apache.drill.exec.store.hdf5.writers;
+import java.time.Instant;
+
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.store.hdf5.HDF5Utils;
import org.apache.drill.exec.vector.accessor.ValueWriter;
-import org.joda.time.Instant;
import ch.systemsx.cisd.hdf5.IHDF5Reader;
@@ -45,7 +46,7 @@ public class HDF5TimestampDataWriter extends HDF5DataWriter {
if (counter > data.length) {
return false;
} else {
- colWriter.setTimestamp(new Instant(data[counter++]));
+ colWriter.setTimestamp(Instant.ofEpochMilli(data[counter++]));
return true;
}
}
diff --git a/contrib/format-httpd/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java b/contrib/format-httpd/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
index c30b468..90f2748 100644
--- a/contrib/format-httpd/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
+++ b/contrib/format-httpd/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
@@ -32,13 +32,13 @@ import java.util.Map;
import nl.basjes.parse.core.Casts;
import nl.basjes.parse.core.Parser;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
import java.util.Date;
public class HttpdLogRecord {
@@ -129,7 +129,7 @@ public class HttpdLogRecord {
final ScalarWriter w = dates.get(field);
if (w != null) {
logger.debug("Parsed field: {}, as long: {}", field, value);
- w.setDate(new LocalDate(value));
+ w.setDate(LocalDate.parse(value));
} else {
logger.warn("No 'date' writer found for field: {}", field);
}
@@ -149,7 +149,7 @@ public class HttpdLogRecord {
final ScalarWriter w = times.get(field);
if (w != null) {
logger.debug("Parsed field: {}, as long: {}", field, value);
- w.setTime(new LocalTime(value));
+ w.setTime(LocalTime.parse(value));
} else {
logger.warn("No 'date' writer found for field: {}", field);
}
@@ -169,14 +169,13 @@ public class HttpdLogRecord {
final ScalarWriter w = timestamps.get(field);
if (w != null) {
logger.debug("Parsed field: {}, as timestamp: {}", field, value);
- w.setTimestamp(new Instant(value));
+ w.setTimestamp(Instant.ofEpochMilli(value));
} else {
logger.warn("No 'timestamp' writer found for field: {}", field);
}
}
}
-
/**
* This method is referenced and called via reflection. This is added as a parsing target for the parser. It will get
* called when the value of a log field is a timestamp data type.
@@ -199,7 +198,7 @@ public class HttpdLogRecord {
final ScalarWriter tw = timestamps.get(field);
if (tw != null) {
logger.debug("Parsed field: {}, as time: {}", field, value);
- tw.setTimestamp(new Instant(ts));
+ tw.setTimestamp(Instant.ofEpochMilli(ts));
} else {
logger.warn("No 'timestamp' writer found for field: {}", field);
}
diff --git a/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReader.java b/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReader.java
index f240a82..ae29a27 100644
--- a/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReader.java
+++ b/contrib/format-httpd/src/test/java/org/apache/drill/exec/store/httpd/TestHTTPDLogReader.java
@@ -31,12 +31,12 @@ import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.rowSet.RowSetComparison;
import org.apache.drill.test.rowSet.RowSetUtilities;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.time.LocalTime;
import java.util.stream.Collectors;
import static org.apache.drill.test.QueryTestUtil.generateCompressedFile;
@@ -228,16 +228,16 @@ public class TestHTTPDLogReader extends ClusterTest {
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(null, new LocalTime("04:11:25"), null)
- .addRow(null, new LocalTime("04:11:26"), null)
- .addRow(null, new LocalTime("04:11:27"), null)
- .addRow(null, new LocalTime("04:24:31"), null)
- .addRow(null, new LocalTime("04:24:32"), null)
- .addRow(null, new LocalTime("04:24:37"), null)
- .addRow(null, new LocalTime("04:24:39"), null)
- .addRow(null, new LocalTime("04:24:41"), null)
- .addRow(null, new LocalTime("04:32:22"), null)
- .addRow(null, new LocalTime("04:34:37"), null)
+ .addRow(null, LocalTime.parse("04:11:25"), null)
+ .addRow(null, LocalTime.parse("04:11:26"), null)
+ .addRow(null, LocalTime.parse("04:11:27"), null)
+ .addRow(null, LocalTime.parse("04:24:31"), null)
+ .addRow(null, LocalTime.parse("04:24:32"), null)
+ .addRow(null, LocalTime.parse("04:24:37"), null)
+ .addRow(null, LocalTime.parse("04:24:39"), null)
+ .addRow(null, LocalTime.parse("04:24:41"), null)
+ .addRow(null, LocalTime.parse("04:32:22"), null)
+ .addRow(null, LocalTime.parse("04:34:37"), null)
.build();
new RowSetComparison(expected).verifyAndClearAll(results);
@@ -259,16 +259,16 @@ public class TestHTTPDLogReader extends ClusterTest {
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(null, new LocalTime("04:11:25"), null)
- .addRow(null, new LocalTime("04:11:26"), null)
- .addRow(null, new LocalTime("04:11:27"), null)
- .addRow(null, new LocalTime("04:24:31"), null)
- .addRow(null, new LocalTime("04:24:32"), null)
- .addRow(null, new LocalTime("04:24:37"), null)
- .addRow(null, new LocalTime("04:24:39"), null)
- .addRow(null, new LocalTime("04:24:41"), null)
- .addRow(null, new LocalTime("04:32:22"), null)
- .addRow(null, new LocalTime("04:34:37"), null)
+ .addRow(null, LocalTime.parse("04:11:25"), null)
+ .addRow(null, LocalTime.parse("04:11:26"), null)
+ .addRow(null, LocalTime.parse("04:11:27"), null)
+ .addRow(null, LocalTime.parse("04:24:31"), null)
+ .addRow(null, LocalTime.parse("04:24:32"), null)
+ .addRow(null, LocalTime.parse("04:24:37"), null)
+ .addRow(null, LocalTime.parse("04:24:39"), null)
+ .addRow(null, LocalTime.parse("04:24:41"), null)
+ .addRow(null, LocalTime.parse("04:32:22"), null)
+ .addRow(null, LocalTime.parse("04:34:37"), null)
.build();
new RowSetComparison(expected).verifyAndClearAll(results);
@@ -405,10 +405,10 @@ public class TestHTTPDLogReader extends ClusterTest {
"GET /linux/doing-pxe-without-dhcp-control HTTP/1.1", "GET",
"HTTP/1.1", "1.1", "/linux/doing-pxe-without-dhcp-control", null, "/linux/doing-pxe-without-dhcp-control", null, null, null, null, null,
"HTTP/1.1", "1.1", "/linux/doing-pxe-without-dhcp-control", null, "/linux/doing-pxe-without-dhcp-control", null, null, null, null, null,
- 1445742685000L, new LocalDate("2015-10-25"), new LocalDate("2015-10-25"), 25, 25, 1445742685000L, 4, 3,
- 1445742685000L, new LocalDate("2015-10-25"), new LocalDate("2015-10-25"), 25, 25, 1445742685000L, 4, 3,
- 0, 0, 0, 0, 11, 11, 10, 10, "October", "October", 0, 0, 25, 25, new LocalTime("04:11:25"), new LocalTime("03:11:25"), "+01:00", 43, 43, 2015, 2015, 2015, 2015,
- 0, 0, 0, 0, 11, 11, 10, 10, "October", "October", 0, 0, 25, 25, new LocalTime("04:11:25"), new LocalTime("03:11:25"), "+01:00", 43, 43, 2015, 2015, 2015, 2015,
+ 1445742685000L, LocalDate.parse("2015-10-25"), LocalDate.parse("2015-10-25"), 25, 25, 1445742685000L, 4, 3,
+ 1445742685000L, LocalDate.parse("2015-10-25"), LocalDate.parse("2015-10-25"), 25, 25, 1445742685000L, 4, 3,
+ 0, 0, 0, 0, 11, 11, 10, 10, "October", "October", 0, 0, 25, 25, LocalTime.parse("04:11:25"), LocalTime.parse("03:11:25"), "+01:00", 43, 43, 2015, 2015, 2015, 2015,
+ 0, 0, 0, 0, 11, 11, 10, 10, "October", "October", 0, 0, 25, 25, LocalTime.parse("04:11:25"), LocalTime.parse("03:11:25"), "+01:00", 43, 43, 2015, 2015, 2015, 2015,
"http://howto.basjes.nl/", "howto.basjes.nl",
"http://howto.basjes.nl/", "howto.basjes.nl",
"/", null, "http", null, null, null,
diff --git a/contrib/format-image/src/main/java/org/apache/drill/exec/store/image/ImageDirectoryProcessor.java b/contrib/format-image/src/main/java/org/apache/drill/exec/store/image/ImageDirectoryProcessor.java
index ba5ddfd..a42c785 100644
--- a/contrib/format-image/src/main/java/org/apache/drill/exec/store/image/ImageDirectoryProcessor.java
+++ b/contrib/format-image/src/main/java/org/apache/drill/exec/store/image/ImageDirectoryProcessor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.image;
+import java.time.Instant;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
@@ -29,7 +30,6 @@ import org.apache.drill.exec.store.image.ImageBatchReader.MapColumnDefn;
import org.apache.drill.exec.vector.accessor.ArrayWriter;
import org.apache.drill.exec.vector.accessor.ColumnWriter;
import org.apache.drill.exec.vector.accessor.TupleWriter;
-import org.joda.time.Instant;
import org.slf4j.LoggerFactory;
import com.adobe.internal.xmp.XMPException;
diff --git a/contrib/format-image/src/test/java/org/apache/drill/exec/store/image/TestImageRecordReader.java b/contrib/format-image/src/test/java/org/apache/drill/exec/store/image/TestImageRecordReader.java
index 5c2a24e..0f2e140 100644
--- a/contrib/format-image/src/test/java/org/apache/drill/exec/store/image/TestImageRecordReader.java
+++ b/contrib/format-image/src/test/java/org/apache/drill/exec/store/image/TestImageRecordReader.java
@@ -24,6 +24,7 @@ import static org.apache.drill.test.rowSet.RowSetUtilities.strArray;
import static org.junit.Assert.assertEquals;
import java.nio.file.Paths;
+import java.time.Instant;
import java.util.Arrays;
import org.apache.drill.categories.RowSetTests;
@@ -37,7 +38,6 @@ import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.QueryBuilder;
import org.apache.drill.test.QueryTestUtil;
import org.apache.drill.test.rowSet.RowSetComparison;
-import org.joda.time.Instant;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
diff --git a/contrib/format-syslog/src/main/java/org/apache/drill/exec/store/syslog/SyslogBatchReader.java b/contrib/format-syslog/src/main/java/org/apache/drill/exec/store/syslog/SyslogBatchReader.java
index 6dd3e55..4573753 100644
--- a/contrib/format-syslog/src/main/java/org/apache/drill/exec/store/syslog/SyslogBatchReader.java
+++ b/contrib/format-syslog/src/main/java/org/apache/drill/exec/store/syslog/SyslogBatchReader.java
@@ -35,7 +35,6 @@ import org.apache.drill.exec.store.dfs.easy.EasySubScan;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.TupleWriter;
import org.apache.hadoop.mapred.FileSplit;
-import org.joda.time.Instant;
import org.realityforge.jsyslog.message.StructuredDataParameter;
import org.realityforge.jsyslog.message.SyslogMessage;
import org.slf4j.Logger;
@@ -45,6 +44,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -231,7 +231,7 @@ public class SyslogBatchReader implements ManagedReader<FileSchemaNegotiator> {
private void writeStructuredColumns(SyslogMessage parsedMessage) {
long milliseconds = parsedMessage.getTimestamp().getMillis();
- writerArray.get(0).setTimestamp(new Instant(milliseconds));
+ writerArray.get(0).setTimestamp(Instant.ofEpochMilli(milliseconds));
writerArray.get(1).setInt(parsedMessage.getLevel().ordinal());
writerArray.get(2).setInt(parsedMessage.getFacility().ordinal());
setString(writerArray.get(3), parsedMessage.getLevel().name());
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 9535721..756265b 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -32,6 +32,7 @@
<libpam4j.version>1.8-rev2</libpam4j.version>
<aether.version>1.1.0</aether.version>
<wagon.version>3.3.4</wagon.version>
+ <okhttp.version>4.5.0</okhttp.version>
</properties>
<dependencies>
@@ -620,6 +621,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ <version>${okhttp.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SendingAccountor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SendingAccountor.java
index eca4011..d588b60 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SendingAccountor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/SendingAccountor.java
@@ -20,14 +20,18 @@ package org.apache.drill.exec.ops;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
- * Account for whether all messages sent have been completed. Necessary before finishing a task so we don't think
- * buffers are hanging when they will be released.
+ * Account for whether all messages sent have been completed. Necessary before
+ * finishing a task so we don't think buffers are hanging when they will be
+ * released.
*
* TODO: Need to update to use long for number of pending messages.
*/
public class SendingAccountor {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SendingAccountor.class);
+ private static final Logger logger = LoggerFactory.getLogger(SendingAccountor.class);
private final AtomicInteger batchesSent = new AtomicInteger(0);
private final Semaphore wait = new Semaphore(0);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/OperatorDriver.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/OperatorDriver.java
index 7193d1c..94adb1c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/OperatorDriver.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/OperatorDriver.java
@@ -198,7 +198,7 @@ public class OperatorDriver {
*/
private IterOutcome doNext() {
- if (! operatorExec.next()) {
+ if (!operatorExec.next()) {
state = State.END;
return IterOutcome.NONE;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertDateToString.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertDateToString.java
index 136d0e7..57b8acd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertDateToString.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertDateToString.java
@@ -17,12 +17,11 @@
*/
package org.apache.drill.exec.physical.impl.scan.convert;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+
import org.apache.drill.exec.vector.accessor.InvalidConversionError;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
-import org.joda.time.LocalDate;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
public class ConvertDateToString extends DirectConverter {
@@ -32,7 +31,7 @@ public class ConvertDateToString extends DirectConverter {
super(baseWriter);
final String formatValue = baseWriter.schema().format();
dateTimeFormatter = formatValue == null
- ? ISODateTimeFormat.date() : DateTimeFormat.forPattern(formatValue);
+ ? DateTimeFormatter.ISO_LOCAL_DATE : DateTimeFormatter.ofPattern(formatValue);
}
@Override
@@ -41,7 +40,7 @@ public class ConvertDateToString extends DirectConverter {
baseWriter.setNull();
} else {
try {
- baseWriter.setString(dateTimeFormatter.print(value));
+ baseWriter.setString(dateTimeFormatter.format(value));
}
catch (final IllegalStateException e) {
throw InvalidConversionError.writeError(schema(), value, e);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToDate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToDate.java
index 68c3c19..5e2bae1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToDate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToDate.java
@@ -17,12 +17,12 @@
*/
package org.apache.drill.exec.physical.impl.scan.convert;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
import java.util.Map;
import org.apache.drill.exec.vector.accessor.InvalidConversionError;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
-import org.joda.time.LocalDate;
-import org.joda.time.format.DateTimeFormatter;
/**
* Convert a VARCHAR column to an DATE column following the Java rules
@@ -48,7 +48,7 @@ public class ConvertStringToDate extends AbstractConvertFromString {
try {
baseWriter.setDate(LocalDate.parse(prepared, dateTimeFormatter));
}
- catch (final IllegalStateException e) {
+ catch (final Exception e) {
throw InvalidConversionError.writeError(schema(), value, e);
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToTime.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToTime.java
index 945c395..f6701fb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToTime.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToTime.java
@@ -17,12 +17,12 @@
*/
package org.apache.drill.exec.physical.impl.scan.convert;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
import java.util.Map;
import org.apache.drill.exec.vector.accessor.InvalidConversionError;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
-import org.joda.time.LocalTime;
-import org.joda.time.format.DateTimeFormatter;
/**
* Convert a VARCHAR column to an TIME column following the Java rules
@@ -48,7 +48,7 @@ public class ConvertStringToTime extends AbstractConvertFromString {
try {
baseWriter.setTime(LocalTime.parse(prepared, dateTimeFormatter));
}
- catch (final IllegalStateException e) {
+ catch (final Exception e) {
throw InvalidConversionError.writeError(schema(), value, e);
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToTimeStamp.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToTimeStamp.java
index d73f3ca..f957a89 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToTimeStamp.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertStringToTimeStamp.java
@@ -17,12 +17,13 @@
*/
package org.apache.drill.exec.physical.impl.scan.convert;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.Map;
import org.apache.drill.exec.vector.accessor.InvalidConversionError;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
-import org.joda.time.Instant;
-import org.joda.time.format.DateTimeFormatter;
/**
* Convert a VARCHAR column to an TIMESTAMP column following the Java rules
@@ -46,9 +47,10 @@ public class ConvertStringToTimeStamp extends AbstractConvertFromString {
return;
}
try {
- baseWriter.setTimestamp(Instant.parse(prepared, dateTimeFormatter));
+ baseWriter.setTimestamp(LocalDateTime.parse(value, dateTimeFormatter)
+ .toInstant(ZoneOffset.UTC));
}
- catch (final IllegalStateException e) {
+ catch (final Exception e) {
throw InvalidConversionError.writeError(schema(), value, e);
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertTimeStampToString.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertTimeStampToString.java
index b17e0ec..3bc3b9e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertTimeStampToString.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertTimeStampToString.java
@@ -17,12 +17,12 @@
*/
package org.apache.drill.exec.physical.impl.scan.convert;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
import org.apache.drill.exec.vector.accessor.InvalidConversionError;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
-import org.joda.time.Instant;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
public class ConvertTimeStampToString extends DirectConverter {
@@ -31,8 +31,9 @@ public class ConvertTimeStampToString extends DirectConverter {
public ConvertTimeStampToString(ScalarWriter baseWriter) {
super(baseWriter);
final String formatValue = baseWriter.schema().format();
- dateTimeFormatter = formatValue == null
- ? ISODateTimeFormat.dateTime() : DateTimeFormat.forPattern(formatValue);
+ dateTimeFormatter = (formatValue == null
+ ? DateTimeFormatter.ISO_LOCAL_DATE_TIME : DateTimeFormatter.ofPattern(formatValue))
+ .withZone(ZoneId.systemDefault());
}
@Override
@@ -41,9 +42,9 @@ public class ConvertTimeStampToString extends DirectConverter {
baseWriter.setNull();
} else {
try {
- baseWriter.setString(dateTimeFormatter.print(value));
+ baseWriter.setString(dateTimeFormatter.format(value));
}
- catch (final IllegalStateException e) {
+ catch (final Exception e) {
throw InvalidConversionError.writeError(schema(), value, e);
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertTimeToString.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertTimeToString.java
index d9222fa..993eb66 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertTimeToString.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/ConvertTimeToString.java
@@ -17,12 +17,11 @@
*/
package org.apache.drill.exec.physical.impl.scan.convert;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+
import org.apache.drill.exec.vector.accessor.InvalidConversionError;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
-import org.joda.time.LocalTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
public class ConvertTimeToString extends DirectConverter {
@@ -32,7 +31,7 @@ public class ConvertTimeToString extends DirectConverter {
super(baseWriter);
final String formatValue = baseWriter.schema().format();
dateTimeFormatter = formatValue == null
- ? ISODateTimeFormat.time() : DateTimeFormat.forPattern(formatValue);
+ ? DateTimeFormatter.ISO_LOCAL_TIME : DateTimeFormatter.ofPattern(formatValue);
}
@Override
@@ -41,9 +40,9 @@ public class ConvertTimeToString extends DirectConverter {
baseWriter.setNull();
} else {
try {
- baseWriter.setString(dateTimeFormatter.print(value));
+ baseWriter.setString(dateTimeFormatter.format(value));
}
- catch (final IllegalStateException e) {
+ catch (final Exception e) {
throw InvalidConversionError.writeError(schema(), value, e);
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/DirectConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/DirectConverter.java
index 344184c..2f1fcb9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/DirectConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/convert/DirectConverter.java
@@ -18,12 +18,12 @@
package org.apache.drill.exec.physical.impl.scan.convert;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.ValueWriter;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
/**
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/util/JsonWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/util/JsonWriter.java
new file mode 100644
index 0000000..e7bc9e2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/util/JsonWriter.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.resultSet.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.drill.exec.vector.accessor.ArrayReader;
+import org.apache.drill.exec.vector.accessor.ObjectReader;
+import org.apache.drill.exec.vector.accessor.ScalarReader;
+import org.apache.drill.exec.vector.accessor.TupleReader;
+import org.apache.drill.exec.vector.complex.fn.BasicJsonOutput;
+import org.apache.drill.exec.vector.complex.fn.ExtendedJsonOutput;
+import org.apache.drill.exec.vector.complex.fn.JsonOutput;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonGenerator.Feature;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
+
+public class JsonWriter {
+ private final JsonOutput gen;
+
+ public JsonWriter(OutputStream out, boolean pretty, boolean useExtendedOutput) throws IOException {
+ JsonFactory factory = new JsonFactory();
+ JsonGenerator writer = factory.disable(Feature.FLUSH_PASSED_TO_STREAM).createGenerator(out);
+
+ // Need at least a minimal pretty printer to get line-separated objects
+ writer.setPrettyPrinter(pretty ?
+ new DefaultPrettyPrinter("\n") :
+ new MinimalPrettyPrinter("\n"));
+ if (useExtendedOutput) {
+ gen = new ExtendedJsonOutput(writer);
+ } else {
+ gen = new BasicJsonOutput(writer);
+ }
+ }
+
+ public JsonWriter(JsonOutput gen) {
+ this.gen = gen;
+ }
+
+ public JsonOutput jsonOutput() { return gen; }
+
+ public void writeRow(TupleReader reader) throws IOException {
+ writeObject(reader);
+ gen.flush();
+ }
+
+ private void writeObject(TupleReader reader) throws IOException {
+ gen.writeStartObject();
+ for (int i = 0; i < reader.tupleSchema().size(); i++) {
+ writeColumn(reader.column(i));
+ }
+ gen.writeEndObject();
+ }
+
+ private void writeColumn(ObjectReader reader) throws IOException {
+ gen.writeFieldName(reader.schema().name());
+ writeValue(reader);
+ }
+
+ private void writeValue(ObjectReader reader) throws IOException {
+ if (reader.isNull()) {
+ gen.writeUntypedNull();
+ return;
+ }
+ switch (reader.type()) {
+ case ARRAY:
+ writeArray(reader.array());
+ break;
+ case SCALAR:
+ writeScalar(reader.scalar());
+ break;
+ case TUPLE:
+ writeObject(reader.tuple());
+ break;
+ case VARIANT:
+ writeColumn(reader.variant().member());
+ break;
+ default:
+ throw new IllegalStateException(reader.type().name());
+ }
+ }
+
+ private void writeArray(ArrayReader reader) throws IOException {
+ gen.writeStartArray();
+ ObjectReader entryWriter = reader.entry();
+ while (reader.next()) {
+ writeValue(entryWriter);
+ }
+ gen.writeEndArray();
+ }
+
+ private void writeScalar(ScalarReader reader) throws IOException {
+ switch (reader.valueType()) {
+ case BOOLEAN:
+ gen.writeBoolean(reader.getBoolean());
+ break;
+ case BYTES:
+ gen.writeBinary(reader.getBytes());
+ break;
+ case DATE:
+ gen.writeDate(reader.getDate());
+ break;
+ case DECIMAL:
+ gen.writeDecimal(reader.getDecimal());
+ break;
+ case FLOAT:
+ gen.writeDouble(reader.getFloat());
+ break;
+ case DOUBLE:
+ gen.writeDouble(reader.getDouble());
+ break;
+ case INTEGER:
+ gen.writeInt(reader.getInt());
+ break;
+ case LONG:
+ gen.writeBigInt(reader.getLong());
+ break;
+ case NULL:
+ gen.writeUntypedNull();
+ break;
+ case PERIOD:
+ gen.writeInterval(reader.getPeriod());
+ break;
+ case STRING:
+ gen.writeVarChar(reader.getString());
+ break;
+ case TIME:
+ gen.writeTime(reader.getTime());
+ break;
+ case TIMESTAMP:
+ gen.writeTimestamp(reader.getTimestamp());
+ break;
+ default:
+ throw new IllegalStateException(reader.valueType().name());
+ }
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
index 4992ac3..88dc115 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
@@ -249,7 +249,7 @@ public class TypeValidators {
super(name, description);
}
- public String getAdminUsers(OptionManager optionManager) {
+ public String getAdminUsers(OptionSet optionManager) {
String adminUsers = optionManager.getOption(ExecConstants.ADMIN_USERS_VALIDATOR);
// if this option has not been changed by the user then return the
// process user
@@ -273,7 +273,7 @@ public class TypeValidators {
super(name, description);
}
- public String getAdminUserGroups(OptionManager optionManager) {
+ public String getAdminUserGroups(OptionSet optionManager) {
String adminUserGroups = optionManager.getOption(ExecConstants.ADMIN_USER_GROUPS_VALIDATOR);
// if this option has not been changed by the user then return the
// process user groups
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/BaseQueryRunner.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/BaseQueryRunner.java
new file mode 100644
index 0000000..842cb8e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/BaseQueryRunner.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest;
+
+import java.util.Map;
+
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.tools.ValidationException;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserBitShared.QueryType;
+import org.apache.drill.exec.proto.UserProtos.QueryResultsMode;
+import org.apache.drill.exec.proto.UserProtos.RunQuery;
+import org.apache.drill.exec.rpc.UserClientConnection;
+import org.apache.drill.exec.rpc.user.InboundImpersonationManager;
+import org.apache.drill.exec.server.options.OptionSet;
+import org.apache.drill.exec.server.options.SessionOptionManager;
+import org.apache.drill.exec.store.SchemaTreeProvider;
+import org.apache.drill.exec.util.ImpersonationUtil;
+import org.apache.drill.exec.work.WorkManager;
+import org.apache.parquet.Strings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class BaseQueryRunner {
+ private static final Logger logger = LoggerFactory.getLogger(BaseQueryRunner.class);
+
+ protected final WorkManager workManager;
+ protected final WebUserConnection webUserConnection;
+ private final OptionSet options;
+ protected int maxRows;
+ protected QueryId queryId;
+
+ public BaseQueryRunner(final WorkManager workManager, final WebUserConnection webUserConnection) {
+ this.workManager = workManager;
+ this.webUserConnection = webUserConnection;
+ this.options = webUserConnection.getSession().getOptions();
+ this.maxRows = options.getInt(ExecConstants.QUERY_MAX_ROWS);
+ }
+
+ protected void applyUserName(String userName) {
+ if (!Strings.isNullOrEmpty(userName)) {
+ DrillConfig config = workManager.getContext().getConfig();
+ if (!config.getBoolean(ExecConstants.IMPERSONATION_ENABLED)) {
+ throw UserException.permissionError()
+ .message("User impersonation is not enabled")
+ .build(logger);
+ }
+ InboundImpersonationManager inboundImpersonationManager = new InboundImpersonationManager();
+ boolean isAdmin = !config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED) ||
+ ImpersonationUtil.hasAdminPrivileges(
+ webUserConnection.getSession().getCredentials().getUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options));
+ if (isAdmin) {
+ // Admin user can impersonate any user they want to (when authentication is disabled, all users are admin)
+ webUserConnection.getSession().replaceUserCredentials(
+ inboundImpersonationManager,
+ UserBitShared.UserCredentials.newBuilder().setUserName(userName).build());
+ } else {
+ // Check configured impersonation rules to see if this user is allowed to impersonate the given user
+ inboundImpersonationManager.replaceUserOnSession(userName, webUserConnection.getSession());
+ }
+ }
+ }
+
+ protected void applyOptions(Map<String, String> options) {
+ if (options != null) {
+ SessionOptionManager sessionOptionManager = webUserConnection.getSession().getOptions();
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ sessionOptionManager.setLocalOption(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ protected void applyDefaultSchema(String defaultSchema) throws ValidationException {
+ if (!Strings.isNullOrEmpty(defaultSchema)) {
+ SessionOptionManager options = webUserConnection.getSession().getOptions();
+ @SuppressWarnings("resource")
+ SchemaTreeProvider schemaTreeProvider = new SchemaTreeProvider(workManager.getContext());
+ SchemaPlus rootSchema = schemaTreeProvider.createRootSchema(options);
+ webUserConnection.getSession().setDefaultSchemaPath(defaultSchema, rootSchema);
+ }
+ }
+
+ protected void applyRowLimit(int limit) {
+ if (limit > 0 && maxRows > 0) {
+ maxRows = Math.min(limit, maxRows);
+ } else {
+ maxRows = Math.max(limit, maxRows);
+ }
+ }
+
+ protected void startQuery(QueryType queryType, String query, UserClientConnection clientConn) {
+ final RunQuery runQuery = RunQuery.newBuilder()
+ .setType(queryType)
+ .setPlan(query)
+ .setResultsMode(QueryResultsMode.STREAM_FULL)
+ .setAutolimitRowcount(maxRows)
+ .build();
+
+ // Submit user query to Drillbit work queue.
+ queryId = workManager.getUserWorker().submitWork(clientConn, runQuery);
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/BaseWebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/BaseWebUserConnection.java
index 42d93f2..f847323 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/BaseWebUserConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/BaseWebUserConnection.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.server.rest;
import java.net.SocketAddress;
+import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.exec.rpc.AbstractDisposableUserClientConnection;
import org.apache.drill.exec.rpc.ConnectionThrottle;
import org.apache.drill.exec.rpc.user.UserSession;
@@ -54,4 +55,27 @@ public abstract class BaseWebUserConnection extends AbstractDisposableUserClient
public WebSessionResources resources() {
return webSessionResources;
}
+
+ protected String webDataType(MajorType majorType) {
+ StringBuilder dataType = new StringBuilder(majorType.getMinorType().name());
+
+ // For DECIMAL type
+ if (majorType.hasPrecision()) {
+ dataType.append("(");
+ dataType.append(majorType.getPrecision());
+
+ if (majorType.hasScale()) {
+ dataType.append(", ");
+ dataType.append(majorType.getScale());
+ }
+
+ dataType.append(")");
+ } else if (majorType.hasWidth()) {
+ // Case for VARCHAR columns with specified width
+ dataType.append("(");
+ dataType.append(majorType.getWidth());
+ dataType.append(")");
+ }
+ return dataType.toString();
+ }
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
index 6e6005e..93d2d11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
@@ -26,6 +26,7 @@ import org.apache.drill.exec.server.rest.DrillRestServer.UserAuthEnabled;
import org.apache.drill.exec.server.rest.QueryWrapper.RestQueryBuilder;
import org.apache.drill.exec.server.rest.RestQueryRunner.QueryResult;
import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal;
+import org.apache.drill.exec.server.rest.stream.QueryRunner;
import org.apache.drill.exec.work.WorkManager;
import org.glassfish.jersey.server.mvc.Viewable;
import org.slf4j.Logger;
@@ -41,9 +42,14 @@ import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Form;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.StreamingOutput;
+
+import java.io.IOException;
+import java.io.OutputStream;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -90,14 +96,40 @@ public class QueryResources {
@Path("/query.json")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
- public QueryResult submitQueryJSON(QueryWrapper query) throws Exception {
+ public StreamingOutput submitQueryJSON(QueryWrapper query) throws Exception {
+
+ // Prior to Drill 1.18, REST queries would batch the entire result set
+ // in memory, limiting query size. In Drill 1.19 and later, results are
+ // streamed from the executor, to the JSON writer and to the HTTP connection
+ // with no buffering.
+ //
+ // For compatibility with Drill 1.18 and before, Drill places the query
+ // schema *after* the data. Any tool that needs to know the schema will
+ // need to buffer the entire result set to wait for the schema. An obvious
+ // improvement is to provide an option to send the schema *before* the data.
+ // One drawback of doing so is that the schema will report that of the first
+ // batch: Drill allows schema to change across batches and thus the schema
+ // of the JSON-encoded data would change. This is more a bug with how Drill
+ // handles schemas than a JSON issue. (ODBC and JDBC have the same issues.)
+ QueryRunner runner = new QueryRunner(work, webUserConnection);
try {
- // Run the query
- return new RestQueryRunner(query, work, webUserConnection).run();
- } finally {
- // no-op for authenticated user
- webUserConnection.cleanupSession();
+ runner.start(query);
+ } catch (Exception e) {
+ throw new WebApplicationException("Query submission failed", e);
}
+ return new StreamingOutput() {
+ @Override
+ public void write(OutputStream output)
+ throws IOException, WebApplicationException {
+ try {
+ runner.sendResults(output);
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new WebApplicationException("JSON query failed", e);
+ }
+ }
+ };
}
@POST
@@ -111,15 +143,20 @@ public class QueryResources {
@FormParam("defaultSchema") String defaultSchema,
Form form) throws Exception {
try {
- final QueryResult result = submitQueryJSON(
- new RestQueryBuilder()
- .query(query)
- .queryType(queryType)
- .rowLimit(autoLimit)
- .userName(userName)
- .defaultSchema(defaultSchema)
- .sessionOptions(readOptionsFromForm(form))
- .build());
+ // Run the query and wrap the result sets in a model to be
+ // transformed to HTML. This can be memory-intensive for larger
+ // queries.
+ QueryWrapper wrapper = new RestQueryBuilder()
+ .query(query)
+ .queryType(queryType)
+ .rowLimit(autoLimit)
+ .userName(userName)
+ .defaultSchema(defaultSchema)
+ .sessionOptions(readOptionsFromForm(form))
+ .build();
+ final QueryResult result = new RestQueryRunner(wrapper,
+ work, webUserConnection)
+ .run();
List<Integer> rowsPerPageValues = work.getContext().getConfig().getIntList(
ExecConstants.HTTP_WEB_CLIENT_RESULTSET_ROWS_PER_PAGE_VALUES);
Collections.sort(rowsPerPageValues);
@@ -128,6 +165,9 @@ public class QueryResources {
} catch (Exception | Error e) {
logger.error("Query from Web UI Failed: {}", e);
return ViewableWithPermissions.create(authEnabled.get(), "/rest/errorMessage.ftl", sc, e);
+ } finally {
+ // no-op for authenticated user
+ webUserConnection.cleanupSession();
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/RestQueryRunner.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/RestQueryRunner.java
index 69d81ae..826a9a8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/RestQueryRunner.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/RestQueryRunner.java
@@ -24,127 +24,45 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.tools.ValidationException;
-import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
-import org.apache.drill.exec.proto.UserProtos.QueryResultsMode;
-import org.apache.drill.exec.proto.UserProtos.RunQuery;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
-import org.apache.drill.exec.rpc.user.InboundImpersonationManager;
-import org.apache.drill.exec.server.options.SessionOptionManager;
-import org.apache.drill.exec.store.SchemaTreeProvider;
-import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.drill.exec.work.WorkManager;
-import org.apache.parquet.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class RestQueryRunner {
+public class RestQueryRunner extends BaseQueryRunner {
private static final Logger logger = LoggerFactory.getLogger(QueryWrapper.class);
private static final MemoryMXBean memMXBean = ManagementFactory.getMemoryMXBean();
private final QueryWrapper query;
- private final WorkManager workManager;
- private final WebUserConnection webUserConnection;
- private final SessionOptionManager options;
public RestQueryRunner(final QueryWrapper query, final WorkManager workManager, final WebUserConnection webUserConnection) {
+ super(workManager, webUserConnection);
this.query = query;
- this.workManager = workManager;
- this.webUserConnection = webUserConnection;
- this.options = webUserConnection.getSession().getOptions();
}
public QueryResult run() throws Exception {
- applyUserName();
- applyOptions();
- applyDefaultSchema();
- int maxRows = applyRowLimit();
- return submitQuery(maxRows);
+ applyUserName(query.getUserName());
+ applyOptions(query.getOptions());
+ applyDefaultSchema(query.getDefaultSchema());
+ applyRowLimit(query.getAutoLimitRowCount());
+ return submitQuery();
}
- private void applyUserName() {
- String userName = query.getUserName();
- if (!Strings.isNullOrEmpty(userName)) {
- DrillConfig config = workManager.getContext().getConfig();
- if (!config.getBoolean(ExecConstants.IMPERSONATION_ENABLED)) {
- throw UserException.permissionError()
- .message("User impersonation is not enabled")
- .build(logger);
- }
- InboundImpersonationManager inboundImpersonationManager = new InboundImpersonationManager();
- boolean isAdmin = !config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED) ||
- ImpersonationUtil.hasAdminPrivileges(
- webUserConnection.getSession().getCredentials().getUserName(),
- ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
- ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options));
- if (isAdmin) {
- // Admin user can impersonate any user they want to (when authentication is disabled, all users are admin)
- webUserConnection.getSession().replaceUserCredentials(
- inboundImpersonationManager,
- UserBitShared.UserCredentials.newBuilder().setUserName(userName).build());
- } else {
- // Check configured impersonation rules to see if this user is allowed to impersonate the given user
- inboundImpersonationManager.replaceUserOnSession(userName, webUserConnection.getSession());
- }
- }
- }
-
- private void applyOptions() {
- Map<String, String> options = query.getOptions();
- if (options != null) {
- SessionOptionManager sessionOptionManager = webUserConnection.getSession().getOptions();
- for (Map.Entry<String, String> entry : options.entrySet()) {
- sessionOptionManager.setLocalOption(entry.getKey(), entry.getValue());
- }
- }
- }
-
- private void applyDefaultSchema() throws ValidationException {
- String defaultSchema = query.getDefaultSchema();
- if (!Strings.isNullOrEmpty(defaultSchema)) {
- SessionOptionManager options = webUserConnection.getSession().getOptions();
- @SuppressWarnings("resource")
- SchemaTreeProvider schemaTreeProvider = new SchemaTreeProvider(workManager.getContext());
- SchemaPlus rootSchema = schemaTreeProvider.createRootSchema(options);
- webUserConnection.getSession().setDefaultSchemaPath(defaultSchema, rootSchema);
- }
- }
-
- private int applyRowLimit() {
- int defaultMaxRows = webUserConnection.getSession().getOptions().getInt(ExecConstants.QUERY_MAX_ROWS);
- int maxRows;
- int limit = query.getAutoLimitRowCount();
- if (limit > 0 && defaultMaxRows > 0) {
- maxRows = Math.min(limit, defaultMaxRows);
- } else {
- maxRows = Math.max(limit, defaultMaxRows);
- }
+ private QueryResult submitQuery() {
webUserConnection.setAutoLimitRowCount(maxRows);
- return maxRows;
- }
-
- public QueryResult submitQuery(int maxRows) {
- final RunQuery runQuery = RunQuery.newBuilder()
- .setType(QueryType.valueOf(query.getQueryType()))
- .setPlan(query.getQuery())
- .setResultsMode(QueryResultsMode.STREAM_FULL)
- .setAutolimitRowcount(maxRows)
- .build();
+ startQuery(QueryType.valueOf(query.getQueryType()),
+ query.getQuery(),
+ webUserConnection);
// Heap usage threshold/trigger to provide resiliency on web server for queries submitted via HTTP
double memoryFailureThreshold = workManager.getContext().getConfig().getDouble(ExecConstants.HTTP_MEMORY_HEAP_FAILURE_THRESHOLD);
- // Submit user query to Drillbit work queue.
- final QueryId queryId = workManager.getUserWorker().submitWork(webUserConnection, runQuery);
-
boolean isComplete = false;
boolean nearlyOutOfHeapSpace = false;
float usagePercent = getHeapUsage();
@@ -181,7 +99,6 @@ public class RestQueryRunner {
throw new UserRemoteException(webUserConnection.getError());
}
- // Return the QueryResult.
return new QueryResult(queryId, webUserConnection, webUserConnection.results);
}
@@ -213,4 +130,4 @@ public class RestQueryRunner {
return queryId;
}
}
-}
\ No newline at end of file
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
index 9427fc3..c81e5b6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java
@@ -91,26 +91,7 @@ public class WebUserConnection extends BaseWebUserConnection {
// DRILL-6847: This section adds query metadata to the REST results
MaterializedField col = schema.getColumn(i);
columns.add(col.getName());
- StringBuilder dataType = new StringBuilder(col.getType().getMinorType().name());
-
- // For DECIMAL type
- if (col.getType().hasPrecision()) {
- dataType.append("(");
- dataType.append(col.getType().getPrecision());
-
- if (col.getType().hasScale()) {
- dataType.append(", ");
- dataType.append(col.getType().getScale());
- }
-
- dataType.append(")");
- } else if (col.getType().hasWidth()) {
- // Case for VARCHAR columns with specified width
- dataType.append("(");
- dataType.append(col.getType().getWidth());
- dataType.append(")");
- }
- metadata.add(dataType.toString());
+ metadata.add(webDataType(col.getType()));
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/stream/QueryRunner.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/stream/QueryRunner.java
new file mode 100644
index 0000000..f921e4f
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/stream/QueryRunner.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest.stream;
+
+import java.io.OutputStream;
+
+import org.apache.calcite.tools.ValidationException;
+import org.apache.drill.exec.proto.UserBitShared.QueryType;
+import org.apache.drill.exec.server.rest.BaseQueryRunner;
+import org.apache.drill.exec.server.rest.QueryWrapper;
+import org.apache.drill.exec.server.rest.WebUserConnection;
+import org.apache.drill.exec.work.WorkManager;
+
+/**
+ * Query runner for streaming JSON results. This version is backward-compatible
+ * with pre-Drill 1.19 JSON queries.
+ */
+public class QueryRunner extends BaseQueryRunner {
+
+ private StreamingHttpConnection userConn;
+
+ public QueryRunner(final WorkManager workManager, final WebUserConnection webUserConnection) {
+ super(workManager, webUserConnection);
+ }
+
+ public void start(QueryWrapper query) throws ValidationException {
+ applyUserName(query.getUserName());
+ applyOptions(query.getOptions());
+ applyDefaultSchema(query.getDefaultSchema());
+ applyRowLimit(query.getAutoLimitRowCount());
+
+ userConn = new StreamingHttpConnection(webUserConnection.resources());
+ startQuery(QueryType.valueOf(query.getQueryType()),
+ query.getQuery(),
+ userConn);
+ userConn.onStart(queryId, maxRows);
+
+ // Query is now running in a separate thread, possibly sending
+ // the first batch to the user connection.
+ }
+
+ public void sendResults(OutputStream output) throws Exception {
+ try {
+ // Hand output stream to the user connection which will now
+ // stream batches
+ userConn.outputAvailable(output);
+
+ // Wait for the query to succeed or fail
+ try {
+ userConn.await();
+ userConn.finish();
+ } catch (InterruptedException e) {
+ // No-op
+ }
+ } finally {
+ // no-op for authenticated user
+ webUserConnection.cleanupSession();
+ }
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/stream/StreamingHttpConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/stream/StreamingHttpConnection.java
new file mode 100644
index 0000000..f283cd0
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/stream/StreamingHttpConnection.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest.stream;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.physical.impl.materialize.QueryDataPackage;
+import org.apache.drill.exec.physical.resultSet.PushResultSetReader;
+import org.apache.drill.exec.physical.resultSet.impl.PushResultSetReaderImpl;
+import org.apache.drill.exec.physical.resultSet.impl.PushResultSetReaderImpl.BatchHolder;
+import org.apache.drill.exec.physical.resultSet.util.JsonWriter;
+import org.apache.drill.exec.physical.rowSet.RowSetReader;
+import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.helper.QueryIdHelper;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.rpc.Acks;
+import org.apache.drill.exec.rpc.RpcOutcomeListener;
+import org.apache.drill.exec.server.rest.BaseWebUserConnection;
+import org.apache.drill.exec.server.rest.WebSessionResources;
+import org.apache.drill.exec.vector.complex.fn.JsonOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Stream the results of a query to a REST client as JSON, following the schema
+ * defined by {@code QueryResult} to maintain backward compatibility. That schema
+ * is not ideal for streaming, but it is what it is.
+ * <p>
+ * Streaming requires that we use a low-level JSON writer and handle serialization
+ * ourselves. While we could build up, then serialize objects, there is little
+ * advantage of doing so, and it would simply increase cost -- the very thing that
+ * this design tries to decrease.
+ * <p>
+ * Does minimal pretty-printing: just inserts newlines in a few places.
+ * <p>
+ * Enforces the row limit, discarding all rows after the requested limit.
+ * Trusts that the query will enforce the limit at the batch level to avoid
+ * wasting resources.
+ */
+public class StreamingHttpConnection extends BaseWebUserConnection {
+ private static final Logger logger = LoggerFactory.getLogger(StreamingHttpConnection.class);
+
+ private final CountDownLatch startSignal = new CountDownLatch(1);
+ private QueryId queryId;
+ private int rowLimit;
+ private int batchCount;
+ private int rowCount;
+ private OutputStream out;
+ private JsonWriter writer;
+ private BatchHolder batchHolder;
+ private PushResultSetReader reader;
+
+ public StreamingHttpConnection(WebSessionResources webSessionResources) {
+ super(webSessionResources);
+ }
+
+ /**
+ * Provide query info once the query starts. Sent from the REST request
+ * thread.
+ */
+ public void onStart(QueryId queryId, int rowLimit) {
+ this.queryId = queryId;
+ this.rowLimit = rowLimit;
+ }
+
+ /**
+ * Set the output stream. Sent from the REST request thread in the
+ * {@code StreamingOutput} callback once the output stream is
+ * available. Unblocks the Screen thread.
+ */
+ public void outputAvailable(OutputStream out) throws IOException {
+ this.out = out;
+ writer = new JsonWriter(out, false, false);
+ startSignal.countDown();
+ }
+
+ /**
+ * Called from query thread, specifically from the Screen operator,
+ * for each batch.
+ */
+ @Override
+ public void sendData(RpcOutcomeListener<Ack> listener, QueryDataPackage data) {
+ VectorContainer batch = data.batch();
+ try {
+ if (batchCount == 0) {
+ batchHolder = new BatchHolder(batch);
+ reader = new PushResultSetReaderImpl(batchHolder);
+ startSignal.await();
+ }
+ batchHolder.newBatch();
+ RowSetReader batchReader = reader.start();
+ emitBatch(batchReader);
+ batchCount++;
+ } catch (IOException e) {
+ throw UserException.dataWriteError(e)
+ .addContext("Failed to send JSON results to the REST client")
+ .build(logger);
+ } catch (InterruptedException e) {
+ throw new DrillRuntimeException("Interrupted", e);
+ } finally {
+ batch.zeroVectors();
+ listener.success(Acks.OK, null);
+ }
+ }
+
+ public void emitBatch(RowSetReader batchReader) throws IOException {
+ if (batchCount == 0) {
+ emitHeader(batchReader.tupleSchema());
+ }
+ if (rowLimit == 0 || rowCount < rowLimit) {
+ emitRows(batchReader);
+ }
+ }
+
+ /**
+ * Emit the JSON header. Must follow the same structure as
+ * {@code QueryResult} to ensure backward compatibility. However the order
+ * of fields can vary. In particular, we ensure metadata is sent first.
+ * <pre><code>
+ * {
+ * "queryId" : "21504178-fd2e-55f8-f1dc-c99dfa707db7",
+ * "columns" : [ "employee_id", "full_name" ],
+ * "metadata" : [ "BIGINT", "VARCHAR" ],
+ * "attemptedAutoLimit" : 0,
+ * "rows" : [ {
+ * </code></pre>
+ */
+ private void emitHeader(TupleMetadata rowSchema) throws IOException {
+ JsonOutput gen = writer.jsonOutput();
+ gen.writeStartObject();
+ gen.writeFieldName("queryId");
+ gen.writeVarChar(QueryIdHelper.getQueryId(queryId));
+ writeNewline(gen);
+ gen.writeFieldName("columns");
+ writeColNames(gen, rowSchema);
+ writeNewline(gen);
+ gen.writeFieldName("metadata");
+ writeColTypes(gen, rowSchema);
+ writeNewline(gen);
+ gen.writeFieldName("attemptedAutoLimit");
+ gen.writeInt(rowLimit);
+ writeNewline(gen);
+ gen.writeFieldName("rows");
+ gen.writeStartArray();
+ writeNewline(gen);
+
+ // The array of rows is now open, ready to send batches.
+ }
+
+ public void writeNewline(JsonOutput gen) throws IOException {
+ gen.flush();
+ out.write('\n');
+ }
+
+ private void writeColNames(JsonOutput gen, TupleMetadata rowSchema) throws IOException {
+ gen.writeStartArray();
+ for (ColumnMetadata col : rowSchema) {
+ gen.writeVarChar(col.name());
+ }
+ gen.writeEndArray();
+ }
+
+ private void writeColTypes(JsonOutput gen, TupleMetadata rowSchema) throws IOException {
+ gen.writeStartArray();
+ for (ColumnMetadata col : rowSchema) {
+ gen.writeVarChar(webDataType(col.majorType()));
+ }
+ gen.writeEndArray();
+ }
+
+ private void emitRows(RowSetReader batchReader) throws IOException {
+ while (batchReader.next()) {
+ writer.writeRow(batchReader);
+ writeNewline(writer.jsonOutput());
+ if (rowLimit > 0 && ++rowCount >= rowLimit) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Called from the REST request, after the query completes,
+ * to emit the end of the JSON payload:
+ * <pre><code>
+ * } ],
+ * "queryState" : "COMPLETED"
+ * }
+ * </code></pre>
+ * <p>
+ * Admittedly the tail is pretty lame, but we need it to maintain
+ * backward compatibility.
+ * <p>
+ * Note that, under the original design, there is no good way to report
+ * an error that occurs once the query starts running. Here, we report
+ * the query state, which can indicate an error. But, the design of the
+ * API does not currently provide the error message itself.
+ */
+ public void finish() throws IOException {
+ JsonOutput gen = writer.jsonOutput();
+ gen.writeEndArray();
+ writeNewline(gen);
+ gen.writeFieldName("queryState");
+ gen.writeVarChar(getQueryState());
+ writeNewline(gen);
+ gen.writeEndObject();
+ writeNewline(gen);
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimeValueListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimeValueListener.java
index 31d2fb7..6cc791f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimeValueListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/values/TimeValueListener.java
@@ -53,7 +53,7 @@ public class TimeValueListener extends ScalarListener {
case VALUE_STRING:
try {
LocalTime localTime = LocalTime.parse(tokenizer.stringValue(), TIME_FORMAT);
- writer.setInt((int) ((localTime.toNanoOfDay() + 500_000L) / 1_000_000L)); // round to milliseconds
+ writer.setTime(localTime);
} catch (Exception e) {
throw loader.dataConversionError(schema(), "string", tokenizer.stringValue());
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/reader/HeaderBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/reader/HeaderBuilder.java
index c561171..5ac3451 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/reader/HeaderBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/reader/HeaderBuilder.java
@@ -70,7 +70,6 @@ public class HeaderBuilder implements TextOutput {
* Prefix used to create numbered columns for missing
* headers. Typical names: column_1, column_2, ...
*/
-
public static final String ANONYMOUS_COLUMN_PREFIX = "column_";
private final Path filePath;
@@ -98,7 +97,6 @@ public class HeaderBuilder implements TextOutput {
public boolean endEmptyField() {
// Empty header will be rewritten to "column_<n>".
-
return endField();
}
@@ -149,7 +147,6 @@ public class HeaderBuilder implements TextOutput {
* @param header the original header
* @return the rewritten header, valid for SQL
*/
-
private String rewriteHeader(String header) {
final StringBuilder buf = new StringBuilder();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java
index 3d9d8f1..83e2115 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java
@@ -30,12 +30,12 @@ import org.apache.drill.exec.store.pcap.decoder.TcpSession;
import org.apache.drill.exec.store.pcap.schema.Schema;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.hadoop.mapred.FileSplit;
-import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
+import java.time.Instant;
import java.util.HashMap;
import java.util.Map;
@@ -407,7 +407,7 @@ public class PcapBatchReader implements ManagedReader<FileSchemaNegotiator> {
rowWriter.start();
typeWriter.setString(packet.getPacketType());
- timestampWriter.setTimestamp(new Instant(packet.getTimestamp()));
+ timestampWriter.setTimestamp(Instant.ofEpochMilli(packet.getTimestamp()));
timestampMicroWriter.setLong(packet.getTimestampMicro());
networkWriter.setInt(networkType);
srcMacAddressWriter.setString(packet.getEthernetSource());
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java
index 5900fd6..f67b85f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java
@@ -18,12 +18,12 @@
package org.apache.drill.exec.store.pcap.decoder;
-import org.joda.time.Instant;
import org.joda.time.Period;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetAddress;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -240,7 +240,7 @@ public class TcpSession {
}
public Instant getSessionStartTime() {
- return new Instant(startTime);
+ return Instant.ofEpochMilli(startTime);
}
public Period getSessionDuration() {
@@ -252,7 +252,7 @@ public class TcpSession {
}
public Instant getSessionEndTime() {
- return new Instant(endTime);
+ return Instant.ofEpochMilli(endTime);
}
public String getSrcMac() {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java
index ec9d5f5..4640d42 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.pcapng;
+import java.time.Instant;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -27,7 +28,6 @@ import org.apache.drill.common.types.Types;
import org.apache.drill.exec.store.pcap.PcapFormatUtils;
import org.apache.drill.exec.store.pcapng.decoder.PacketDecoder;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
-import org.joda.time.Instant;
import fr.bmartel.pcapdecoder.structure.options.inter.IOptionsStatisticsHeader;
import fr.bmartel.pcapdecoder.structure.types.IPcapngType;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/BasicJsonOutput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/BasicJsonOutput.java
index 8e248b6..d571af5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/BasicJsonOutput.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/BasicJsonOutput.java
@@ -31,8 +31,8 @@ import com.fasterxml.jackson.core.JsonGenerator;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
/**
- * A JSON output class that generates standard JSON. By default, literals are output such that they can be implicitly
- * cast.
+ * A JSON output class that generates standard JSON. By default, literals are
+ * output such that they can be implicitly cast.
*/
public class BasicJsonOutput implements JsonOutput {
@@ -52,21 +52,21 @@ public class BasicJsonOutput implements JsonOutput {
this.gen = gen;
switch (dateOutput) {
- case SQL: {
- dateFormatter = DateUtility.formatDate;
- timeFormatter = DateUtility.formatTime;
- timestampFormatter = DateUtility.formatTimeStamp;
- break;
- }
- case ISO: {
- dateFormatter = DateUtility.isoFormatDate;
- timeFormatter = DateUtility.isoFormatTime;
- timestampFormatter = DateUtility.isoFormatTimeStamp;
- break;
- }
-
- default:
- throw new UnsupportedOperationException(String.format("Unable to support date output of type %s.", dateOutput));
+ case SQL: {
+ dateFormatter = DateUtility.formatDate;
+ timeFormatter = DateUtility.formatTime;
+ timestampFormatter = DateUtility.formatTimeStamp;
+ break;
+ }
+ case ISO: {
+ dateFormatter = DateUtility.isoFormatDate;
+ timeFormatter = DateUtility.isoFormatTime;
+ timestampFormatter = DateUtility.isoFormatTimeStamp;
+ break;
+ }
+
+ default:
+ throw new UnsupportedOperationException(String.format("Unable to support date output of type %s.", dateOutput));
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/ExtendedJsonOutput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/ExtendedJsonOutput.java
index 9b509e7..5dd9898 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/ExtendedJsonOutput.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/ExtendedJsonOutput.java
@@ -29,8 +29,9 @@ import org.joda.time.Period;
import com.fasterxml.jackson.core.JsonGenerator;
/**
- * Writes JSON Output that will wrap Binary, Date, Time, Timestamp, Integer, Decimal and Interval types with wrapping
- * maps for better type resolution upon deserialization.
+ * Writes JSON Output that will wrap Binary, Date, Time, Timestamp, Integer,
+ * Decimal and Interval types with wrapping maps for better type resolution upon
+ * deserialization.
*/
public class ExtendedJsonOutput extends BasicJsonOutput {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonOutput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonOutput.java
index 572386b..f1012e6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonOutput.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonOutput.java
@@ -25,8 +25,9 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
import org.joda.time.Period;
/**
- * Interface through which UDFs, RecordWriters and other systems can write out a JSON output.
- * Generally used to control how non-json types are mapped to a json output stream.
+ * Interface through which UDFs, RecordWriters and other systems can write out a
+ * JSON output. Generally used to control how non-JSON types are mapped to a
+ * JSON output stream.
*/
public interface JsonOutput {
@@ -39,7 +40,6 @@ public interface JsonOutput {
void writeUntypedNull() throws IOException;
void writeFieldName(String name) throws IOException;
-
// literals
void writeDecimal(BigDecimal value) throws IOException;
void writeTinyInt(byte value) throws IOException;
@@ -72,7 +72,6 @@ public interface JsonOutput {
void writeTimestampNull() throws IOException;
void writeIntervalNull() throws IOException;
-
// scalars reader
void writeDecimal(FieldReader reader) throws IOException;
void writeTinyInt(FieldReader reader) throws IOException;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java
index 4633267..269293e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonWriter.java
@@ -23,28 +23,29 @@ import java.io.OutputStream;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonGenerator;
public class JsonWriter {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonWriter.class);
+ static final Logger logger = LoggerFactory.getLogger(JsonWriter.class);
private final JsonFactory factory = new JsonFactory();
private final JsonOutput gen;
- public JsonWriter(OutputStream out, boolean pretty, boolean useExtendedOutput) throws IOException{
- JsonGenerator writer = factory.configure(JsonGenerator.Feature.QUOTE_NON_NUMERIC_NUMBERS, false).createJsonGenerator(out);
- if(pretty){
+ public JsonWriter(OutputStream out, boolean pretty, boolean useExtendedOutput) throws IOException {
+ JsonGenerator writer = factory.configure(JsonGenerator.Feature.QUOTE_NON_NUMERIC_NUMBERS, false).createGenerator(out);
+ if (pretty) {
writer = writer.useDefaultPrettyPrinter();
}
- if(useExtendedOutput){
+ if (useExtendedOutput) {
gen = new ExtendedJsonOutput(writer);
- }else{
+ } else {
gen = new BasicJsonOutput(writer);
}
-
}
public JsonWriter(JsonOutput gen) {
@@ -64,7 +65,6 @@ public class JsonWriter {
case OPTIONAL:
case REQUIRED:
-
switch (mt) {
case FLOAT4:
gen.writeFloat(reader);
@@ -147,7 +147,6 @@ public class JsonWriter {
case VARCHAR:
gen.writeVarChar(reader);
break;
-
}
break;
@@ -158,7 +157,6 @@ public class JsonWriter {
for(int i = 0; i < reader.size(); i++){
gen.writeFloat(i, reader);
}
-
break;
case FLOAT8:
for(int i = 0; i < reader.size(); i++){
@@ -270,7 +268,5 @@ public class JsonWriter {
gen.writeEndArray();
break;
}
-
}
-
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java
index 748a8fc..27d589a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestSchemaCommands.java
@@ -32,7 +32,6 @@ import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterFixtureBuilder;
import org.apache.drill.test.ClusterTest;
import org.apache.hadoop.fs.Path;
-import org.joda.time.LocalDate;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -42,6 +41,7 @@ import org.junit.rules.ExpectedException;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.time.LocalDate;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java
index 3bc0820..dd1f741 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/metastore/TestMetastoreWithEasyFormatPlugin.java
@@ -79,7 +79,7 @@ public class TestMetastoreWithEasyFormatPlugin extends ClusterTest {
"`o_custkey` INT not null, " +
"`o_orderstatus` VARCHAR not null, " +
"`o_totalprice` DOUBLE not null, " +
- "`o_orderdate` DATE not null format \"yyyy-MM-dd''T''HH:mm:ss.SSSZZ\", " +
+ "`o_orderdate` DATE not null format \"yyyy-MM-dd''T''HH:mm:ss.SSSXXX\", " +
"`o_orderpriority` VARCHAR not null, " +
"`o_clerk` VARCHAR not null, " +
"`o_shippriority` INT not null, " +
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/convert/TestDirectConverter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/convert/TestDirectConverter.java
index 1615827..418a2de 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/convert/TestDirectConverter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/convert/TestDirectConverter.java
@@ -23,6 +23,12 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
import java.util.HashMap;
import java.util.Map;
@@ -47,10 +53,6 @@ import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.ValueWriter;
import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSetUtilities;
-import org.joda.time.DateTimeZone;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
import org.junit.Test;
@@ -484,17 +486,14 @@ public class TestDirectConverter extends SubOperatorTest {
ConversionTestFixture testFixture = new ConversionTestFixture(fixture.allocator(), outputSchema);
testFixture.createConvertersFor(inputSchema);
RowSet actual = testFixture
- .addRow("2019-03-28", "12:34:56", "2019-03-28T12:34:56Z")
- .addRow("2019-03-28", "12:34:56", "2019-03-28T12:34:56Z")
+ .addRow("2019-03-28", "12:34:56", "2019-03-28T12:34:56")
.build();
- LocalTime lt = new LocalTime(12, 34, 56);
- LocalDate ld = new LocalDate(2019, 3, 28);
- Instant ts = ld.toDateTime(lt, DateTimeZone.UTC).toInstant();
+ LocalTime lt = LocalTime.of(12, 34, 56);
+ LocalDate ld = LocalDate.of(2019, 3, 28);
+ Instant ts = LocalDateTime.of(ld, lt).toInstant(ZoneOffset.UTC);
final SingleRowSet expected = fixture.rowSetBuilder(outputSchema)
.addRow(ld, lt, ts)
- .addRow(ld.toDateTimeAtStartOfDay(DateTimeZone.UTC).toInstant().getMillis(),
- lt.getMillisOfDay(), ts.getMillis())
.build();
RowSetUtilities.verify(expected, actual);
@@ -511,7 +510,7 @@ public class TestDirectConverter extends SubOperatorTest {
outputSchema.metadata("date").setFormat("M/d/yyyy");
outputSchema.metadata("time").setFormat("hh:mm:ss a");
- outputSchema.metadata("ts").setFormat("M/d/yyyy hh:mm:ss a Z");
+ outputSchema.metadata("ts").setFormat("M/d/yyyy hh:mm:ss a VV");
TupleMetadata inputSchema = new SchemaBuilder()
.add("date", MinorType.VARCHAR)
@@ -523,16 +522,13 @@ public class TestDirectConverter extends SubOperatorTest {
testFixture.createConvertersFor(inputSchema);
RowSet actual = testFixture
.addRow("3/28/2019", "12:34:56 PM", "3/28/2019 12:34:56 PM Z")
- .addRow("3/28/2019", "12:34:56 PM", "3/28/2019 12:34:56 PM Z")
.build();
- LocalTime lt = new LocalTime(12, 34, 56);
- LocalDate ld = new LocalDate(2019, 3, 28);
- Instant ts = ld.toDateTime(lt, DateTimeZone.UTC).toInstant();
+ LocalTime lt = LocalTime.of(12, 34, 56);
+ LocalDate ld = LocalDate.of(2019, 3, 28);
+ Instant ts = LocalDateTime.of(ld, lt).toInstant(ZoneOffset.UTC);
final SingleRowSet expected = fixture.rowSetBuilder(outputSchema)
.addRow(ld, lt, ts)
- .addRow(ld.toDateTimeAtStartOfDay(DateTimeZone.UTC).toInstant().getMillis(),
- lt.getMillisOfDay(), ts.getMillis())
.build();
RowSetUtilities.verify(expected, actual);
@@ -553,9 +549,9 @@ public class TestDirectConverter extends SubOperatorTest {
.add("ts", MinorType.TIMESTAMP)
.buildSchema();
- LocalTime lt = new LocalTime(12, 34, 56);
- LocalDate ld = new LocalDate(2019, 3, 28);
- Instant ts = ld.toDateTime(lt, DateTimeZone.UTC).toInstant();
+ LocalTime lt = LocalTime.of(12, 34, 56);
+ LocalDate ld = LocalDate.of(2019, 3, 28);
+ Instant ts = LocalDateTime.of(ld, lt).atZone(ZoneId.systemDefault()).toInstant();
ConversionTestFixture testFixture = new ConversionTestFixture(fixture.allocator(), outputSchema);
testFixture.createConvertersFor(inputSchema);
RowSet actual = testFixture
@@ -563,7 +559,7 @@ public class TestDirectConverter extends SubOperatorTest {
.build();
final SingleRowSet expected = fixture.rowSetBuilder(outputSchema)
- .addRow("2019-03-28", "12:34:56.000", "2019-03-28T12:34:56.000Z")
+ .addRow("2019-03-28", "12:34:56", "2019-03-28T12:34:56")
.build();
RowSetUtilities.verify(expected, actual);
@@ -572,7 +568,6 @@ public class TestDirectConverter extends SubOperatorTest {
/**
* Test conversion to/from Java-style Booleans.
*/
-
@Test
public void testBooleanToFromString() {
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java
index 5aeea22..32cddc0 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestExternalSort.java
@@ -44,9 +44,6 @@ public class TestExternalSort extends BaseTestQuery {
* This test simply verifies that the sort handles these types, even though
* Drill does not.
*
- * @param testLegacy
- * true to test the old (pre-1.11) sort, false to test the new (1.11
- * and later) sort
* @throws Exception
*/
@Test
@@ -66,7 +63,7 @@ public class TestExternalSort extends BaseTestQuery {
final RowSet rowSet = rowSetBuilder.build();
final File tableFile = createTableFile(tableDirName, "a.json");
- new JsonFileBuilder(rowSet).build(tableFile);
+ new JsonFileBuilder(rowSet).prettyPrint(false).build(tableFile);
rowSet.clear();
}
@@ -83,7 +80,6 @@ public class TestExternalSort extends BaseTestQuery {
final RowSet rowSet = rowSetBuilder.build();
final File tableFile = createTableFile(tableDirName, "b.json");
new JsonFileBuilder(rowSet)
- .setCustomFormatter("a", "%.2f")
.build(tableFile);
rowSet.clear();
}
@@ -137,7 +133,6 @@ public class TestExternalSort extends BaseTestQuery {
final RowSet rowSet = rowSetBuilder.build();
final File tableFile = createTableFile(tableDirName, "b.json");
new JsonFileBuilder(rowSet)
- .setCustomFormatter("a", "\"%05d\"")
.build(tableFile);
rowSet.clear();
}
@@ -216,7 +211,6 @@ public class TestExternalSort extends BaseTestQuery {
builder.go();
// TODO: Useless test: just dumps to console
-
test("select * from dfs.`%s` order by a desc", tableDirName);
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/TestFillEmpties.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/TestFillEmpties.java
index 6ccb78a..b918b3c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/TestFillEmpties.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/TestFillEmpties.java
@@ -49,7 +49,6 @@ import org.junit.experimental.categories.Category;
* Note that this test also has the handy side-effect of testing
* null handling in the accessor classes.
*/
-
@Category(RowSetTests.class)
public class TestFillEmpties extends SubOperatorTest {
@@ -62,7 +61,6 @@ public class TestFillEmpties extends SubOperatorTest {
* the alternatives which are either 1) leave the value as
* garbage, or 2) raise an exception about the missing value.
*/
-
@Test
public void testFillEmptiesRequired() {
testFillEmpties(DataMode.REQUIRED);
@@ -73,7 +71,6 @@ public class TestFillEmpties extends SubOperatorTest {
* "natural" type for omitted values.
* Nullable vectors fill empties with nulls.
*/
-
@Test
public void testFillEmptiesNullable() {
testFillEmpties(DataMode.OPTIONAL);
@@ -84,7 +81,6 @@ public class TestFillEmpties extends SubOperatorTest {
* Drill defines a null (omitted) array as the same thing as
* a zero-length array.
*/
-
@Test
public void testFillEmptiesRepeated() {
testFillEmpties(DataMode.REPEATED);
@@ -190,14 +186,14 @@ public class TestFillEmpties extends SubOperatorTest {
assertTrue(reader.next());
if (i % 5 != 0) {
if (nullable) {
- // Nullable types fill with nulls.
+ // Nullable types fill with nulls.
assertTrue(colReader.isNull());
continue;
}
if (isVariable) {
- // Variable width types fill with a zero-length value.
+ // Variable width types fill with a zero-length value.
assertEquals(0, colReader.getBytes().length);
continue;
}
@@ -205,7 +201,6 @@ public class TestFillEmpties extends SubOperatorTest {
// All other types fill with zero-bytes, interpreted as some form
// of zero for each type.
-
Object actual = colReader.getObject();
Object expected = RowSetUtilities.testDataFromInt(valueType, majorType,
i % 5 == 0 ? i : 0);
@@ -226,8 +221,8 @@ public class TestFillEmpties extends SubOperatorTest {
ValueType valueType = colWriter.valueType();
for (int i = 0; i < ROW_COUNT; i++) {
if (i % 5 == 0) {
- // Write two values so we can exercise a bit of the array logic.
+ // Write two values so we can exercise a bit of the array logic.
colWriter.setObject(RowSetUtilities.testDataFromInt(valueType, majorType, i));
colWriter.setObject(RowSetUtilities.testDataFromInt(valueType, majorType, i+1));
}
@@ -240,8 +235,8 @@ public class TestFillEmpties extends SubOperatorTest {
for (int i = 0; i < ROW_COUNT; i++) {
assertTrue(reader.next());
if (i % 5 != 0) {
- // Empty arrays are defined to be the same as a zero-length array.
+ // Empty arrays are defined to be the same as a zero-length array.
assertEquals(0, aReader.size());
} else {
for (int j = 0; j < 2; j++) {
@@ -263,7 +258,6 @@ public class TestFillEmpties extends SubOperatorTest {
* the default value to string, which is awkward for some types when
* using the generic "test value from int" tool.
*/
-
@Test
public void testDefaultValue() {
doTestDefaultValue(Types.required(MinorType.VARCHAR));
@@ -327,7 +321,6 @@ public class TestFillEmpties extends SubOperatorTest {
* for the column writer from the (properly formed) default value in the
* column metadata.
*/
-
@Test
public void testDefaultInSchema() {
TupleMetadata schema = new SchemaBuilder()
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/TestScalarAccessors.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/TestScalarAccessors.java
index 7f990c8..c1e803b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/TestScalarAccessors.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/TestScalarAccessors.java
@@ -26,6 +26,11 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
import java.util.Arrays;
import org.apache.drill.categories.RowSetTests;
@@ -58,10 +63,6 @@ import org.apache.drill.exec.vector.accessor.ValueType;
import org.apache.drill.exec.vector.complex.RepeatedValueVector;
import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSetUtilities;
-import org.joda.time.DateTimeZone;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -554,9 +555,10 @@ public class TestScalarAccessors extends SubOperatorTest {
rs.clear();
}
- private void nullableDoubleTester(MinorType type) {
+ @Test
+ public void testNullableFloat() {
TupleMetadata schema = new SchemaBuilder()
- .addNullable("col", type)
+ .addNullable("col", MinorType.FLOAT4)
.buildSchema();
SingleRowSet rs = fixture.rowSetBuilder(schema)
.addRow(10F)
@@ -570,6 +572,7 @@ public class TestScalarAccessors extends SubOperatorTest {
assertTrue(reader.next());
assertFalse(colReader.isNull());
+ assertEquals(10, colReader.getFloat(), 0.000001);
assertEquals(10, colReader.getDouble(), 0.000001);
assertTrue(reader.next());
@@ -579,6 +582,7 @@ public class TestScalarAccessors extends SubOperatorTest {
// Data value is undefined, may be garbage
assertTrue(reader.next());
+ assertEquals(30, colReader.getFloat(), 0.000001);
assertEquals(30, colReader.getDouble(), 0.000001);
assertFalse(reader.next());
@@ -586,13 +590,9 @@ public class TestScalarAccessors extends SubOperatorTest {
}
@Test
- public void testNullableFloat() {
- nullableDoubleTester(MinorType.FLOAT8);
- }
-
- private void doubleArrayTester(MinorType type) {
+ public void testFloatArray() {
TupleMetadata schema = new SchemaBuilder()
- .addArray("col", type)
+ .addArray("col", MinorType.FLOAT4)
.buildSchema();
SingleRowSet rs = fixture.rowSetBuilder(schema)
.addSingleCol(new double[] {})
@@ -603,7 +603,7 @@ public class TestScalarAccessors extends SubOperatorTest {
RowSetReader reader = rs.reader();
ArrayReader arrayReader = reader.array(0);
ScalarReader colReader = arrayReader.scalar();
- assertEquals(ValueType.DOUBLE, colReader.valueType());
+ assertEquals(ValueType.FLOAT, colReader.valueType());
assertTrue(reader.next());
assertEquals(0, arrayReader.size());
@@ -613,33 +613,28 @@ public class TestScalarAccessors extends SubOperatorTest {
assertTrue(arrayReader.next());
assertEquals(0, colReader.getDouble(), 0.00001);
- assertEquals(0, (double) colReader.getObject(), 0.00001);
+ assertEquals(0, (float) colReader.getObject(), 0.00001);
assertEquals("0.0", colReader.getAsString());
assertTrue(arrayReader.next());
assertEquals(20.5, colReader.getDouble(), 0.00001);
- assertEquals(20.5, (double) colReader.getObject(), 0.00001);
+ assertEquals(20.5, (float) colReader.getObject(), 0.00001);
assertEquals("20.5", colReader.getAsString());
assertTrue(arrayReader.next());
assertEquals(30.0, colReader.getDouble(), 0.00001);
- assertEquals(30.0, (double) colReader.getObject(), 0.00001);
+ assertEquals(30.0, (float) colReader.getObject(), 0.00001);
assertEquals("30.0", colReader.getAsString());
assertFalse(arrayReader.next());
assertEquals("[0.0, 20.5, 30.0]", arrayReader.getAsString());
- assertEquals(Arrays.asList(0.0D, 20.5D, 30D), arrayReader.getObject());
+ assertEquals(Arrays.asList(0.0F, 20.5F, 30F), arrayReader.getObject());
assertFalse(reader.next());
rs.clear();
}
@Test
- public void testFloatArray() {
- doubleArrayTester(MinorType.FLOAT8);
- }
-
- @Test
public void testDoubleRW() {
TupleMetadata schema = new SchemaBuilder()
.add("col", MinorType.FLOAT8)
@@ -677,12 +672,34 @@ public class TestScalarAccessors extends SubOperatorTest {
@Test
public void testNullableDouble() {
- nullableDoubleTester(MinorType.FLOAT8);
- }
+ TupleMetadata schema = new SchemaBuilder()
+ .addNullable("col", MinorType.FLOAT8)
+ .buildSchema();
+ SingleRowSet rs = fixture.rowSetBuilder(schema)
+ .addRow(10D)
+ .addSingleCol(null)
+ .addRow(30D)
+ .build();
+ assertEquals(3, rs.rowCount());
- @Test
- public void testDoubleArray() {
- doubleArrayTester(MinorType.FLOAT8);
+ RowSetReader reader = rs.reader();
+ ScalarReader colReader = reader.scalar(0);
+
+ assertTrue(reader.next());
+ assertFalse(colReader.isNull());
+ assertEquals(10, colReader.getDouble(), 0.000001);
+
+ assertTrue(reader.next());
+ assertTrue(colReader.isNull());
+ assertNull(colReader.getObject());
+ assertEquals("null", colReader.getAsString());
+ // Data value is undefined, may be garbage
+
+ assertTrue(reader.next());
+ assertEquals(30, colReader.getDouble(), 0.000001);
+
+ assertFalse(reader.next());
+ rs.clear();
}
@Test
@@ -831,7 +848,6 @@ public class TestScalarAccessors extends SubOperatorTest {
* as explained in the Javadoc for
* @{link org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriterImpl}
*/
-
@Test
public void testEmptyVarcharArray() {
TupleMetadata schema = new SchemaBuilder()
@@ -880,7 +896,6 @@ public class TestScalarAccessors extends SubOperatorTest {
/**
* Test the low-level interval-year utilities used by the column accessors.
*/
-
@Test
public void testIntervalYearUtils() {
{
@@ -1028,7 +1043,6 @@ public class TestScalarAccessors extends SubOperatorTest {
/**
* Test the low-level interval-day utilities used by the column accessors.
*/
-
@Test
public void testIntervalDayUtils() {
{
@@ -1698,7 +1712,7 @@ public class TestScalarAccessors extends SubOperatorTest {
.add("col", MinorType.DATE)
.buildSchema();
- LocalDate v1 = new LocalDate(2019, 3, 24);
+ LocalDate v1 = LocalDate.of(2019, 3, 24);
SingleRowSet rs = fixture.rowSetBuilder(schema)
.addSingleCol(v1)
@@ -1722,7 +1736,7 @@ public class TestScalarAccessors extends SubOperatorTest {
.add("col", MinorType.TIME)
.buildSchema();
- LocalTime v1 = new LocalTime(12, 13, 14);
+ LocalTime v1 = LocalTime.of(12, 13, 14);
SingleRowSet rs = fixture.rowSetBuilder(schema)
.addSingleCol(v1)
@@ -1746,9 +1760,9 @@ public class TestScalarAccessors extends SubOperatorTest {
.add("col", MinorType.TIMESTAMP)
.buildSchema();
- LocalDate dt = new LocalDate(2019, 3, 24);
- LocalTime lt = new LocalTime(12, 13, 14);
- Instant v1 = dt.toDateTime(lt, DateTimeZone.UTC).toInstant();
+ LocalDate dt = LocalDate.of(2019, 3, 24);
+ LocalTime lt = LocalTime.of(12, 13, 14);
+ Instant v1 = LocalDateTime.of(dt, lt).toInstant(ZoneOffset.UTC);
SingleRowSet rs = fixture.rowSetBuilder(schema)
.addSingleCol(v1)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/FileVerifier.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/FileVerifier.java
new file mode 100644
index 0000000..ec08aa8
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/FileVerifier.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.net.URL;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * Verifier for execution plans. A handy tool to ensure that the
+ * planner produces the expected plan given some set of conditions.
+ * <p>
+ * The test works by comparing the actual {@code EXPLAIN} output
+ * to a "golden" file with the expected ("golden") plan.
+ * <p>
+ * To create a test, just write it and let it fail due to a missing file.
+ * The output will go to the console. Inspect it. If it looks good,
+ * copy the plan to the golden file and run again.
+ * <p>
+ * If comparison fails, you can optionally ask the verifier to write the
+ * output to {@code /tmp/drill/test} so you can compare the golden and actual
+ * outputs using your favorite diff tool to understand changes. If the changes
+ * are expected, use that same IDE to copy changes from the actual
+ * to the golden file.
+ * <p>
+ * The JSON properties of the serialized classes are all controlled
+ * to have a fixed order to ensure that files compare across test
+ * runs. If you see spurious failures do to changed JSON order, consider
+ * adding a {@code @JsonPropertyOrder} tag to enforce a consistent order.
+ * <p>
+ * Lines can be prefixed with "!" to indicate that they are a regex.
+ * The pattern "!!" is treated as a non-regex line that starts with a bang.
+ * The pattern "!\!" is a regex that starts with a bang.
+ */
+public class FileVerifier {
+
+ private final String basePath;
+
+ public FileVerifier(String goldenFileDir) {
+ if (!goldenFileDir.startsWith("/")) {
+ goldenFileDir = "/" + goldenFileDir;
+ }
+ if (!goldenFileDir.endsWith("/")) {
+ goldenFileDir += "/";
+ }
+ this.basePath = goldenFileDir;
+ }
+
+ public void verifyFileWithResource(File actual, String relativePath) {
+ URL url = getClass().getResource(basePath + relativePath);
+ assertNotNull("Golden file is missing: " + relativePath, url);
+ File resource = new File(url.getPath());
+ verifyFiles(actual, resource);
+ }
+
+ public void verifyFiles(File actual, File resource) {
+ try (Reader expected = new FileReader(resource);
+ Reader aReader = new FileReader(actual)) {
+ verify(aReader, expected);
+ } catch (FileNotFoundException e) {
+ fail("Missing resource file: " + resource.getAbsolutePath());
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ }
+
+ public void verify(String actual, File resource) {
+ try (Reader expected = new FileReader(resource)) {
+ verify(new StringReader(actual), expected);
+ } catch (FileNotFoundException e) {
+ fail("Missing resource file: " + resource.getAbsolutePath());
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ }
+
+ private void verify(Reader actualReader, Reader expectedReader) throws IOException {
+ try (BufferedReader actual = new BufferedReader(actualReader);
+ BufferedReader expected = new BufferedReader(expectedReader);) {
+ for (int lineNo = 1; ;lineNo++ ) {
+ String aLine = actual.readLine();
+ String eLine = expected.readLine();
+ if (aLine == null && eLine == null) {
+ break;
+ }
+ if (eLine == null) {
+ fail("Too many actual lines");
+ }
+ if (aLine == null) {
+ fail("Missing actual lines");
+ }
+ compareLines(lineNo, eLine, aLine);
+ }
+ }
+ }
+
+ private void compareLines(int lineNo, String eLine, String aLine) {
+ if (eLine.startsWith("!!")) {
+ assertEquals("Line: " + lineNo, eLine.substring(1), aLine);
+ } else if (eLine.startsWith("!")) {
+ compareRegex(lineNo, eLine, aLine);
+ } else {
+ assertEquals("Line: " + lineNo, eLine, aLine);
+ }
+ }
+
+ private void compareRegex(int lineNo, String eLine, String aLine) {
+ if (eLine.startsWith("!\\!")) {
+ eLine = eLine.substring(2);
+ } else {
+ eLine = eLine.substring(1);
+ }
+ try {
+ Pattern p = Pattern.compile(eLine);
+ Matcher m = p.matcher(aLine);
+ assertTrue("Pattern match failed at line: " + lineNo, m.matches());
+ } catch (PatternSyntaxException e) {
+ fail("Bad regex pattern at line " + lineNo + ": " + e.getMessage());
+ }
+ }
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestResponseHeaders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestResponseHeaders.java
index d57e438..2f1f8ec 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestResponseHeaders.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestResponseHeaders.java
@@ -38,6 +38,7 @@ public class TestResponseHeaders extends ClusterTest {
private static final String BASE_URL = "";
+ @SuppressWarnings("serial")
@BeforeClass
public static void setUp() throws Exception {
ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestRestJson.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestRestJson.java
new file mode 100644
index 0000000..b4ca496
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/TestRestJson.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.server.rest;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.proto.UserBitShared.QueryType;
+import org.apache.drill.exec.store.easy.text.TextFormatPlugin.TextFormatConfig;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.ClusterTest;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+/**
+ * Simple sanity test of the Drill REST client query JSON feature.
+ * In Drill 1.18, this test verified the original, buffered JSON
+ * output. In Drill 1.19, it verifies the streamed output (though only
+ * for a small query.) A large test is available below, but is normally
+ * disabled.
+ * <p>
+ * Ad-hoc tests can be run by setting a breakpoint in one of the tests,
+ * then run the test in the debugger to start the REST server. Use your
+ * favorite browser-based tool:
+ * <ul>
+ * <li>Method: <tt>POST</tt></li>
+ * <li>Custom header: <tt>content-type: application/json</tt></li>
+ * <li>URL: <tt>http://localhost:8047/query.json</tt></li>
+ * <li>Body: <pre><code>
+ * {"query": "SELECT * FROM cp.`employee.json` LIMIT 20",
+ * "queryType":"SQL"}</code></pre><li>
+ * </ul>
+ */
+public class TestRestJson extends ClusterTest {
+
+ public static final MediaType JSON_MEDIA_TYPE =
+ MediaType.parse("application/json");
+ public static final int TIMEOUT = 3000; // for debugging
+
+ protected static File testDir;
+ private final OkHttpClient httpClient = new OkHttpClient.Builder()
+ .connectTimeout(TIMEOUT, TimeUnit.SECONDS)
+ .writeTimeout(TIMEOUT, TimeUnit.SECONDS)
+ .readTimeout(TIMEOUT, TimeUnit.SECONDS)
+ .build();
+
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final FileVerifier verifier = new FileVerifier("/rest");
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ ClusterFixtureBuilder builder = new ClusterFixtureBuilder(dirTestWatcher);
+ builder.configBuilder().put(ExecConstants.HTTP_ENABLE, true);
+ startCluster(builder);
+
+ // Set up CSV storage plugin using headers.
+ TextFormatConfig csvFormat = new TextFormatConfig(
+ null,
+ null, // line delimiter
+ null, // field delimiter
+ null, // quote
+ null, // escape
+ null, // comment
+ false, // skipFirstLine,
+ true // extractHeader
+ );
+
+ testDir = cluster.makeDataDir("data", "csv", csvFormat);
+ }
+
+ @Test
+ public void testSmallQuery() throws IOException {
+ File outFile = new File(dirTestWatcher.getTmpDir(), "small.json");
+ String sql = "SELECT * FROM cp.`employee.json` LIMIT 20";
+ QueryWrapper query = new QueryWrapper(sql, QueryType.SQL.name(),
+ "10", null, null, null);
+ runQuery(query, outFile);
+ verifier.verifyFileWithResource(outFile, "small.json");
+ }
+
+ @Test
+ public void testNoLimit() throws IOException {
+ File outFile = new File(dirTestWatcher.getTmpDir(), "cust20.json");
+ String sql = "SELECT * FROM cp.`employee.json` LIMIT 20";
+ QueryWrapper query = new QueryWrapper(sql, QueryType.SQL.name(),
+ null, null, null, null);
+ runQuery(query, outFile);
+ verifier.verifyFileWithResource(outFile, "cust20.json");
+ }
+
+ @Test
+ public void testFailedQuery() throws IOException {
+ File outFile = new File(dirTestWatcher.getTmpDir(), "failed.json");
+ String sql = "SELECT * FROM cp.`employee.json` LIMIT 20";
+ QueryWrapper query = new QueryWrapper(sql, QueryType.SQL.name(),
+ null, "bogusUser", null, null);
+ runQuery(query, outFile);
+ verifier.verifyFileWithResource(outFile, "failed.json");
+ }
+
+ @SuppressWarnings("unused")
+ @Test
+ @Ignore("Manual test")
+ public void testLargeQuery() throws Exception {
+ String tableName = writeBigFile();
+ if (false) {
+ // Time COUNT(*) which reads and discards data. Establishes
+ // a minimum query run time.
+ long start = System.currentTimeMillis();
+ String sql = String.format(
+ "SELECT COUNT(*) FROM dfs.data.`%s`", tableName);
+ System.out.println(
+ client.queryBuilder().sql(sql).singletonLong());
+ long end = System.currentTimeMillis();
+ System.out.println(String.format("COUNT(*) - Elapsed: %d ms", end - start));
+ }
+
+ // Run the query and dump to a file to do a rough check
+ // to see if all results appear.
+ String sql = String.format(
+ "SELECT * FROM dfs.data.`%s`", tableName);
+ if (true) {
+ File outFile = new File(dirTestWatcher.getTmpDir(), "big.json");
+ QueryWrapper query = new QueryWrapper(sql, QueryType.SQL.name(),
+ null, null, null, null);
+ long start = System.currentTimeMillis();
+ runQuery(query, outFile);
+ long end = System.currentTimeMillis();
+ System.out.println(String.format("Elapsed: %d ms", end - start));
+ System.out.println(String.format("Input size: ~%d bytes",
+ LINE_COUNT * FIELD_COUNT * FIELD_WIDTH));
+ System.out.println(String.format("Query to file - Output size: %d bytes", outFile.length()));
+ }
+
+ // Run the query and discard results. Determines the overhead, relative
+ // to COUNT(*) of serializing to JSON and sending results. Avoids the above
+ // cost of writing to a file.
+ if (false) {
+ QueryWrapper query = new QueryWrapper(sql, QueryType.SQL.name(),
+ null, null, null, null);
+ long start = System.currentTimeMillis();
+ runQuery(query);
+ long end = System.currentTimeMillis();
+ System.out.println(String.format("Query only - Elapsed: %d ms", end - start));
+ }
+ }
+
+ private static final int LINE_COUNT = 1_000_000;
+ private static final int FIELD_COUNT = 20; // Limited to 26
+ private static final int FIELD_WIDTH = 100;
+
+ private String writeBigFile() throws IOException {
+ File dataFile = new File(testDir, "big.csv");
+ try (PrintWriter writer = new PrintWriter(
+ new BufferedWriter(new FileWriter(dataFile)))) {
+ writer.print("id");
+ for (int j = 0; j < FIELD_COUNT; j++) {
+ writer.print(",");
+ writer.print((char) ('a' + j));
+ }
+ writer.println();
+ for (int i = 0; i < LINE_COUNT; i++) {
+ writer.print(i + 1);
+ for (int j = 0; j < FIELD_COUNT; j++) {
+ writer.print(",");
+ int c = 'A' + (i + j) % 26;
+ writer.print(StringUtils.repeat((char) c, FIELD_WIDTH));
+ }
+ writer.println();
+ }
+ }
+ return dataFile.getName();
+ }
+
+ /**
+ * Run a query by hitting the REST endpoint via an HTTP request. The
+ * query is JSON, serialized from {@link QueryWrapper} sent via POST.
+ * The result is written to the given file.
+ *
+ * @param query query definition
+ * @param destFile destination for the results
+ */
+ private void runQuery(QueryWrapper query, File destFile) throws IOException {
+ ObjectWriter writer = mapper.writerFor(QueryWrapper.class);
+ String json = writer.writeValueAsString(query);
+ String url = "http://localhost:8047/query.json";
+ Request request = new Request.Builder()
+ .url(url)
+ .post(RequestBody.create(json, JSON_MEDIA_TYPE))
+ .build();
+ try (Response response = httpClient.newCall(request).execute()) {
+ try (InputStream inStream = response.body().byteStream()) {
+ FileUtils.copyInputStreamToFile(inStream, destFile);
+ }
+ }
+ }
+
+ private void runQuery(QueryWrapper query) throws IOException {
+ ObjectWriter writer = mapper.writerFor(QueryWrapper.class);
+ String json = writer.writeValueAsString(query);
+ String url = "http://localhost:8047/query.json";
+ Request request = new Request.Builder()
+ .url(url)
+ .post(RequestBody.create(json, JSON_MEDIA_TYPE))
+ .build();
+ try (Response response = httpClient.newCall(request).execute()) {
+ try (InputStream inStream = response.body().byteStream()) {
+ IOUtils.skip(inStream, Integer.MAX_VALUE);
+ }
+ }
+ }
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/WebSessionResourcesTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/WebSessionResourcesTest.java
index 78cbd1e..69fa942 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/WebSessionResourcesTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/WebSessionResourcesTest.java
@@ -42,14 +42,10 @@ import static org.mockito.Mockito.verify;
* associated with it.
*/
public class WebSessionResourcesTest extends BaseTest {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WebSessionResourcesTest.class);
private WebSessionResources webSessionResources;
-
private boolean listenerComplete;
-
private CountDownLatch latch;
-
private EventExecutor executor;
// A close listener added in close future in one of the test to see if it's invoked correctly.
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedArrays.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedArrays.java
index 62a4a02..ced6463 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedArrays.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedArrays.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.time.Instant;
+import java.time.LocalDate;
import java.time.LocalDateTime;
+import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.TimeZone;
@@ -341,7 +343,7 @@ public class TestExtendedArrays extends BaseJsonLoaderTest {
TupleMetadata expectedSchema = new SchemaBuilder()
.addNullable("a", MinorType.DATE)
.build();
- org.joda.time.LocalDate date = new org.joda.time.LocalDate(2020, 04, 21);
+ LocalDate date = LocalDate.of(2020, 04, 21);
RowSet expected = fixture.rowSetBuilder(expectedSchema)
.addRow(date)
.addSingleCol(null)
@@ -368,7 +370,7 @@ public class TestExtendedArrays extends BaseJsonLoaderTest {
TupleMetadata expectedSchema = new SchemaBuilder()
.addNullable("a", MinorType.TIME)
.build();
- org.joda.time.LocalTime time = new org.joda.time.LocalTime(11, 22, 33);
+ LocalTime time = LocalTime.of(11, 22, 33);
RowSet expected = fixture.rowSetBuilder(expectedSchema)
.addRow(time)
.addSingleCol(null)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedTypes.java
index 866aea2..7c46e02 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedTypes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedTypes.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.time.Instant;
+import java.time.LocalDate;
import java.time.LocalDateTime;
+import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.TimeZone;
@@ -327,7 +329,7 @@ public class TestExtendedTypes extends BaseJsonLoaderTest {
TupleMetadata expectedSchema = new SchemaBuilder()
.addNullable("a", MinorType.DATE)
.build();
- org.joda.time.LocalDate date = new org.joda.time.LocalDate(2020, 04, 21);
+ LocalDate date = LocalDate.of(2020, 04, 21);
RowSet expected = fixture.rowSetBuilder(expectedSchema)
.addRow(date)
.addSingleCol(null)
@@ -357,8 +359,8 @@ public class TestExtendedTypes extends BaseJsonLoaderTest {
TupleMetadata expectedSchema = new SchemaBuilder()
.addNullable("a", MinorType.TIME)
.build();
- org.joda.time.LocalTime time = new org.joda.time.LocalTime(11, 22, 33);
- org.joda.time.LocalTime time2 = new org.joda.time.LocalTime(11, 22, 33, 123);
+ LocalTime time = LocalTime.of(11, 22, 33);
+ LocalTime time2 = LocalTime.of(11, 22, 33, 123_000_000);
RowSet expected = fixture.rowSetBuilder(expectedSchema)
.addRow(time)
.addRow(time2)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedWithSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedWithSchema.java
index 7560192..450374d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedWithSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/loader/TestExtendedWithSchema.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import java.time.Instant;
+import java.time.LocalDate;
import java.time.LocalDateTime;
+import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.TimeZone;
@@ -325,7 +327,7 @@ public class TestExtendedWithSchema extends BaseJsonLoaderTest {
RowSet results = loader.next();
assertNotNull(results);
- org.joda.time.LocalDate date = new org.joda.time.LocalDate(2020, 04, 21);
+ LocalDate date = LocalDate.of(2020, 04, 21);
RowSet expected = fixture.rowSetBuilder(providedSchema)
.addRow(date, null, date)
.addRow(date, date, null)
@@ -357,7 +359,7 @@ public class TestExtendedWithSchema extends BaseJsonLoaderTest {
RowSet results = loader.next();
assertNotNull(results);
- org.joda.time.LocalTime time = new org.joda.time.LocalTime(11, 22, 33);
+ LocalTime time = LocalTime.of(11, 22, 33);
RowSet expected = fixture.rowSetBuilder(providedSchema)
.addRow(time, null, time)
.addRow(time, time, null)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsvWithSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsvWithSchema.java
index e56fcfe..7358d59 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsvWithSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsvWithSchema.java
@@ -22,6 +22,11 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
import java.util.Iterator;
import org.apache.drill.categories.EvfTest;
@@ -34,9 +39,6 @@ import org.apache.drill.exec.physical.rowSet.RowSet;
import org.apache.drill.exec.physical.rowSet.RowSetBuilder;
import org.apache.drill.test.rowSet.RowSetComparison;
import org.apache.drill.test.rowSet.RowSetUtilities;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -129,7 +131,7 @@ public class TestCsvWithSchema extends BaseCsvTest {
.add("missing", MinorType.VARCHAR) // No data, no schema, default type
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(10, new LocalDate(2019, 3, 20), "it works!", 1234.5D, 20L, "")
+ .addRow(10, LocalDate.of(2019, 3, 20), "it works!", 1234.5D, 20L, "")
.build();
RowSetUtilities.verify(expected, actual);
} finally {
@@ -188,7 +190,7 @@ public class TestCsvWithSchema extends BaseCsvTest {
.addNullable("date", MinorType.DATE)
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(-1, "dino", new LocalDate(2018, 9, 1))
+ .addRow(-1, "dino", LocalDate.of(2018, 9, 1))
.build();
RowSetUtilities.verify(expected, actual);
} finally {
@@ -218,7 +220,7 @@ public class TestCsvWithSchema extends BaseCsvTest {
.add("date", MinorType.DATE)
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(0, "dino", new LocalDate(2001, 2, 3))
+ .addRow(0, "dino", LocalDate.of(2001, 2, 3))
.build();
RowSetUtilities.verify(expected, actual);
} finally {
@@ -257,12 +259,12 @@ public class TestCsvWithSchema extends BaseCsvTest {
.add("comment", MinorType.VARCHAR)
.buildSchema();
expected1 = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(1, "wilma", new LocalDate(2019, 1, 18), "female", "ABC")
- .addRow(2, "fred", new LocalDate(2019, 1, 19), "male", "ABC")
- .addRow(4, "betty", new LocalDate(2019, 5, 4), "NA", "ABC")
+ .addRow(1, "wilma", LocalDate.of(2019, 1, 18), "female", "ABC")
+ .addRow(2, "fred", LocalDate.of(2019, 1, 19), "male", "ABC")
+ .addRow(4, "betty", LocalDate.of(2019, 5, 4), "NA", "ABC")
.build();
expected2 = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(3, "barney", new LocalDate(2001, 1, 16), "NA", "ABC")
+ .addRow(3, "barney", LocalDate.of(2001, 1, 16), "NA", "ABC")
.build();
// Loop 10 times so that, as the two reader fragments read the two
@@ -446,11 +448,11 @@ public class TestCsvWithSchema extends BaseCsvTest {
.add("comment", MinorType.VARCHAR)
.buildSchema();
expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(0, "dino", new LocalDate(2018, 9, 1), "NA", "ABC")
- .addRow(1, "wilma", new LocalDate(2019, 1, 18), "female", "ABC")
- .addRow(2, "fred", new LocalDate(2019, 1, 19), "male", "ABC")
- .addRow(3, "barney", new LocalDate(2001, 1, 16), "NA", "ABC")
- .addRow(4, "betty", new LocalDate(2019, 5, 4), "NA", "ABC")
+ .addRow(0, "dino", LocalDate.of(2018, 9, 1), "NA", "ABC")
+ .addRow(1, "wilma", LocalDate.of(2019, 1, 18), "female", "ABC")
+ .addRow(2, "fred", LocalDate.of(2019, 1, 19), "male", "ABC")
+ .addRow(3, "barney", LocalDate.of(2001, 1, 16), "NA", "ABC")
+ .addRow(4, "betty", LocalDate.of(2019, 5, 4), "NA", "ABC")
.build();
// Loop 10 times so that, as the two reader fragments read the two
@@ -496,10 +498,10 @@ public class TestCsvWithSchema extends BaseCsvTest {
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
.addRow(0, null, "NA", "ABC", "dino")
- .addRow(1, new LocalDate(2019, 1, 18), "female", "ABC", "wilma")
- .addRow(2, new LocalDate(2019, 1, 19), "male", "ABC", "fred")
- .addRow(3, new LocalDate(2001, 1, 16), "NA", "ABC", "barney")
- .addRow(4, new LocalDate(2019, 5, 4), "NA", "ABC", "betty")
+ .addRow(1, LocalDate.of(2019, 1, 18), "female", "ABC", "wilma")
+ .addRow(2, LocalDate.of(2019, 1, 19), "male", "ABC", "fred")
+ .addRow(3, LocalDate.of(2001, 1, 16), "NA", "ABC", "barney")
+ .addRow(4, LocalDate.of(2019, 5, 4), "NA", "ABC", "betty")
.build();
RowSetUtilities.verify(expected, actual);
} finally {
@@ -533,10 +535,10 @@ public class TestCsvWithSchema extends BaseCsvTest {
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
.addRow(0, null, "NA", "ABC")
- .addRow(1, new LocalDate(2019, 1, 18), "female", "ABC")
- .addRow(2, new LocalDate(2019, 1, 19), "male", "ABC")
- .addRow(3, new LocalDate(2001, 1, 16), "NA", "ABC")
- .addRow(4, new LocalDate(2019, 5, 4), "NA", "ABC")
+ .addRow(1, LocalDate.of(2019, 1, 18), "female", "ABC")
+ .addRow(2, LocalDate.of(2019, 1, 19), "male", "ABC")
+ .addRow(3, LocalDate.of(2001, 1, 16), "NA", "ABC")
+ .addRow(4, LocalDate.of(2019, 5, 4), "NA", "ABC")
.build();
RowSetUtilities.verify(expected, actual);
} finally {
@@ -573,10 +575,10 @@ public class TestCsvWithSchema extends BaseCsvTest {
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
.addRow(0, null, "NA", "ABC")
- .addRow(1, new LocalDate(2019, 1, 18), "female", "ABC")
- .addRow(2, new LocalDate(2019, 1, 19), "male", "ABC")
- .addRow(3, new LocalDate(2001, 1, 16), "NA", "ABC")
- .addRow(4, new LocalDate(2019, 5, 4), "NA", "ABC")
+ .addRow(1, LocalDate.of(2019, 1, 18), "female", "ABC")
+ .addRow(2, LocalDate.of(2019, 1, 19), "male", "ABC")
+ .addRow(3, LocalDate.of(2001, 1, 16), "NA", "ABC")
+ .addRow(4, LocalDate.of(2019, 5, 4), "NA", "ABC")
.build();
RowSetUtilities.verify(expected, actual);
} finally {
@@ -1047,9 +1049,9 @@ public class TestCsvWithSchema extends BaseCsvTest {
.add("col_timestamp", MinorType.TIMESTAMP)
.add("id", MinorType.VARCHAR)
.buildSchema();
- LocalTime lt = new LocalTime(12, 34, 56);
- LocalDate ld = new LocalDate(2019, 3, 28);
- Instant ts = ld.toDateTime(lt).toInstant();
+ LocalTime lt = LocalTime.of(12, 34, 56);
+ LocalDate ld = LocalDate.of(2019, 3, 28);
+ Instant ts = LocalDateTime.of(ld, lt).toInstant(ZoneOffset.UTC);
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
.addRow(10, 10L, 10.5, 10.5f, "foo", true, new Period(0).plusDays(10),
lt, ld, ts, "1")
@@ -1091,7 +1093,7 @@ public class TestCsvWithSchema extends BaseCsvTest {
.addNullable("start_date", MinorType.DATE)
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(1, 20, new LocalDate(2019, 1, 1))
+ .addRow(1, 20, LocalDate.of(2019, 1, 1))
.addRow(2, null, null)
.addRow(3, 30, null)
.build();
@@ -1117,7 +1119,6 @@ public class TestCsvWithSchema extends BaseCsvTest {
* even nullable, because a blank string is neither a valid number nor
* a valid date.
*/
-
@Test
public void testBlankCols() throws Exception {
String tableName = "blankCols";
@@ -1169,7 +1170,7 @@ public class TestCsvWithSchema extends BaseCsvTest {
.addNullable("start_date", MinorType.DATE)
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(1, 20, new LocalDate(2019, 1, 1))
+ .addRow(1, 20, LocalDate.of(2019, 1, 1))
.addRow(2, 0, null)
.addRow(3, 30, null)
.build();
@@ -1203,7 +1204,7 @@ public class TestCsvWithSchema extends BaseCsvTest {
.addNullable("start_date", MinorType.DATE)
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(1, 20, new LocalDate(2019, 1, 1))
+ .addRow(1, 20, LocalDate.of(2019, 1, 1))
.addRow(2, null, null)
.addRow(3, 30, null)
.build();
@@ -1237,7 +1238,7 @@ public class TestCsvWithSchema extends BaseCsvTest {
.addNullable("start_date", MinorType.DATE)
.buildSchema();
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
- .addRow(1, 20, new LocalDate(2019, 1, 1))
+ .addRow(1, 20, LocalDate.of(2019, 1, 1))
.addRow(2, -1, null)
.addRow(3, 30, null)
.build();
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/log/TestLogReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/log/TestLogReader.java
index 8b4d8a3..0ba1f22 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/log/TestLogReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/log/TestLogReader.java
@@ -147,7 +147,7 @@ public class TestLogReader extends ClusterTest {
private static LogFormatConfig dateTimeConfig() {
String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}),(\\d+)\\s\\[(\\w+)\\]\\s([A-Z]+)\\s(.+)";
List<LogFormatField> schema = Lists.newArrayList(
- new LogFormatField("entry_date", "TIMESTAMP", "yy-MM-dd hh:mm:ss"),
+ new LogFormatField("entry_date", "TIMESTAMP", "yyyy-MM-dd HH:mm:ss"),
new LogFormatField("pid", "INT"),
new LogFormatField("location"),
new LogFormatField("message_type"),
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java
index d0c4efc..ad4223e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.pcapng;
import static org.junit.Assert.assertEquals;
import java.nio.file.Paths;
+import java.time.Instant;
import org.apache.drill.categories.RowSetTests;
import org.apache.drill.common.exceptions.UserRemoteException;
@@ -33,7 +34,6 @@ import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.QueryBuilder;
import org.apache.drill.test.QueryTestUtil;
import org.apache.drill.test.rowSet.RowSetComparison;
-import org.joda.time.Instant;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java
index 95758d1..0e1a162 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java
@@ -34,12 +34,10 @@ import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.metadata.MetadataUtils;
import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.exec.record.selection.SelectionVector2;
+import org.apache.drill.exec.vector.DateUtilities;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.ValueType;
import org.joda.time.Duration;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
/**
@@ -116,11 +114,11 @@ public class RowSetUtilities {
case PERIOD:
return periodFromInt(dataType.getMinorType(), value);
case DATE:
- return new LocalDate(value);
+ return DateUtilities.fromDrillDate(value);
case TIME:
- return new LocalTime(value);
+ return DateUtilities.fromDrillTime(value);
case TIMESTAMP:
- return new Instant(value);
+ return DateUtilities.fromDrillTimestamp(value);
default:
throw new IllegalStateException("Unknown writer type: " + valueType);
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
index 61e08b1..11e6604 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/file/JsonFileBuilder.java
@@ -17,54 +17,32 @@
*/
package org.apache.drill.test.rowSet.file;
-import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
-import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableMap;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.vector.accessor.ScalarReader;
-import org.apache.drill.exec.vector.accessor.ValueType;
-import org.apache.drill.exec.physical.rowSet.RowSet;
-import org.apache.drill.exec.physical.rowSet.RowSetReader;
-
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+
+import org.apache.drill.exec.physical.resultSet.util.JsonWriter;
+import org.apache.drill.exec.physical.rowSet.RowSet;
+import org.apache.drill.exec.physical.rowSet.RowSetReader;
+import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
/**
* <h4>Overview</h4>
* <p>
- * Builds a json file containing the data in a {@link RowSet}.
+ * Builds a JSON file containing the data in a {@link RowSet}.
* </p>
* <h4>Example</h4>
* <p>
- * You can find an example of how to use {@link JsonFileBuilder} at {@link org.apache.drill.test.ExampleTest#secondTest()}.
+ * You can find an example of how to use {@link JsonFileBuilder} at
+ * {@link org.apache.drill.test.ExampleTest#secondTest()}.
* </p>
*/
-public class JsonFileBuilder
-{
- public static final String DEFAULT_DOUBLE_FORMATTER = "%f";
- public static final String DEFAULT_INTEGER_FORMATTER = "%d";
- public static final String DEFAULT_LONG_FORMATTER = "%d";
- public static final String DEFAULT_STRING_FORMATTER = "\"%s\"";
- public static final String DEFAULT_DECIMAL_FORMATTER = "%s";
- public static final String DEFAULT_PERIOD_FORMATTER = "%s";
-
- public static final Map<ValueType, String> DEFAULT_FORMATTERS = new ImmutableMap.Builder<ValueType, String>()
- .put(ValueType.DOUBLE, DEFAULT_DOUBLE_FORMATTER)
- .put(ValueType.INTEGER, DEFAULT_INTEGER_FORMATTER)
- .put(ValueType.LONG, DEFAULT_LONG_FORMATTER)
- .put(ValueType.STRING, DEFAULT_STRING_FORMATTER)
- .put(ValueType.DECIMAL, DEFAULT_DECIMAL_FORMATTER)
- .put(ValueType.PERIOD, DEFAULT_PERIOD_FORMATTER)
- .build();
+public class JsonFileBuilder {
private final RowSet rowSet;
- private final Map<String, String> customFormatters = new HashMap<>();
+ private boolean pretty = true;
+ private boolean useExtendedOutput;
/**
* Creates a {@link JsonFileBuilder} that will write the given {@link RowSet} to a file.
@@ -76,33 +54,13 @@ public class JsonFileBuilder
Preconditions.checkArgument(rowSet.rowCount() > 0, "The given rowset is empty.");
}
- /**
- * Sets a custom formatter for a column using {@link String#format(String, Object...)} notation.
- * @param columnName The name of the column to change the formatter for.
- * @param columnFormatter The {@link String#format(String, Object...)} to use when writing a column value to the json file.
- * @return The {@link JsonFileBuilder}.
- */
- public JsonFileBuilder setCustomFormatter(final String columnName, final String columnFormatter) {
- Preconditions.checkNotNull(columnName);
- Preconditions.checkNotNull(columnFormatter);
-
- Iterator<MaterializedField> fields = rowSet
- .batchSchema()
- .iterator();
-
- boolean hasColumn = false;
-
- while (!hasColumn && fields.hasNext()) {
- hasColumn = fields.next()
- .getName()
- .equals(columnName);
- }
-
- final String message = String.format("(%s) is not a valid column", columnName);
- Preconditions.checkArgument(hasColumn, message);
-
- customFormatters.put(columnName, columnFormatter);
+ public JsonFileBuilder prettyPrint(boolean flag) {
+ this.pretty = flag;
+ return this;
+ }
+ public JsonFileBuilder extended(boolean flag) {
+ this.useExtendedOutput = flag;
return this;
}
@@ -115,67 +73,10 @@ public class JsonFileBuilder
tableFile.getParentFile().mkdirs();
try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(tableFile))) {
+ JsonWriter jsonWriter = new JsonWriter(os, pretty, useExtendedOutput);
final RowSetReader reader = rowSet.reader();
- final int numCols = rowSet
- .batchSchema()
- .getFieldCount();
- final Iterator<MaterializedField> fieldIterator = rowSet
- .batchSchema()
- .iterator();
- final List<String> columnNames = new ArrayList<>();
- final List<String> columnFormatters = new ArrayList<>();
-
- // Build formatters from first row.
- while (fieldIterator.hasNext()) {
- final String columnName = fieldIterator.next().getName();
- final ScalarReader columnReader = reader.scalar(columnName);
- final ValueType valueType = columnReader.valueType();
- final String columnFormatter;
-
- if (customFormatters.containsKey(columnName)) {
- columnFormatter = customFormatters.get(columnName);
- } else if (DEFAULT_FORMATTERS.containsKey(valueType)) {
- columnFormatter = DEFAULT_FORMATTERS.get(valueType);
- } else {
- final String message = String.format("Unsupported column type %s", valueType);
- throw new UnsupportedOperationException(message);
- }
-
- columnNames.add(columnName);
- columnFormatters.add(columnFormatter);
- }
-
- final StringBuilder sb = new StringBuilder();
- String lineSeparator = "";
-
- for (int index = 0; index < rowSet.rowCount(); index++) {
- reader.next();
- sb.append(lineSeparator);
- sb.append('{');
- String separator = "";
-
- for (int columnIndex = 0; columnIndex < numCols; columnIndex++) {
- sb.append(separator);
-
- final String columnName = columnNames.get(columnIndex);
- final ScalarReader columnReader = reader.scalar(columnIndex);
- final String columnFormatter = columnFormatters.get(columnIndex);
- final Object columnObject = columnReader.getObject();
- final String columnString = String.format(columnFormatter, columnObject);
-
- sb.append('"')
- .append(columnName)
- .append('"')
- .append(':')
- .append(columnString);
-
- separator = ",";
- }
-
- sb.append('}');
- lineSeparator = "\n";
- os.write(sb.toString().getBytes());
- sb.delete(0, sb.length());
+ while (reader.next()) {
+ jsonWriter.writeRow(reader);
}
}
}
diff --git a/exec/java-exec/src/test/resources/rest/cust20.json b/exec/java-exec/src/test/resources/rest/cust20.json
new file mode 100644
index 0000000..eb0e55d
--- /dev/null
+++ b/exec/java-exec/src/test/resources/rest/cust20.json
@@ -0,0 +1,28 @@
+!\{"queryId":"[^"]+"
+,"columns":["employee_id","full_name","first_name","last_name","position_id","position_title","store_id","department_id","birth_date","hire_date","salary","supervisor_id","education_level","marital_status","gender","management_role"]
+,"metadata":["BIGINT","VARCHAR","VARCHAR","VARCHAR","BIGINT","VARCHAR","BIGINT","BIGINT","VARCHAR","VARCHAR","FLOAT8","BIGINT","VARCHAR","VARCHAR","VARCHAR","VARCHAR"]
+,"attemptedAutoLimit":0
+,"rows":[
+{"employee_id":1,"full_name":"Sheri Nowmer","first_name":"Sheri","last_name":"Nowmer","position_id":1,"position_title":"President","store_id":0,"department_id":1,"birth_date":"1961-08-26","hire_date":"1994-12-01 00:00:00.0","salary":80000.0,"supervisor_id":0,"education_level":"Graduate Degree","marital_status":"S","gender":"F","management_role":"Senior Management"}
+,{"employee_id":2,"full_name":"Derrick Whelply","first_name":"Derrick","last_name":"Whelply","position_id":2,"position_title":"VP Country Manager","store_id":0,"department_id":1,"birth_date":"1915-07-03","hire_date":"1994-12-01 00:00:00.0","salary":40000.0,"supervisor_id":1,"education_level":"Graduate Degree","marital_status":"M","gender":"M","management_role":"Senior Management"}
+,{"employee_id":4,"full_name":"Michael Spence","first_name":"Michael","last_name":"Spence","position_id":2,"position_title":"VP Country Manager","store_id":0,"department_id":1,"birth_date":"1969-06-20","hire_date":"1998-01-01 00:00:00.0","salary":40000.0,"supervisor_id":1,"education_level":"Graduate Degree","marital_status":"S","gender":"M","management_role":"Senior Management"}
+,{"employee_id":5,"full_name":"Maya Gutierrez","first_name":"Maya","last_name":"Gutierrez","position_id":2,"position_title":"VP Country Manager","store_id":0,"department_id":1,"birth_date":"1951-05-10","hire_date":"1998-01-01 00:00:00.0","salary":35000.0,"supervisor_id":1,"education_level":"Bachelors Degree","marital_status":"M","gender":"F","management_role":"Senior Management"}
+,{"employee_id":6,"full_name":"Roberta Damstra","first_name":"Roberta","last_name":"Damstra","position_id":3,"position_title":"VP Information Systems","store_id":0,"department_id":2,"birth_date":"1942-10-08","hire_date":"1994-12-01 00:00:00.0","salary":25000.0,"supervisor_id":1,"education_level":"Bachelors Degree","marital_status":"M","gender":"F","management_role":"Senior Management"}
+,{"employee_id":7,"full_name":"Rebecca Kanagaki","first_name":"Rebecca","last_name":"Kanagaki","position_id":4,"position_title":"VP Human Resources","store_id":0,"department_id":3,"birth_date":"1949-03-27","hire_date":"1994-12-01 00:00:00.0","salary":15000.0,"supervisor_id":1,"education_level":"Bachelors Degree","marital_status":"M","gender":"F","management_role":"Senior Management"}
+,{"employee_id":8,"full_name":"Kim Brunner","first_name":"Kim","last_name":"Brunner","position_id":11,"position_title":"Store Manager","store_id":9,"department_id":11,"birth_date":"1922-08-10","hire_date":"1998-01-01 00:00:00.0","salary":10000.0,"supervisor_id":5,"education_level":"Bachelors Degree","marital_status":"S","gender":"F","management_role":"Store Management"}
+,{"employee_id":9,"full_name":"Brenda Blumberg","first_name":"Brenda","last_name":"Blumberg","position_id":11,"position_title":"Store Manager","store_id":21,"department_id":11,"birth_date":"1979-06-23","hire_date":"1998-01-01 00:00:00.0","salary":17000.0,"supervisor_id":5,"education_level":"Graduate Degree","marital_status":"M","gender":"F","management_role":"Store Management"}
+,{"employee_id":10,"full_name":"Darren Stanz","first_name":"Darren","last_name":"Stanz","position_id":5,"position_title":"VP Finance","store_id":0,"department_id":5,"birth_date":"1949-08-26","hire_date":"1994-12-01 00:00:00.0","salary":50000.0,"supervisor_id":1,"education_level":"Partial College","marital_status":"M","gender":"M","management_role":"Senior Management"}
+,{"employee_id":11,"full_name":"Jonathan Murraiin","first_name":"Jonathan","last_name":"Murraiin","position_id":11,"position_title":"Store Manager","store_id":1,"department_id":11,"birth_date":"1967-06-20","hire_date":"1998-01-01 00:00:00.0","salary":15000.0,"supervisor_id":5,"education_level":"Graduate Degree","marital_status":"S","gender":"M","management_role":"Store Management"}
+,{"employee_id":12,"full_name":"Jewel Creek","first_name":"Jewel","last_name":"Creek","position_id":11,"position_title":"Store Manager","store_id":5,"department_id":11,"birth_date":"1971-10-18","hire_date":"1998-01-01 00:00:00.0","salary":8500.0,"supervisor_id":5,"education_level":"Graduate Degree","marital_status":"S","gender":"F","management_role":"Store Management"}
+,{"employee_id":13,"full_name":"Peggy Medina","first_name":"Peggy","last_name":"Medina","position_id":11,"position_title":"Store Manager","store_id":10,"department_id":11,"birth_date":"1975-10-12","hire_date":"1998-01-01 00:00:00.0","salary":15000.0,"supervisor_id":5,"education_level":"Bachelors Degree","marital_status":"S","gender":"F","management_role":"Store Management"}
+,{"employee_id":14,"full_name":"Bryan Rutledge","first_name":"Bryan","last_name":"Rutledge","position_id":11,"position_title":"Store Manager","store_id":8,"department_id":11,"birth_date":"1912-07-09","hire_date":"1998-01-01 00:00:00.0","salary":17000.0,"supervisor_id":5,"education_level":"Bachelors Degree","marital_status":"M","gender":"M","management_role":"Store Management"}
+,{"employee_id":15,"full_name":"Walter Cavestany","first_name":"Walter","last_name":"Cavestany","position_id":11,"position_title":"Store Manager","store_id":4,"department_id":11,"birth_date":"1941-11-05","hire_date":"1998-01-01 00:00:00.0","salary":12000.0,"supervisor_id":5,"education_level":"Bachelors Degree","marital_status":"M","gender":"M","management_role":"Store Management"}
+,{"employee_id":16,"full_name":"Peggy Planck","first_name":"Peggy","last_name":"Planck","position_id":11,"position_title":"Store Manager","store_id":12,"department_id":11,"birth_date":"1919-06-02","hire_date":"1998-01-01 00:00:00.0","salary":17000.0,"supervisor_id":5,"education_level":"Bachelors Degree","marital_status":"S","gender":"F","management_role":"Store Management"}
+,{"employee_id":17,"full_name":"Brenda Marshall","first_name":"Brenda","last_name":"Marshall","position_id":11,"position_title":"Store Manager","store_id":18,"department_id":11,"birth_date":"1928-03-20","hire_date":"1998-01-01 00:00:00.0","salary":10000.0,"supervisor_id":5,"education_level":"Partial College","marital_status":"S","gender":"F","management_role":"Store Management"}
+,{"employee_id":18,"full_name":"Daniel Wolter","first_name":"Daniel","last_name":"Wolter","position_id":11,"position_title":"Store Manager","store_id":19,"department_id":11,"birth_date":"1914-09-21","hire_date":"1998-01-01 00:00:00.0","salary":17000.0,"supervisor_id":4,"education_level":"Partial College","marital_status":"S","gender":"M","management_role":"Store Management"}
+,{"employee_id":19,"full_name":"Dianne Collins","first_name":"Dianne","last_name":"Collins","position_id":11,"position_title":"Store Manager","store_id":20,"department_id":11,"birth_date":"1953-07-20","hire_date":"1998-01-01 00:00:00.0","salary":10000.0,"supervisor_id":4,"education_level":"Bachelors Degree","marital_status":"S","gender":"F","management_role":"Store Management"}
+,{"employee_id":20,"full_name":"Beverly Baker","first_name":"Beverly","last_name":"Baker","position_id":2,"position_title":"VP Country Manager","store_id":0,"department_id":1,"birth_date":"1974-04-16","hire_date":"1994-12-01 00:00:00.0","salary":30000.0,"supervisor_id":2,"education_level":"Bachelors Degree","marital_status":"M","gender":"F","management_role":"Senior Management"}
+,{"employee_id":21,"full_name":"Pedro Castillo","first_name":"Pedro","last_name":"Castillo","position_id":2,"position_title":"VP Country Manager","store_id":0,"department_id":1,"birth_date":"1918-11-04","hire_date":"1994-12-01 00:00:00.0","salary":35000.0,"supervisor_id":2,"education_level":"Bachelors Degree","marital_status":"M","gender":"M","management_role":"Senior Management"}
+]
+,"queryState":"COMPLETED"
+}
diff --git a/exec/java-exec/src/test/resources/rest/failed.json b/exec/java-exec/src/test/resources/rest/failed.json
new file mode 100644
index 0000000..cd1b6df
--- /dev/null
+++ b/exec/java-exec/src/test/resources/rest/failed.json
@@ -0,0 +1,3 @@
+{
+ "errorMessage" : "Query submission failed"
+}
\ No newline at end of file
diff --git a/exec/java-exec/src/test/resources/rest/small.json b/exec/java-exec/src/test/resources/rest/small.json
new file mode 100644
index 0000000..f97df9a
--- /dev/null
+++ b/exec/java-exec/src/test/resources/rest/small.json
@@ -0,0 +1,18 @@
+!\{"queryId":"[^"]+"
+,"columns":["employee_id","full_name","first_name","last_name","position_id","position_title","store_id","department_id","birth_date","hire_date","salary","supervisor_id","education_level","marital_status","gender","management_role"]
+,"metadata":["BIGINT","VARCHAR","VARCHAR","VARCHAR","BIGINT","VARCHAR","BIGINT","BIGINT","VARCHAR","VARCHAR","FLOAT8","BIGINT","VARCHAR","VARCHAR","VARCHAR","VARCHAR"]
+,"attemptedAutoLimit":10
+,"rows":[
+{"employee_id":1,"full_name":"Sheri Nowmer","first_name":"Sheri","last_name":"Nowmer","position_id":1,"position_title":"President","store_id":0,"department_id":1,"birth_date":"1961-08-26","hire_date":"1994-12-01 00:00:00.0","salary":80000.0,"supervisor_id":0,"education_level":"Graduate Degree","marital_status":"S","gender":"F","management_role":"Senior Management"}
+,{"employee_id":2,"full_name":"Derrick Whelply","first_name":"Derrick","last_name":"Whelply","position_id":2,"position_title":"VP Country Manager","store_id":0,"department_id":1,"birth_date":"1915-07-03","hire_date":"1994-12-01 00:00:00.0","salary":40000.0,"supervisor_id":1,"education_level":"Graduate Degree","marital_status":"M","gender":"M","management_role":"Senior Management"}
+,{"employee_id":4,"full_name":"Michael Spence","first_name":"Michael","last_name":"Spence","position_id":2,"position_title":"VP Country Manager","store_id":0,"department_id":1,"birth_date":"1969-06-20","hire_date":"1998-01-01 00:00:00.0","salary":40000.0,"supervisor_id":1,"education_level":"Graduate Degree","marital_status":"S","gender":"M","management_role":"Senior Management"}
+,{"employee_id":5,"full_name":"Maya Gutierrez","first_name":"Maya","last_name":"Gutierrez","position_id":2,"position_title":"VP Country Manager","store_id":0,"department_id":1,"birth_date":"1951-05-10","hire_date":"1998-01-01 00:00:00.0","salary":35000.0,"supervisor_id":1,"education_level":"Bachelors Degree","marital_status":"M","gender":"F","management_role":"Senior Management"}
+,{"employee_id":6,"full_name":"Roberta Damstra","first_name":"Roberta","last_name":"Damstra","position_id":3,"position_title":"VP Information Systems","store_id":0,"department_id":2,"birth_date":"1942-10-08","hire_date":"1994-12-01 00:00:00.0","salary":25000.0,"supervisor_id":1,"education_level":"Bachelors Degree","marital_status":"M","gender":"F","management_role":"Senior Management"}
+,{"employee_id":7,"full_name":"Rebecca Kanagaki","first_name":"Rebecca","last_name":"Kanagaki","position_id":4,"position_title":"VP Human Resources","store_id":0,"department_id":3,"birth_date":"1949-03-27","hire_date":"1994-12-01 00:00:00.0","salary":15000.0,"supervisor_id":1,"education_level":"Bachelors Degree","marital_status":"M","gender":"F","management_role":"Senior Management"}
+,{"employee_id":8,"full_name":"Kim Brunner","first_name":"Kim","last_name":"Brunner","position_id":11,"position_title":"Store Manager","store_id":9,"department_id":11,"birth_date":"1922-08-10","hire_date":"1998-01-01 00:00:00.0","salary":10000.0,"supervisor_id":5,"education_level":"Bachelors Degree","marital_status":"S","gender":"F","management_role":"Store Management"}
+,{"employee_id":9,"full_name":"Brenda Blumberg","first_name":"Brenda","last_name":"Blumberg","position_id":11,"position_title":"Store Manager","store_id":21,"department_id":11,"birth_date":"1979-06-23","hire_date":"1998-01-01 00:00:00.0","salary":17000.0,"supervisor_id":5,"education_level":"Graduate Degree","marital_status":"M","gender":"F","management_role":"Store Management"}
+,{"employee_id":10,"full_name":"Darren Stanz","first_name":"Darren","last_name":"Stanz","position_id":5,"position_title":"VP Finance","store_id":0,"department_id":5,"birth_date":"1949-08-26","hire_date":"1994-12-01 00:00:00.0","salary":50000.0,"supervisor_id":1,"education_level":"Partial College","marital_status":"M","gender":"M","management_role":"Senior Management"}
+,{"employee_id":11,"full_name":"Jonathan Murraiin","first_name":"Jonathan","last_name":"Murraiin","position_id":11,"position_title":"Store Manager","store_id":1,"department_id":11,"birth_date":"1967-06-20","hire_date":"1998-01-01 00:00:00.0","salary":15000.0,"supervisor_id":5,"education_level":"Graduate Degree","marital_status":"S","gender":"M","management_role":"Store Management"}
+]
+,"queryState":"COMPLETED"
+}
diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
index 06a3d4c..8b96898 100644
--- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java
+++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
@@ -124,6 +124,10 @@ package org.apache.drill.exec.vector.accessor;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.LocalDateTime;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.Types;
@@ -142,11 +146,7 @@ import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
import io.netty.buffer.DrillBuf;
-import org.joda.time.DateTimeZone;
import org.joda.time.Period;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
/**
* Basic accessors for most Drill vector types and modes. Each class has a bare-bones
@@ -163,6 +163,7 @@ import org.joda.time.LocalTime;
*/
public class ColumnAccessors {
+ public static final LocalDateTime LOCAL_EPOCH = LocalDateTime.of(1970, 1, 1, 0, 0, 0);
<#list vv.types as type>
<#list type.minor as minor>
@@ -321,29 +322,29 @@ public class ColumnAccessors {
BigInteger unscaledValue = bytes.length == 0 ? BigInteger.ZERO : new BigInteger(bytes);
return new BigDecimal(unscaledValue, type.getScale());
}
+ <#elseif drillType == "Float4">
+
+ @Override
+ public double getDouble() {
+ return getFloat();
+ }
<#elseif drillType == "Date">
@Override
public final LocalDate getDate() {
- <#-- Java 8:
- return LocalDate.ofEpochDay(getLong() / DateUtilities.daysToStandardMillis); -->
- return new LocalDate(getLong(), DateTimeZone.UTC);
+ return DateUtilities.fromDrillDate(getLong());
}
<#elseif drillType == "Time">
@Override
public final LocalTime getTime() {
- <#-- Java 8:
- return LocalTime.ofNanoOfDay(getInt() * 1_000_000L); -->
- return new LocalTime(getInt(), DateTimeZone.UTC);
+ return DateUtilities.fromDrillTime(getInt());
}
<#elseif drillType == "TimeStamp">
@Override
public final Instant getTimestamp() {
- <#-- Java 8:
- return Instant.ofEpochMilli(getLong()); -->
- return new Instant(getLong());
+ return DateUtilities.fromDrillTimestamp(getLong());
}
</#if>
}
@@ -415,8 +416,8 @@ public class ColumnAccessors {
buf.writerIndex(VALUE_WIDTH);
}
</#if>
-
<#if drillType == "VarChar" || drillType == "Var16Char" || drillType == "VarBinary">
+
@Override
public final void appendBytes(final byte[] value, final int len) {
vectorIndex.prevElement();
@@ -551,25 +552,19 @@ public class ColumnAccessors {
@Override
public final void setDate(final LocalDate value) {
- <#-- Java 8:
- setLong(value.toEpochDay() * DateUtilities.daysToStandardMillis); -->
- setLong(value.toDateTimeAtStartOfDay(DateTimeZone.UTC).toInstant().getMillis());
+ setLong(DateUtilities.toDrillDate(value));
}
<#elseif drillType == "Time">
@Override
public final void setTime(final LocalTime value) {
- <#-- Java 8:
- setInt((int) ((value.toNanoOfDay() + 500_000) / 1_000_000L)); -->
- setInt(value.getMillisOfDay());
+ setInt(DateUtilities.toDrillTime(value));
}
<#elseif drillType == "TimeStamp">
@Override
public final void setTimestamp(final Instant value) {
- <#-- Java 8:
- setLong(value.toEpochMilli()); -->
- setLong(value.getMillis());
+ setLong(DateUtilities.toDrillTimestamp(value));
}
</#if>
<#if ! intType>
@@ -614,11 +609,11 @@ public class ColumnAccessors {
<#else>
try (DrillBuf buf = vector.getAllocator().buffer(VALUE_WIDTH)) {
<#if drillType = "Date">
- writeLong(buf, ((LocalDate) value).toDateTimeAtStartOfDay(DateTimeZone.UTC).toInstant().getMillis());
+ writeLong(buf, DateUtilities.toDrillDate((LocalDate) value));
<#elseif drillType = "Time">
- writeInt(buf, ((LocalTime) value).getMillisOfDay());
+ writeInt(buf, DateUtilities.toDrillTime((LocalTime) value));
<#elseif drillType = "TimeStamp">
- writeLong(buf, ((Instant) value).getMillis());
+ writeLong(buf, DateUtilities.toDrillTimestamp((Instant) value));
<#elseif putArgs != "">
throw new InvalidConversionError("Generic object not supported for type ${drillType}, "
+ "set${label}(${accessorType}${putArgs})");
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
index 1338c91..3a20fad 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.record.metadata;
import java.io.IOException;
+import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
@@ -29,7 +30,6 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.metadata.schema.parser.SchemaExprParser;
-import org.joda.time.format.DateTimeFormatter;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonCreator;
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java
index 4d55787..2117bed 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java
@@ -17,12 +17,13 @@
*/
package org.apache.drill.exec.record.metadata;
+import java.time.format.DateTimeFormatter;
+
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.vector.accessor.ColumnWriter;
-import org.joda.time.format.DateTimeFormatter;
/**
* Metadata description of a column including names, types and structure
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/PrimitiveColumnMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/PrimitiveColumnMetadata.java
index 256f55c..bb37b0f 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/PrimitiveColumnMetadata.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/PrimitiveColumnMetadata.java
@@ -19,6 +19,12 @@ package org.apache.drill.exec.record.metadata;
import java.math.BigDecimal;
import java.util.Objects;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import org.apache.drill.common.types.BooleanType;
import org.apache.drill.common.types.TypeProtos.DataMode;
@@ -26,13 +32,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.exec.record.MaterializedField;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -133,15 +133,15 @@ public class PrimitiveColumnMetadata extends AbstractColumnMetadata {
switch (type) {
case TIME:
return formatValue == null
- ? ISODateTimeFormat.localTimeParser() : DateTimeFormat.forPattern(formatValue);
+ ? DateTimeFormatter.ISO_LOCAL_TIME : DateTimeFormatter.ofPattern(formatValue);
case DATE:
formatValue = format();
return formatValue == null
- ? ISODateTimeFormat.localDateParser() : DateTimeFormat.forPattern(formatValue);
+ ? DateTimeFormatter.ISO_LOCAL_DATE : DateTimeFormatter.ofPattern(formatValue);
case TIMESTAMP:
formatValue = format();
return formatValue == null
- ? ISODateTimeFormat.dateTimeNoMillis() : DateTimeFormat.forPattern(formatValue);
+ ? DateTimeFormatter.ISO_LOCAL_DATE_TIME : DateTimeFormatter.ofPattern(formatValue);
default:
throw new IllegalArgumentException("Column is not a date/time type: " + type.toString());
}
@@ -264,7 +264,7 @@ public class PrimitiveColumnMetadata extends AbstractColumnMetadata {
case DATE:
return LocalDate.parse(value, dateTimeFormatter());
case TIMESTAMP:
- return Instant.parse(value, dateTimeFormatter());
+ return LocalDateTime.parse(value, dateTimeFormatter()).toInstant(ZoneOffset.UTC);
case INTERVAL:
case INTERVALDAY:
case INTERVALYEAR:
@@ -293,11 +293,11 @@ public class PrimitiveColumnMetadata extends AbstractColumnMetadata {
}
switch (type) {
case TIME:
- return dateTimeFormatter().print((LocalTime) value);
+ return dateTimeFormatter().format((LocalTime) value);
case DATE:
- return dateTimeFormatter().print((LocalDate) value);
+ return dateTimeFormatter().format((LocalDate) value);
case TIMESTAMP:
- return dateTimeFormatter().print((Instant) value);
+ return dateTimeFormatter().format((Instant) value);
default:
return value.toString();
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/DateUtilities.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/DateUtilities.java
index d3fb6f7..330a742 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/DateUtilities.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/DateUtilities.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.vector;
+import java.time.Instant;
+import java.time.LocalDate;
import java.time.LocalTime;
import org.joda.time.Period;
@@ -195,7 +197,27 @@ public class DateUtilities {
* @param localTime Java local time
* @return Drill form of the time
*/
- public static int toTime(LocalTime localTime) {
+ public static int toDrillTime(LocalTime localTime) {
return (int) ((localTime.toNanoOfDay() + 500_000L) / 1_000_000L); // round to milliseconds
}
+
+ public static LocalTime fromDrillTime(int value) {
+ return LocalTime.ofNanoOfDay(value * 1_000_000L);
+ }
+
+ public static long toDrillDate(LocalDate localDate) {
+ return localDate.toEpochDay() * daysToStandardMillis;
+ }
+
+ public static LocalDate fromDrillDate(long value) {
+ return LocalDate.ofEpochDay(value / daysToStandardMillis);
+ }
+
+ public static long toDrillTimestamp(Instant instant) {
+ return instant.toEpochMilli();
+ }
+
+ public static Instant fromDrillTimestamp(long value) {
+ return Instant.ofEpochMilli(value);
+ }
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java
index 8758e55..738a1ca 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ScalarReader.java
@@ -18,10 +18,10 @@
package org.apache.drill.exec.vector.accessor;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
/**
@@ -44,9 +44,9 @@ import org.joda.time.Period;
* require another level of reader abstraction to access each value
* in the structure.
*
- * <h4>Date/Time and the Joda Classes</h4>
+ * <h4>Joda Period</h4>
*
- * Note that the date/time columns here use the old Joda classes.
+ * Note that the interval columns here use the old Joda classes.
* As it turns out, JSR-310, the specification on which the Java 8 date/time
* classes are based, does not include the equivalent of the old Joda
* Interval class: a single object which can hold years, months, days,
@@ -57,7 +57,6 @@ import org.joda.time.Period;
* <p>
* See {@link ScalarWriter}
*/
-
public interface ScalarReader extends ColumnReader {
/**
* Describe the type of the value. This is a compression of the
@@ -66,7 +65,6 @@ public interface ScalarReader extends ColumnReader {
* @return the value type which indicates which get method
* is valid for the column
*/
-
ValueType valueType();
/**
@@ -74,7 +72,6 @@ public interface ScalarReader extends ColumnReader {
* for DATE, TIME and TIMESTAMP for which the value type is
* int or long.
*/
-
ValueType extendedType();
int getInt();
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueWriter.java
index 1ec1fd1..69e39ab 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ValueWriter.java
@@ -18,10 +18,10 @@
package org.apache.drill.exec.vector.accessor;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
/**
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
index e5e221e..90b9b04 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
@@ -18,6 +18,9 @@
package org.apache.drill.exec.vector.accessor.reader;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.vector.accessor.ColumnReader;
@@ -27,9 +30,6 @@ import org.apache.drill.exec.vector.accessor.ScalarReader;
import org.apache.drill.exec.vector.accessor.UnsupportedConversionError;
import org.apache.drill.exec.vector.accessor.ValueType;
import org.apache.drill.exec.vector.accessor.impl.AccessorUtilities;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
public abstract class AbstractScalarReader implements ScalarReader, ReaderEvents {
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java
index 43426b9..98dd105 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java
@@ -18,15 +18,15 @@
package org.apache.drill.exec.vector.accessor.writer;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
import java.util.Date;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.exec.vector.accessor.UnsupportedConversionError;
import org.apache.drill.exec.vector.accessor.ValueType;
import org.apache.drill.exec.vector.accessor.writer.WriterEvents.ColumnWriterListener;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
/**
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/BaseScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/BaseScalarWriter.java
index 7938b29..e511fa0 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/BaseScalarWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/BaseScalarWriter.java
@@ -18,13 +18,13 @@
package org.apache.drill.exec.vector.accessor.writer;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.vector.accessor.UnsupportedConversionError;
import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
import io.netty.buffer.DrillBuf;
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/NullableScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/NullableScalarWriter.java
index 02343df..6b6959d 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/NullableScalarWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/NullableScalarWriter.java
@@ -18,6 +18,9 @@
package org.apache.drill.exec.vector.accessor.writer;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.vector.BaseDataValueVector;
@@ -27,9 +30,6 @@ import org.apache.drill.exec.vector.accessor.ColumnReader;
import org.apache.drill.exec.vector.accessor.ColumnWriterIndex;
import org.apache.drill.exec.vector.accessor.ValueType;
import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
public class NullableScalarWriter extends AbstractScalarWriterImpl {
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/dummy/DummyScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/dummy/DummyScalarWriter.java
index 5d738c3..5f80913 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/dummy/DummyScalarWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/dummy/DummyScalarWriter.java
@@ -18,6 +18,9 @@
package org.apache.drill.exec.vector.accessor.writer.dummy;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.vector.BaseDataValueVector;
@@ -25,9 +28,6 @@ import org.apache.drill.exec.vector.accessor.ColumnReader;
import org.apache.drill.exec.vector.accessor.ColumnWriterIndex;
import org.apache.drill.exec.vector.accessor.ValueType;
import org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriterImpl;
-import org.joda.time.Instant;
-import org.joda.time.LocalDate;
-import org.joda.time.LocalTime;
import org.joda.time.Period;
/**
diff --git a/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java b/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java
index 318be4a..d3e5973 100644
--- a/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java
+++ b/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java
@@ -23,10 +23,10 @@ import org.apache.drill.exec.record.metadata.DictColumnMetadata;
import org.apache.drill.exec.record.metadata.SchemaBuilder;
import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.test.BaseTest;
-import org.joda.time.LocalDate;
import org.junit.Test;
import java.io.IOException;
+import java.time.LocalDate;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -427,7 +427,7 @@ public class TestSchemaParser extends BaseTest {
TupleMetadata schema = SchemaExprParser.parseSchema(value);
ColumnMetadata columnMetadata = schema.metadata("a");
assertTrue(columnMetadata.decodeDefaultValue() instanceof LocalDate);
- assertEquals(new LocalDate(2018, 12, 31), columnMetadata.decodeDefaultValue());
+ assertEquals(LocalDate.of(2018, 12, 31), columnMetadata.decodeDefaultValue());
assertEquals("2018-12-31", columnMetadata.defaultValue());
assertEquals(value, columnMetadata.columnString());
}