You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ar...@apache.org on 2019/01/30 13:16:28 UTC
[drill] branch master updated (b557b79 -> 8fb85cd)
This is an automated email from the ASF dual-hosted git repository.
arina pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from b557b79 DRILL-7000: Queries failing with 'Failed to aggregate or route the RFW' do not complete
new 85c656e DRILL-7007: Use verify method in row set tests
new 8fb85cd DRILL-7006: Add type conversion to row writers
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../record/metadata/AbstractColumnMetadata.java | 16 ++
.../record/metadata/PrimitiveColumnMetadata.java | 56 ++++++-
.../drill/exec/cache/TestBatchSerialization.java | 4 +-
.../impl/xsort/managed/SortTestUtilities.java | 6 +-
.../physical/impl/xsort/managed/TestSorter.java | 6 +-
.../impl/TestResultSetLoaderEmptyProject.java | 5 +-
.../impl/TestResultSetLoaderOmittedValues.java | 8 +-
.../rowSet/impl/TestResultSetLoaderProjection.java | 14 +-
.../rowSet/impl/TestResultSetLoaderProtocol.java | 54 ++++++
.../rowSet/impl/TestResultSetSchemaChange.java | 5 +-
.../exec/store/easy/text/compliant/TestCsv.java | 11 +-
.../apache/drill/exec/store/log/TestLogReader.java | 28 ++--
.../test/rowSet/test/TestColumnConvertor.java | 145 ++++++++++++++++
.../test/rowSet/test/TestIndirectReaders.java | 5 +-
.../test/{RowSetTest.java => TestRowSet.java} | 45 +++--
.../rowSet/{ => test}/TestRowSetComparison.java | 5 +-
.../drill/exec/record/metadata/ColumnMetadata.java | 40 +++++
.../vector/accessor/ColumnConversionFactory.java | 40 +++++
.../accessor/UnsupportedConversionError.java | 2 +-
.../accessor/writer/AbstractScalarWriter.java | 52 ++----
.../accessor/writer/AbstractWriteConvertor.java | 186 +++++++++++++++++++++
.../vector/accessor/writer/ConcreteWriter.java | 69 ++++++++
.../vector/accessor/writer/ScalarArrayWriter.java | 24 +--
23 files changed, 690 insertions(+), 136 deletions(-)
create mode 100644 exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConvertor.java
rename exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/{RowSetTest.java => TestRowSet.java} (96%)
rename exec/java-exec/src/test/java/org/apache/drill/test/rowSet/{ => test}/TestRowSetComparison.java (96%)
create mode 100644 exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnConversionFactory.java
create mode 100644 exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractWriteConvertor.java
create mode 100644 exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ConcreteWriter.java
[drill] 02/02: DRILL-7006: Add type conversion to row writers
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 8fb85cd4370e6143641cda1ad5b998caca0b6bf7
Author: Paul Rogers <pr...@cloudera.com>
AuthorDate: Sun Jan 27 10:32:46 2019 -0800
DRILL-7006: Add type conversion to row writers
Modifies the column metadata and writer abstractions to allow a type conversion "shim" to be specified as part of the schema, then inserted as part of the row set writer. Allows, say, setting an Int or Date from a string, parsing the string to obtain the proper data type to store in the vector.
Type conversion not yet supported in the result set loader: some additional complexity needs to be resolved.
Adds unit tests for this functionality. Refactors some existing tests to remove rough edges.
closes #1623
---
.../record/metadata/AbstractColumnMetadata.java | 16 ++
.../record/metadata/PrimitiveColumnMetadata.java | 56 ++++++-
.../rowSet/impl/TestResultSetLoaderProtocol.java | 54 ++++++
.../test/rowSet/test/TestColumnConvertor.java | 145 ++++++++++++++++
.../test/{RowSetTest.java => TestRowSet.java} | 45 +++--
.../rowSet/{ => test}/TestRowSetComparison.java | 5 +-
.../drill/exec/record/metadata/ColumnMetadata.java | 40 +++++
.../vector/accessor/ColumnConversionFactory.java | 40 +++++
.../accessor/UnsupportedConversionError.java | 2 +-
.../accessor/writer/AbstractScalarWriter.java | 52 ++----
.../accessor/writer/AbstractWriteConvertor.java | 186 +++++++++++++++++++++
.../vector/accessor/writer/ConcreteWriter.java | 69 ++++++++
.../vector/accessor/writer/ScalarArrayWriter.java | 24 +--
13 files changed, 654 insertions(+), 80 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
index 595c5c4..63dda07 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
@@ -21,6 +21,8 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.accessor.ColumnConversionFactory;
+import org.apache.drill.exec.vector.accessor.UnsupportedConversionError;
/**
* Abstract definition of column metadata. Allows applications to create
@@ -179,6 +181,20 @@ public abstract class AbstractColumnMetadata implements ColumnMetadata {
public boolean isProjected() { return projected; }
@Override
+ public void setDefaultValue(Object value) { }
+
+ @Override
+ public Object defaultValue() { return null; }
+
+ @Override
+ public void setTypeConverter(ColumnConversionFactory factory) {
+ throw new UnsupportedConversionError("Type conversion not supported for non-scalar writers");
+ }
+
+ @Override
+ public ColumnConversionFactory typeConverter() { return null; }
+
+ @Override
public String toString() {
final StringBuilder buf = new StringBuilder()
.append("[")
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/PrimitiveColumnMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/PrimitiveColumnMetadata.java
index dfbd4a9..ead6134 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/PrimitiveColumnMetadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/PrimitiveColumnMetadata.java
@@ -23,16 +23,52 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.accessor.ColumnConversionFactory;
/**
- * Primitive (non-map) column. Describes non-nullable, nullable and
- * array types (which differ only in mode, but not in metadata structure.)
+ * Primitive (non-map) column. Describes non-nullable, nullable and array types
+ * (which differ only in mode, but not in metadata structure.)
+ * <p>
+ * Metadata is of two types:
+ * <ul>
+ * <li>Storage metadata that describes how the column is materialized in a
+ * vector. Storage metadata is immutable because revising an existing vector is
+ * a complex operation.</li>
+ * <li>Supplemental metadata used when reading or writing the column.
+ * Supplemental metadata can be changed after the column is created, though it
+ * should generally be set before invoking code that uses the metadata.</li>
+ * </ul>
*/
public class PrimitiveColumnMetadata extends AbstractColumnMetadata {
+ /**
+ * Expected (average) width for variable-width columns.
+ */
+
private int expectedWidth;
+ /**
+ * Default value to use for filling a vector when no real data is
+ * available, such as for columns added in new files but which does not
+ * exist in existing files. The ultimate default value is the SQL null
+ * value, which works only for nullable columns.
+ */
+
+ private Object defaultValue;
+
+ /**
+ * Factory for an optional shim writer that translates from the type of
+ * data available to the code that creates the vectors on the one hand,
+ * and the actual type of the column on the other. For example, a shim
+ * might parse a string form of a date into the form stored in vectors.
+ * <p>
+ * The default is to use the "natural" type: that is, to insert no
+ * conversion shim.
+ */
+
+ private ColumnConversionFactory shimFactory;
+
public PrimitiveColumnMetadata(MaterializedField schema) {
super(schema);
expectedWidth = estimateWidth(schema.getType());
@@ -99,6 +135,22 @@ public class PrimitiveColumnMetadata extends AbstractColumnMetadata {
}
@Override
+ public void setDefaultValue(Object value) {
+ defaultValue = value;
+ }
+
+ @Override
+ public Object defaultValue() { return defaultValue; }
+
+ @Override
+ public void setTypeConverter(ColumnConversionFactory factory) {
+ shimFactory = factory;
+ }
+
+ @Override
+ public ColumnConversionFactory typeConverter() { return shimFactory; }
+
+ @Override
public ColumnMetadata cloneEmpty() {
return new PrimitiveColumnMetadata(this);
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
index 088e8f4..8fb600d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.physical.rowSet.impl;
+import static org.apache.drill.test.rowSet.RowSetUtilities.intArray;
import static org.apache.drill.test.rowSet.RowSetUtilities.strArray;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -43,8 +44,10 @@ import org.apache.drill.exec.vector.accessor.TupleWriter.UndefinedColumnExceptio
import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
+import org.apache.drill.test.rowSet.test.TestColumnConvertor.TestConvertor;
import org.apache.drill.test.rowSet.RowSetReader;
import org.apache.drill.test.rowSet.RowSetUtilities;
+import org.junit.Ignore;
import org.junit.Test;
/**
@@ -602,4 +605,55 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest {
rsLoader.close();
}
+
+ /**
+ * Test the use of a column type converter in the result set loader for
+ * required, nullable and repeated columns.
+ */
+
+ @Ignore("Not yet")
+ @Test
+ public void testTypeConversion() {
+ TupleMetadata schema = new SchemaBuilder()
+ .add("n1", MinorType.INT)
+ .addNullable("n2", MinorType.INT)
+ .addArray("n3", MinorType.INT)
+ .buildSchema();
+
+ // Add a type convertor. Passed in as a factory
+ // since we must create a new one for each row set writer.
+
+ schema.metadata("n1").setTypeConverter(TestConvertor.factory());
+ schema.metadata("n2").setTypeConverter(TestConvertor.factory());
+ schema.metadata("n3").setTypeConverter(TestConvertor.factory());
+
+ ResultSetLoaderImpl.ResultSetOptions options = new OptionBuilder()
+ .setSchema(schema)
+ .setRowCountLimit(ValueVector.MAX_ROW_COUNT)
+ .build();
+ ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
+
+ // Write data as both a string as an integer
+
+ RowSetLoader rootWriter = rsLoader.writer();
+ rootWriter.addRow("123", "12", strArray("123", "124"));
+ rootWriter.addRow(234, 23, intArray(234, 235));
+ RowSet actual = fixture.wrap(rsLoader.harvest());
+
+ // Build the expected vector without a type convertor.
+
+ TupleMetadata expectedSchema = new SchemaBuilder()
+ .add("n1", MinorType.INT)
+ .addNullable("n2", MinorType.INT)
+ .addArray("n3", MinorType.INT)
+ .buildSchema();
+ final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema)
+ .addRow(123, 12, intArray(123, 124))
+ .addRow(234, 23, intArray(234, 235))
+ .build();
+
+ // Compare
+
+ RowSetUtilities.verify(expected, actual);
+ }
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConvertor.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConvertor.java
new file mode 100644
index 0000000..b7d865d
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestColumnConvertor.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.test.rowSet.test;
+
+import static org.apache.drill.test.rowSet.RowSetUtilities.strArray;
+import static org.apache.drill.test.rowSet.RowSetUtilities.intArray;
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.record.metadata.SchemaBuilder;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.vector.accessor.ColumnConversionFactory;
+import org.apache.drill.exec.vector.accessor.ScalarWriter;
+import org.apache.drill.exec.vector.accessor.writer.AbstractWriteConvertor;
+import org.apache.drill.exec.vector.accessor.writer.ConcreteWriter;
+import org.apache.drill.test.SubOperatorTest;
+import org.apache.drill.test.rowSet.RowSet;
+import org.apache.drill.test.rowSet.RowSetBuilder;
+import org.apache.drill.test.rowSet.RowSetUtilities;
+import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
+import org.junit.Test;
+
+/**
+ * Tests the column type convertor feature of the column metadata
+ * and of the RowSetWriter.
+ */
+
+public class TestColumnConvertor extends SubOperatorTest {
+
+ /**
+ * Simple type converter that allows string-to-int conversions.
+ * Inherits usual int value support from the base writer.
+ */
+ public static class TestConvertor extends AbstractWriteConvertor {
+
+ public TestConvertor(ScalarWriter baseWriter) {
+ super(baseWriter);
+ }
+
+ @Override
+ public void setString(String value) {
+ setInt(Integer.parseInt(value));
+ }
+
+ public static ColumnConversionFactory factory() {
+ return new ColumnConversionFactory() {
+ @Override
+ public ConcreteWriter newWriter(ColumnMetadata colDefn,
+ ConcreteWriter baseWriter) {
+ return new TestConvertor(baseWriter);
+ }
+ };
+ }
+ }
+
+ @Test
+ public void testScalarConvertor() {
+
+ // Create the schema
+
+ TupleMetadata schema = new SchemaBuilder()
+ .add("n1", MinorType.INT)
+ .addNullable("n2", MinorType.INT)
+ .buildSchema();
+
+ // Add a type convertor. Passed in as a factory
+ // since we must create a new one for each row set writer.
+
+ schema.metadata("n1").setTypeConverter(TestConvertor.factory());
+ schema.metadata("n2").setTypeConverter(TestConvertor.factory());
+
+ // Write data as both a string as an integer
+
+ RowSet actual = new RowSetBuilder(fixture.allocator(), schema)
+ .addRow("123", "12")
+ .addRow(234, 23)
+ .build();
+
+ // Build the expected vector without a type convertor.
+
+ TupleMetadata expectedSchema = new SchemaBuilder()
+ .add("n1", MinorType.INT)
+ .addNullable("n2", MinorType.INT)
+ .buildSchema();
+ final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema)
+ .addRow(123, 12)
+ .addRow(234, 23)
+ .build();
+
+ // Compare
+
+ RowSetUtilities.verify(expected, actual);
+ }
+
+ @Test
+ public void testArrayConvertor() {
+
+ // Create the schema
+
+ TupleMetadata schema = new SchemaBuilder()
+ .addArray("n", MinorType.INT)
+ .buildSchema();
+
+ // Add a type convertor. Passed in as a factory
+ // since we must create a new one for each row set writer.
+
+ schema.metadata("n").setTypeConverter(TestConvertor.factory());
+
+ // Write data as both a string as an integer
+
+ RowSet actual = new RowSetBuilder(fixture.allocator(), schema)
+ .addSingleCol(strArray("123", "124"))
+ .addSingleCol(intArray(234, 235))
+ .build();
+
+ // Build the expected vector without a type convertor.
+
+ TupleMetadata expectedSchema = new SchemaBuilder()
+ .addArray("n", MinorType.INT)
+ .buildSchema();
+ final SingleRowSet expected = fixture.rowSetBuilder(expectedSchema)
+ .addSingleCol(intArray(123, 124))
+ .addSingleCol(intArray(234, 235))
+ .build();
+
+ // Compare
+
+ RowSetUtilities.verify(expected, actual);
+ }
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSet.java
similarity index 96%
rename from exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
rename to exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSet.java
index ee5c599..b660672 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSet.java
@@ -29,7 +29,6 @@ import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.metadata.SchemaBuilder;
import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.exec.vector.ValueVector;
@@ -47,8 +46,8 @@ import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.DirectRowSet;
import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
-import org.apache.drill.test.rowSet.RowSetComparison;
import org.apache.drill.test.rowSet.RowSetReader;
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.apache.drill.test.rowSet.RowSetWriter;
import org.junit.Test;
@@ -72,8 +71,8 @@ import org.junit.Test;
* A list is an array of variants. Variants are tested elsewhere.
*/
-public class RowSetTest extends SubOperatorTest {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RowSetTest.class);
+public class TestRowSet extends SubOperatorTest {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestRowSet.class);
/**
* Test the simplest constructs: a row with top-level scalar
@@ -169,7 +168,7 @@ public class RowSetTest extends SubOperatorTest {
.addRow(30)
.addRow(40)
.build();
- new RowSetComparison(expected).verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
}
/**
@@ -303,8 +302,7 @@ public class RowSetTest extends SubOperatorTest {
.addSingleCol(intArray(30))
.addSingleCol(intArray(40, 41))
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
}
/**
@@ -438,8 +436,7 @@ public class RowSetTest extends SubOperatorTest {
.addRow(20, objArray(intArray(21, 22)))
.addRow(30, objArray(intArray(31, 32)))
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
}
@Test
@@ -583,8 +580,7 @@ public class RowSetTest extends SubOperatorTest {
.addRow(20, objArray(objArray(201, 202), objArray(211, 212)))
.addRow(30, objArray(objArray(301, 302), objArray(311, 312)))
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
}
/**
@@ -594,12 +590,12 @@ public class RowSetTest extends SubOperatorTest {
@Test
public void testTopFixedWidthArray() {
- final BatchSchema batchSchema = new SchemaBuilder()
+ final TupleMetadata schema = new SchemaBuilder()
.add("c", MinorType.INT)
.addArray("a", MinorType.INT)
- .build();
+ .buildSchema();
- final ExtendableRowSet rs1 = fixture.rowSet(batchSchema);
+ final ExtendableRowSet rs1 = fixture.rowSet(schema);
final RowSetWriter writer = rs1.writer();
writer.scalar(0).setInt(10);
final ScalarWriter array = writer.array(1).scalar();
@@ -644,14 +640,13 @@ public class RowSetTest extends SubOperatorTest {
assertEquals(0, arrayReader.size());
assertFalse(reader.next());
- final SingleRowSet rs2 = fixture.rowSetBuilder(batchSchema)
+ final SingleRowSet rs2 = fixture.rowSetBuilder(schema)
.addRow(10, intArray(100, 110))
.addRow(20, intArray(200, 120, 220))
.addRow(30, null)
.build();
- new RowSetComparison(rs1)
- .verifyAndClearAll(rs2);
+ RowSetUtilities.verify(rs1, rs2);
}
/**
* Test filling a row set up to the maximum number of rows.
@@ -661,11 +656,11 @@ public class RowSetTest extends SubOperatorTest {
@Test
public void testRowBounds() {
- final BatchSchema batchSchema = new SchemaBuilder()
+ final TupleMetadata schema = new SchemaBuilder()
.add("a", MinorType.INT)
- .build();
+ .buildSchema();
- final ExtendableRowSet rs = fixture.rowSet(batchSchema);
+ final ExtendableRowSet rs = fixture.rowSet(schema);
final RowSetWriter writer = rs.writer();
int count = 0;
while (! writer.isFull()) {
@@ -695,10 +690,10 @@ public class RowSetTest extends SubOperatorTest {
@Test
public void testBufferBounds() {
- final BatchSchema batchSchema = new SchemaBuilder()
+ final TupleMetadata schema = new SchemaBuilder()
.add("a", MinorType.INT)
.add("b", MinorType.VARCHAR)
- .build();
+ .buildSchema();
String varCharValue;
try {
@@ -709,7 +704,7 @@ public class RowSetTest extends SubOperatorTest {
throw new IllegalStateException(e);
}
- final ExtendableRowSet rs = fixture.rowSet(batchSchema);
+ final ExtendableRowSet rs = fixture.rowSet(schema);
final RowSetWriter writer = rs.writer();
int count = 0;
try {
@@ -751,14 +746,14 @@ public class RowSetTest extends SubOperatorTest {
// will be provided by a reader, by an incoming batch,
// etc.
- final BatchSchema schema = new SchemaBuilder()
+ final TupleMetadata schema = new SchemaBuilder()
.add("a", MinorType.VARCHAR)
.addArray("b", MinorType.INT)
.addMap("c")
.add("c1", MinorType.INT)
.add("c2", MinorType.VARCHAR)
.resumeSchema()
- .build();
+ .buildSchema();
// Step 2: Create a batch. Done here because this is
// a batch-oriented test. Done automatically in the
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/TestRowSetComparison.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSetComparison.java
similarity index 96%
rename from exec/java-exec/src/test/java/org/apache/drill/test/rowSet/TestRowSetComparison.java
rename to exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSetComparison.java
index 062dd56..867f61f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/TestRowSetComparison.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestRowSetComparison.java
@@ -15,13 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.drill.test.rowSet;
+package org.apache.drill.test.rowSet.test;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.RootAllocator;
import org.apache.drill.exec.record.metadata.SchemaBuilder;
import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.test.rowSet.RowSet;
+import org.apache.drill.test.rowSet.RowSetBuilder;
+import org.apache.drill.test.rowSet.RowSetComparison;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java
index 0e0fb49..85d7d25 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/ColumnMetadata.java
@@ -21,6 +21,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.accessor.ColumnConversionFactory;
/**
* Metadata description of a column including names, types and structure
@@ -182,6 +183,45 @@ public interface ColumnMetadata {
int expectedElementCount();
/**
+ * Set the default value to use for filling a vector when no real data is
+ * available, such as for columns added in new files but which does not
+ * exist in existing files. The "default default" is null, which works
+ * only for nullable columns.
+ *
+ * @param value column value, represented as a Java object, acceptable
+ * to the {@link ColumnWriter#setObject()} method for this column's writer.
+ */
+ void setDefaultValue(Object value);
+
+ /**
+ * Returns the default value for this column.
+ *
+ * @return the default value, or null if no default value has been set
+ */
+ Object defaultValue();
+
+ /**
+ * Set the factory for an optional shim writer that translates from the type of
+ * data available to the code that creates the vectors on the one hand,
+ * and the actual type of the column on the other. For example, a shim
+ * might parse a string form of a date into the form stored in vectors.
+ * <p>
+ * The shim must write to the base vector for this column using one of
+ * the supported base writer "set" methods.
+ * <p>
+ * The default is to use the "natural" type: that is, to insert no
+ * conversion shim.
+ */
+ void setTypeConverter(ColumnConversionFactory factory);
+
+ /**
+ * Returns the type conversion shim for this column.
+ *
+ * @return the type conversion factory, or null if none is set
+ */
+ ColumnConversionFactory typeConverter();
+
+ /**
* Create an empty version of this column. If the column is a scalar,
* produces a simple copy. If a map, produces a clone without child
* columns.
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnConversionFactory.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnConversionFactory.java
new file mode 100644
index 0000000..02efd6d
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/ColumnConversionFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor;
+
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.vector.accessor.writer.ConcreteWriter;
+
+/**
+ * Create a column type converter for the given column and base writer.
+ * The new writer is expected to be a "shim" writer that implements
+ * additional "set" methods to convert data from the type that the
+ * client requires to the type required by the underlying vector as
+ * represented by the base writer.
+ */
+public interface ColumnConversionFactory {
+ /**
+ * Create a type conversion writer for the given column, converting data
+ * to the type needed by the base writer.
+ * @param colDefn column metadata definition
+ * @param baseWriter base column writer for the column's vector
+ * @return a new scalar writer to insert between the client and
+ * the base vector
+ */
+ ConcreteWriter newWriter(ColumnMetadata colDefn, ConcreteWriter baseWriter);
+}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/UnsupportedConversionError.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/UnsupportedConversionError.java
index dee2612..68ed5e0 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/UnsupportedConversionError.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/UnsupportedConversionError.java
@@ -28,7 +28,7 @@ public class UnsupportedConversionError extends UnsupportedOperationException {
private static final long serialVersionUID = 1L;
- private UnsupportedConversionError(String message) {
+ public UnsupportedConversionError(String message) {
super(message);
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java
index 08e4ac3..9c2e986 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractScalarWriter.java
@@ -17,16 +17,13 @@
*/
package org.apache.drill.exec.vector.accessor.writer;
-import java.math.BigDecimal;
-
import org.apache.drill.exec.record.metadata.ColumnMetadata;
import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.accessor.ColumnConversionFactory;
import org.apache.drill.exec.vector.accessor.ColumnWriterIndex;
import org.apache.drill.exec.vector.accessor.ObjectType;
import org.apache.drill.exec.vector.accessor.ScalarWriter;
-import org.apache.drill.exec.vector.accessor.UnsupportedConversionError;
import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter;
-import org.joda.time.Period;
/**
* Column writer implementation that acts as the basis for the
@@ -35,14 +32,20 @@ import org.joda.time.Period;
* method(s).
*/
-public abstract class AbstractScalarWriter implements ScalarWriter, WriterEvents {
+public abstract class AbstractScalarWriter extends ConcreteWriter {
public static class ScalarObjectWriter extends AbstractObjectWriter {
- private AbstractScalarWriter scalarWriter;
+ private ConcreteWriter scalarWriter;
- public ScalarObjectWriter(AbstractScalarWriter scalarWriter) {
- this.scalarWriter = scalarWriter;
+ public ScalarObjectWriter(ConcreteWriter scalarWriter) {
+ final ColumnMetadata metadata = scalarWriter.schema();
+ final ColumnConversionFactory factory = metadata.typeConverter();
+ if (factory == null) {
+ this.scalarWriter = scalarWriter;
+ } else {
+ this.scalarWriter = factory.newWriter(metadata, scalarWriter);
+ }
}
@Override
@@ -111,40 +114,7 @@ public abstract class AbstractScalarWriter implements ScalarWriter, WriterEvents
@Override
public void saveRow() { }
- protected UnsupportedConversionError conversionError(String javaType) {
- return UnsupportedConversionError.writeError(schema(), javaType);
- }
-
@Override
- public void setObject(Object value) {
- if (value == null) {
- setNull();
- } else if (value instanceof Integer) {
- setInt((Integer) value);
- } else if (value instanceof Long) {
- setLong((Long) value);
- } else if (value instanceof String) {
- setString((String) value);
- } else if (value instanceof BigDecimal) {
- setDecimal((BigDecimal) value);
- } else if (value instanceof Period) {
- setPeriod((Period) value);
- } else if (value instanceof byte[]) {
- byte[] bytes = (byte[]) value;
- setBytes(bytes, bytes.length);
- } else if (value instanceof Byte) {
- setInt((Byte) value);
- } else if (value instanceof Short) {
- setInt((Short) value);
- } else if (value instanceof Double) {
- setDouble((Double) value);
- } else if (value instanceof Float) {
- setDouble((Float) value);
- } else {
- throw conversionError(value.getClass().getSimpleName());
- }
- }
-
public void dump(HierarchicalFormatter format) {
format
.startObject(this)
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractWriteConvertor.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractWriteConvertor.java
new file mode 100644
index 0000000..0d0bc88
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractWriteConvertor.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.writer;
+
+import java.math.BigDecimal;
+
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.vector.accessor.ColumnWriterIndex;
+import org.apache.drill.exec.vector.accessor.ObjectType;
+import org.apache.drill.exec.vector.accessor.ScalarWriter;
+import org.apache.drill.exec.vector.accessor.ValueType;
+import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter;
+import org.joda.time.Period;
+
+/**
+ * Base class for type converting scalar column writers. All methods
+ * pass through to the base writer. Override selected "set" methods to
+ * perform the type conversion, such as overriding "setString" to convert
+ * from a string representation of a value to the actual format.
+ * <p>
+ * The {@link #setObject()} method works here: the object is passed
+ * to this class's set methods, allowing, say, setting a string object
+ * for an int column in the case above.
+ */
+
+// TODO: This organization works fine, but is a bit heavy-weight.
+// It may be time to think about separating the pure writer aspect of
+// a column writer from its plumbing aspects. That is, the base
+// ConcreteWriter class combines the public API (ScalarWriter) with
+// the internal implementation (WriterEvents) into a single class.
+// Might be worth using composition rather than inheritance to keep
+// these aspects distinct.
+
+public class AbstractWriteConvertor extends ConcreteWriter {
+
+ private final ConcreteWriter baseWriter;
+
+ public AbstractWriteConvertor(ScalarWriter baseWriter) {
+ this.baseWriter = (ConcreteWriter) baseWriter;
+ }
+
+ @Override
+ public ValueType valueType() {
+ return baseWriter.valueType();
+ }
+
+ @Override
+ public int lastWriteIndex() {
+ return baseWriter.lastWriteIndex();
+ }
+
+ @Override
+ public void restartRow() {
+ baseWriter.restartRow();
+ }
+
+ @Override
+ public void endWrite() {
+ baseWriter.endWrite();
+ }
+
+ @Override
+ public void preRollover() {
+ baseWriter.preRollover();
+ }
+
+ @Override
+ public void postRollover() {
+ baseWriter.postRollover();
+ }
+
+ @Override
+ public ObjectType type() {
+ return baseWriter.type();
+ }
+
+ @Override
+ public boolean nullable() {
+ return baseWriter.nullable();
+ }
+
+ @Override
+ public ColumnMetadata schema() {
+ return baseWriter.schema();
+ }
+
+ @Override
+ public void setNull() {
+ baseWriter.setNull();
+ }
+
+ @Override
+ public int rowStartIndex() {
+ return baseWriter.rowStartIndex();
+ }
+
+ @Override
+ public int writeIndex() {
+ return baseWriter.writeIndex();
+ }
+
+ @Override
+ public void bindListener(ColumnWriterListener listener) {
+ baseWriter.bindListener(listener);
+ }
+
+ @Override
+ public void bindIndex(ColumnWriterIndex index) {
+ baseWriter.bindIndex(index);
+ }
+
+ @Override
+ public void startWrite() {
+ baseWriter.startWrite();
+ }
+
+ @Override
+ public void startRow() {
+ baseWriter.startRow();
+ }
+
+ @Override
+ public void endArrayValue() {
+ baseWriter.endArrayValue();
+ }
+
+ @Override
+ public void saveRow() {
+ baseWriter.saveRow();
+ }
+
+ @Override
+ public void setInt(int value) {
+ baseWriter.setInt(value);
+ }
+
+ @Override
+ public void setLong(long value) {
+ baseWriter.setLong(value);
+ }
+
+ @Override
+ public void setDouble(double value) {
+ baseWriter.setDouble(value);
+ }
+
+ @Override
+ public void setString(String value) {
+ baseWriter.setString(value);
+ }
+
+ @Override
+ public void setBytes(byte[] value, int len) {
+ baseWriter.setBytes(value, len);
+ }
+
+ @Override
+ public void setDecimal(BigDecimal value) {
+ baseWriter.setDecimal(value);
+ }
+
+ @Override
+ public void setPeriod(Period value) {
+ baseWriter.setPeriod(value);
+ }
+
+ @Override
+ public void dump(HierarchicalFormatter format) {
+ baseWriter.dump(format);
+ }
+}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ConcreteWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ConcreteWriter.java
new file mode 100644
index 0000000..549431f
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ConcreteWriter.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.writer;
+
+import java.math.BigDecimal;
+
+import org.apache.drill.exec.vector.accessor.ScalarWriter;
+import org.apache.drill.exec.vector.accessor.UnsupportedConversionError;
+import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter;
+import org.joda.time.Period;
+
+/**
+ * Base class for concrete scalar column writers including actual vector
+ * writers, wrappers for nullable types, and shims used to convert types.
+ */
+
+public abstract class ConcreteWriter implements ScalarWriter, WriterEvents {
+
+ @Override
+ public void setObject(Object value) {
+ if (value == null) {
+ setNull();
+ } else if (value instanceof Integer) {
+ setInt((Integer) value);
+ } else if (value instanceof Long) {
+ setLong((Long) value);
+ } else if (value instanceof String) {
+ setString((String) value);
+ } else if (value instanceof BigDecimal) {
+ setDecimal((BigDecimal) value);
+ } else if (value instanceof Period) {
+ setPeriod((Period) value);
+ } else if (value instanceof byte[]) {
+ final byte[] bytes = (byte[]) value;
+ setBytes(bytes, bytes.length);
+ } else if (value instanceof Byte) {
+ setInt((Byte) value);
+ } else if (value instanceof Short) {
+ setInt((Short) value);
+ } else if (value instanceof Double) {
+ setDouble((Double) value);
+ } else if (value instanceof Float) {
+ setDouble((Float) value);
+ } else {
+ throw conversionError(value.getClass().getSimpleName());
+ }
+ }
+
+ protected UnsupportedConversionError conversionError(String javaType) {
+ return UnsupportedConversionError.writeError(schema(), javaType);
+ }
+
+ abstract void dump(HierarchicalFormatter format);
+}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ScalarArrayWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ScalarArrayWriter.java
index 2ac7d45..f271bfa 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ScalarArrayWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ScalarArrayWriter.java
@@ -60,13 +60,17 @@ public class ScalarArrayWriter extends BaseArrayWriter {
public final void nextElement() { next(); }
}
- private final BaseScalarWriter elementWriter;
+ private final ConcreteWriter elementWriter;
public ScalarArrayWriter(ColumnMetadata schema,
RepeatedValueVector vector, BaseScalarWriter elementWriter) {
super(schema, vector.getOffsetVector(),
new ScalarObjectWriter(elementWriter));
- this.elementWriter = elementWriter;
+
+ // Save the writer from the scalar object writer created above
+ // which may have wrapped the element writer in a type convertor.
+
+ this.elementWriter = (ConcreteWriter) elementObjWriter.scalar();
}
public static ArrayObjectWriter build(ColumnMetadata schema,
@@ -110,7 +114,7 @@ public class ScalarArrayWriter extends BaseArrayWriter {
return;
}
- String objClass = array.getClass().getName();
+ final String objClass = array.getClass().getName();
if (! objClass.startsWith("[")) {
throw new IllegalArgumentException(
String.format("Argument must be an array. Column `%s`, value = %s",
@@ -119,12 +123,12 @@ public class ScalarArrayWriter extends BaseArrayWriter {
// Figure out type
- char second = objClass.charAt(1);
+ final char second = objClass.charAt(1);
switch ( second ) {
case '[':
// bytes is represented as an array of byte arrays.
- char third = objClass.charAt(2);
+ final char third = objClass.charAt(2);
switch (third) {
case 'B':
setBytesArray((byte[][]) array);
@@ -157,11 +161,11 @@ public class ScalarArrayWriter extends BaseArrayWriter {
setBooleanArray((boolean[]) array);
break;
case 'L':
- int posn = objClass.indexOf(';');
+ final int posn = objClass.indexOf(';');
// If the array is of type Object, then we have no type info.
- String memberClassName = objClass.substring(2, posn);
+ final String memberClassName = objClass.substring(2, posn);
if (memberClassName.equals(String.class.getName())) {
setStringArray((String[]) array);
} else if (memberClassName.equals(Period.class.getName())) {
@@ -215,7 +219,7 @@ public class ScalarArrayWriter extends BaseArrayWriter {
public void setIntObjectArray(Integer[] value) {
for (int i = 0; i < value.length; i++) {
- Integer element = value[i];
+ final Integer element = value[i];
if (element == null) {
elementWriter.setNull();
} else {
@@ -232,7 +236,7 @@ public class ScalarArrayWriter extends BaseArrayWriter {
public void setLongObjectArray(Long[] value) {
for (int i = 0; i < value.length; i++) {
- Long element = value[i];
+ final Long element = value[i];
if (element == null) {
elementWriter.setNull();
} else {
@@ -255,7 +259,7 @@ public class ScalarArrayWriter extends BaseArrayWriter {
public void setDoubleObjectArray(Double[] value) {
for (int i = 0; i < value.length; i++) {
- Double element = value[i];
+ final Double element = value[i];
if (element == null) {
elementWriter.setNull();
} else {
[drill] 01/02: DRILL-7007: Use verify method in row set tests
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 85c656e12bb109147576055c1bed0e1463c32b96
Author: Paul Rogers <pr...@cloudera.com>
AuthorDate: Sun Jan 27 10:42:20 2019 -0800
DRILL-7007: Use verify method in row set tests
Many of the early RowSet-based tests used the pattern:
new RowSetComparison(expected)
.verifyAndClearAll(result);
Revise this to use the simplified form:
RowSetUtilities.verify(expected, result);
The original form is retained when tests use additional functionality, such as the ability to perform multiple verifications on the same expected batch.
closes #1624
---
.../drill/exec/cache/TestBatchSerialization.java | 4 +---
.../impl/xsort/managed/SortTestUtilities.java | 6 ++---
.../physical/impl/xsort/managed/TestSorter.java | 6 ++---
.../impl/TestResultSetLoaderEmptyProject.java | 5 ++--
.../impl/TestResultSetLoaderOmittedValues.java | 8 +++----
.../rowSet/impl/TestResultSetLoaderProjection.java | 14 ++++-------
.../rowSet/impl/TestResultSetSchemaChange.java | 5 ++--
.../exec/store/easy/text/compliant/TestCsv.java | 11 ++++-----
.../apache/drill/exec/store/log/TestLogReader.java | 28 ++++++++++------------
.../test/rowSet/test/TestIndirectReaders.java | 5 ++--
10 files changed, 36 insertions(+), 56 deletions(-)
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestBatchSerialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestBatchSerialization.java
index 8b4941c..4eb9016 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestBatchSerialization.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestBatchSerialization.java
@@ -42,7 +42,6 @@ import org.apache.drill.test.OperatorFixture;
import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
-import org.apache.drill.test.rowSet.RowSetComparison;
import org.apache.drill.test.rowSet.RowSetUtilities;
import org.apache.drill.test.rowSet.RowSetWriter;
import org.junit.AfterClass;
@@ -149,8 +148,7 @@ public class TestBatchSerialization extends DrillTest {
result = fixture.wrap(reader.read(), reader.sv2());
}
- new RowSetComparison(expected)
- .verifyAndClearAll(result);
+ RowSetUtilities.verify(expected, result);
outFile.delete();
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
index 3130c69..1bd9fe3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/SortTestUtilities.java
@@ -39,8 +39,7 @@ import org.apache.drill.test.OperatorFixture;
import org.apache.drill.test.rowSet.DirectRowSet;
import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
-import org.apache.drill.test.rowSet.RowSetComparison;
-
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
public class SortTestUtilities {
@@ -126,8 +125,7 @@ public class SortTestUtilities {
for (RowSet expectedSet : expected) {
assertTrue(merger.next());
RowSet rowSet = DirectRowSet.fromContainer(dest);
- new RowSetComparison(expectedSet)
- .verifyAndClearAll(rowSet);
+ RowSetUtilities.verify(expectedSet, rowSet);
}
assertFalse(merger.next());
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestSorter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestSorter.java
index bb3094f..3977b3e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestSorter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/managed/TestSorter.java
@@ -93,8 +93,7 @@ public class TestSorter extends DrillTest {
try {
sorter.sortBatch(rowSet.container(), rowSet.getSv2());
- new RowSetComparison(expected)
- .verifyAndClearAll(rowSet);
+ RowSetUtilities.verify(expected, rowSet);
sorter.close();
} finally {
opContext.close();
@@ -240,8 +239,7 @@ public class TestSorter extends DrillTest {
}
protected void doVerify(DataItem[] expected, RowSet expectedRows, RowSet actual) {
- new RowSetComparison(expectedRows)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expectedRows, actual);
}
protected abstract void doSort(DataItem[] expected);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderEmptyProject.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderEmptyProject.java
index c7a49d8..15fd049 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderEmptyProject.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderEmptyProject.java
@@ -34,7 +34,7 @@ import org.apache.drill.exec.record.metadata.TupleMetadata;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSetBuilder;
-import org.apache.drill.test.rowSet.RowSetComparison;
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.junit.Test;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
@@ -156,8 +156,7 @@ public class TestResultSetLoaderEmptyProject extends SubOperatorTest {
for (int i = 0; i < rowCount; i++) {
builder.addRow(0, 0);
}
- new RowSetComparison(builder.build())
- .verifyAndClearAll(fixture.wrap(rsLoader.harvest()));
+ RowSetUtilities.verify(builder.build(), fixture.wrap(rsLoader.harvest()));
rsLoader.close();
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java
index bf205a3..a888fce 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderOmittedValues.java
@@ -34,8 +34,8 @@ import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
-import org.apache.drill.test.rowSet.RowSetComparison;
import org.apache.drill.test.rowSet.RowSetReader;
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.junit.Test;
@@ -158,8 +158,7 @@ public class TestResultSetLoaderOmittedValues extends SubOperatorTest {
.addRow( 10, "", null, 0, null, strArray())
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
rsLoader.close();
}
@@ -321,8 +320,7 @@ public class TestResultSetLoaderOmittedValues extends SubOperatorTest {
.addRow(13, null)
.build();
// expected.print();
- new RowSetComparison(expected)
- .verifyAndClearAll(result);
+ RowSetUtilities.verify(expected, result);
rsLoader.close();
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java
index 9c95b5e..3ab0c3c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProjection.java
@@ -40,7 +40,7 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
-import org.apache.drill.test.rowSet.RowSetComparison;
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.junit.Test;
/**
@@ -136,8 +136,7 @@ public class TestResultSetLoaderProjection extends SubOperatorTest {
.build();
RowSet actual = fixture.wrap(rsLoader.harvest());
// actual.print();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
rsLoader.close();
}
@@ -212,8 +211,7 @@ public class TestResultSetLoaderProjection extends SubOperatorTest {
.addRow(mapValue( 1, 2), mapValue( 4))
.addRow(mapValue(11, 12), mapValue(14))
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(fixture.wrap(rsLoader.harvest()));
+ RowSetUtilities.verify(expected, fixture.wrap(rsLoader.harvest()));
rsLoader.close();
}
@@ -258,8 +256,7 @@ public class TestResultSetLoaderProjection extends SubOperatorTest {
.addSingleCol(mapValue( 1, 2))
.addSingleCol(mapValue(11, 12))
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(fixture.wrap(rsLoader.harvest()));
+ RowSetUtilities.verify(expected, fixture.wrap(rsLoader.harvest()));
rsLoader.close();
}
@@ -324,8 +321,7 @@ public class TestResultSetLoaderProjection extends SubOperatorTest {
objArray(objArray(110, 120), objArray(111, 121)),
objArray(objArray(140), objArray(142)))
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(fixture.wrap(rsLoader.harvest()));
+ RowSetUtilities.verify(expected, fixture.wrap(rsLoader.harvest()));
rsLoader.close();
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetSchemaChange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetSchemaChange.java
index 9feb51b..0055376 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetSchemaChange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetSchemaChange.java
@@ -35,8 +35,8 @@ import org.apache.drill.exec.vector.accessor.ScalarWriter;
import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
-import org.apache.drill.test.rowSet.RowSetComparison;
import org.apache.drill.test.rowSet.RowSetReader;
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.junit.Test;
public class TestResultSetSchemaChange extends SubOperatorTest {
@@ -149,8 +149,7 @@ public class TestResultSetSchemaChange extends SubOperatorTest {
.addRow("a_10", 10, "c_10", "d_10", 100, strArray("f_10-1", "f_10-2"))
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
rsLoader.close();
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
index 8ba1e7f..ec6304e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
@@ -33,7 +33,7 @@ import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.rowSet.RowSet;
import org.apache.drill.test.rowSet.RowSetBuilder;
-import org.apache.drill.test.rowSet.RowSetComparison;
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -98,8 +98,7 @@ public class TestCsv extends ClusterTest {
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
.addRow("10", "foo", "bar")
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
}
String invalidHeaders[] = {
@@ -124,8 +123,7 @@ public class TestCsv extends ClusterTest {
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
.addRow("10", "foo", "bar", "fourth", "fifth", "sixth")
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
}
// Test fix for DRILL-5590
@@ -146,8 +144,7 @@ public class TestCsv extends ClusterTest {
RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
.addRow("10", "foo", "bar")
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(actual);
+ RowSetUtilities.verify(expected, actual);
}
private String makeStatement(String fileName) {
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/log/TestLogReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/log/TestLogReader.java
index 47d1c5e..1260ad2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/log/TestLogReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/log/TestLogReader.java
@@ -30,7 +30,7 @@ import org.apache.drill.test.BaseDirTestWatcher;
import org.apache.drill.test.ClusterFixture;
import org.apache.drill.test.ClusterTest;
import org.apache.drill.test.rowSet.RowSet;
-import org.apache.drill.test.rowSet.RowSetComparison;
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
@@ -119,7 +119,6 @@ public class TestLogReader extends ClusterTest {
pluginConfig.getFormats().put("date-log",logDateConfig);
pluginConfig.getFormats().put( "mysql-log", mysqlLogConfig);
pluginRegistry.createOrUpdate("cp", pluginConfig, false);
-
}
@Test
@@ -139,7 +138,7 @@ public class TestLogReader extends ClusterTest {
.addRow(2017, 12, 19)
.build();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@Test
@@ -160,7 +159,7 @@ public class TestLogReader extends ClusterTest {
// results.print();
// expected.print();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@Test
@@ -182,7 +181,7 @@ public class TestLogReader extends ClusterTest {
// results.print();
// expected.print();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@Test
@@ -199,7 +198,7 @@ public class TestLogReader extends ClusterTest {
.addRow("2017-12-18 10:52:37,652 [main] INFO o.a.drill.common.config.DrillConfig - Configuration and plugin file(s) identified in 115ms.")
.addRow("2017-12-19 11:12:27,278 [main] ERROR o.apache.drill.exec.server.Drillbit - Failure during initial startup of Drillbit.")
.build();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@@ -216,7 +215,7 @@ public class TestLogReader extends ClusterTest {
.addRow("TIMESTAMP")
.build();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@@ -257,7 +256,7 @@ public class TestLogReader extends ClusterTest {
//results.print();
//expected.print();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@Test
@@ -281,7 +280,7 @@ public class TestLogReader extends ClusterTest {
.addRow("070917", "16:29:12", "21", "Query","select * from location where id = 1 LIMIT 1" )
.build();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@Test
@@ -302,7 +301,7 @@ public class TestLogReader extends ClusterTest {
.addRow("070917", "select * from location where id = 1 LIMIT 1" )
.build();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@Test
@@ -322,7 +321,7 @@ public class TestLogReader extends ClusterTest {
.addRow("070917 16:29:12 21 Query select * from location where id = 1 LIMIT 1" )
.build();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@Test
@@ -338,7 +337,7 @@ public class TestLogReader extends ClusterTest {
.addRow("dfadkfjaldkjafsdfjlksdjflksjdlkfjsldkfjslkjl")
.build();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
@Test
@@ -360,7 +359,6 @@ public class TestLogReader extends ClusterTest {
.addRow( null, "dfadkfjaldkjafsdfjlksdjflksjdlkfjsldkfjslkjl")
.build();
- new RowSetComparison(expected).verifyAndClearAll(results);
+ RowSetUtilities.verify(expected, results);
}
-
-}
\ No newline at end of file
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java
index 7180dbb..db1c882 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestIndirectReaders.java
@@ -29,8 +29,8 @@ import org.apache.drill.test.SubOperatorTest;
import org.apache.drill.test.rowSet.RowSetWriter;
import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet;
import org.apache.drill.test.rowSet.RowSet.SingleRowSet;
-import org.apache.drill.test.rowSet.RowSetComparison;
import org.apache.drill.test.rowSet.RowSetReader;
+import org.apache.drill.test.rowSet.RowSetUtilities;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -100,8 +100,7 @@ public class TestIndirectReaders extends SubOperatorTest {
.addRow(0)
.build();
- new RowSetComparison(expected)
- .verifyAndClearAll(result);
+ RowSetUtilities.verify(expected, result);
}
/**