You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by GitBox <gi...@apache.org> on 2022/07/25 21:55:28 UTC
[GitHub] [drill] vvysotskyi opened a new pull request, #2613: DRILL-8272: Skip MAP column without children when creating parquet tables
vvysotskyi opened a new pull request, #2613:
URL: https://github.com/apache/drill/pull/2613
# [DRILL-8272](https://issues.apache.org/jira/browse/DRILL-8272): Skip MAP column without children when creating parquet tables
## Description
TBA
## Documentation
NA
## Testing
Unit tests pass
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@drill.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [drill] jnturton commented on a diff in pull request #2613: DRILL-8272: Skip MAP column without children when creating parquet tables
Posted by GitBox <gi...@apache.org>.
jnturton commented on code in PR #2613:
URL: https://github.com/apache/drill/pull/2613#discussion_r929453717
##########
exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java:
##########
@@ -94,23 +94,165 @@ public void setUp(MessageType schema, RecordConsumer consumer) {
protected abstract PrimitiveType getPrimitiveType(MaterializedField field);
+ public abstract class BaseFieldConverter extends FieldConverter {
+
+ public BaseFieldConverter(int fieldId, String fieldName, FieldReader reader) {
+ super(fieldId, fieldName, reader);
+ }
+
+ public abstract void read();
+
+ public abstract void read(int i);
+
+ public abstract void consume();
+
+ @Override
+ public void writeField() throws IOException {
+ read();
+ consume();
+ }
+ }
+
+ public class NullableFieldConverter extends FieldConverter {
+ private BaseFieldConverter delegate;
+
+ public NullableFieldConverter(int fieldId, String fieldName, FieldReader reader, BaseFieldConverter delegate) {
+ super(fieldId, fieldName, reader);
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void writeField() throws IOException {
+ if (!reader.isSet()) {
+ return;
+ }
+ consumer.startField(fieldName, fieldId);
+ delegate.writeField();
+ consumer.endField(fieldName, fieldId);
+ }
+
+ public void setPosition(int index) {
+ delegate.setPosition(index);
+ }
+
+ public void startField() throws IOException {
+ delegate.startField();
+ }
+
+ public void endField() throws IOException {
+ delegate.endField();
+ }
+ }
+
+ public class RequiredFieldConverter extends FieldConverter {
+ private BaseFieldConverter delegate;
+
+ public RequiredFieldConverter(int fieldId, String fieldName, FieldReader reader, BaseFieldConverter delegate) {
+ super(fieldId, fieldName, reader);
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void writeField() throws IOException {
+ consumer.startField(fieldName, fieldId);
+ delegate.writeField();
+ consumer.endField(fieldName, fieldId);
+ }
+
+ public void setPosition(int index) {
+ delegate.setPosition(index);
+ }
+
+ public void startField() throws IOException {
+ delegate.startField();
+ }
+
+ public void endField() throws IOException {
+ delegate.endField();
+ }
+ }
+
+ public class RepeatedFieldConverter extends FieldConverter {
+
+ private BaseFieldConverter delegate;
+
+ public RepeatedFieldConverter(int fieldId, String fieldName, FieldReader reader, BaseFieldConverter delegate) {
+ super(fieldId, fieldName, reader);
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void writeField() throws IOException {
+ // empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements
+ if (reader.size() == 0) {
+ return;
+ }
+ consumer.startField(fieldName, fieldId);
+ for (int i = 0; i < reader.size(); i++) {
+ delegate.read(i);
+ delegate.consume();
+ }
+ consumer.endField(fieldName, fieldId);
+ }
+
+ @Override
+ public void writeListField() {
+ if (reader.size() == 0) {
+ return;
+ }
+ consumer.startField(LIST, ZERO_IDX);
+ for (int i = 0; i < reader.size(); i++) {
+ consumer.startGroup();
+ consumer.startField(ELEMENT, ZERO_IDX);
+
+ delegate.read(i);
+ delegate.consume();
+
+ consumer.endField(ELEMENT, ZERO_IDX);
+ consumer.endGroup();
+ }
+ consumer.endField(LIST, ZERO_IDX);
+ }
+
+ public void setPosition(int index) {
+ delegate.setPosition(index);
+ }
+
+ public void startField() throws IOException {
+ delegate.startField();
+ }
+
+ public void endField() throws IOException {
+ delegate.endField();
+ }
+ }
+
<#list vv.types as type>
<#list type.minor as minor>
<#list vv.modes as mode>
@Override
public FieldConverter getNew${mode.prefix}${minor.class}Converter(int fieldId, String fieldName, FieldReader reader) {
- return new ${mode.prefix}${minor.class}ParquetConverter(fieldId, fieldName, reader);
+ BaseFieldConverter converter = new ${minor.class}ParquetConverter(fieldId, fieldName, reader);
+ <#if mode.prefix == "Nullable">
+ return new NullableFieldConverter(fieldId, fieldName, reader, converter);
+ <#elseif mode.prefix == "Repeated">
+ return new RepeatedFieldConverter(fieldId, fieldName, reader, converter);
+ <#else>
+ return new RequiredFieldConverter(fieldId, fieldName, reader, converter);
+ </#if>
}
- public class ${mode.prefix}${minor.class}ParquetConverter extends FieldConverter {
+ </#list>
+
+ public class ${minor.class}ParquetConverter extends BaseFieldConverter {
private Nullable${minor.class}Holder holder = new Nullable${minor.class}Holder();
<#if minor.class?contains("Interval")>
private final byte[] output = new byte[12];
<#elseif minor.class == "VarDecimal">
private final DecimalValueWriter decimalValueWriter;
</#if>
- public ${mode.prefix}${minor.class}ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
+ public ${minor.class}ParquetConverter(int fieldId, String fieldName, FieldReader reader) {
Review Comment:
It's nice to have the template logic that follows simplified, thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@drill.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [drill] jnturton merged pull request #2613: DRILL-8272: Skip MAP column without children when creating parquet tables
Posted by GitBox <gi...@apache.org>.
jnturton merged PR #2613:
URL: https://github.com/apache/drill/pull/2613
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@drill.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org