You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2017/04/03 05:12:05 UTC
[5/9] drill git commit: DRILL-5351: Minimize bounds checking in var
len vectors for Parquet reader
DRILL-5351: Minimize bounds checking in var len vectors for Parquet reader
close #781
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/152c87aa
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/152c87aa
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/152c87aa
Branch: refs/heads/master
Commit: 152c87aa6cad84c8752b4a87967c7826cc90dbaa
Parents: 0ded1d0
Author: Parth Chandra <pc...@maprtech.com>
Authored: Fri Feb 10 17:40:25 2017 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Fri Mar 31 15:32:13 2017 -0700
----------------------------------------------------------------------
.../codegen/templates/NullableValueVectors.java | 24 +++++--
.../templates/VariableLengthVectors.java | 72 +++++++++++++-------
2 files changed, 65 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/152c87aa/exec/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
index b242728..fcbe79a 100644
--- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -509,7 +509,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if type.major != "VarLen">
throw new UnsupportedOperationException();
<#else>
- fillEmpties(index);
+ if (index > lastSet + 1) {
+ fillEmpties(index);
+ }
bits.getMutator().setSafe(index, 1);
values.getMutator().setSafe(index, value, start, length);
@@ -522,7 +524,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if type.major != "VarLen">
throw new UnsupportedOperationException();
<#else>
- fillEmpties(index);
+ if (index > lastSet + 1) {
+ fillEmpties(index);
+ }
bits.getMutator().setSafe(index, 1);
values.getMutator().setSafe(index, value, start, length);
@@ -587,7 +591,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
<#if type.major == "VarLen">
- fillEmpties(index);
+ if (index > lastSet + 1) {
+ fillEmpties(index);
+ }
</#if>
bits.getMutator().setSafe(index, isSet);
@@ -600,7 +606,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
public void setSafe(int index, Nullable${minor.class}Holder value) {
<#if type.major == "VarLen">
- fillEmpties(index);
+ if (index > lastSet + 1) {
+ fillEmpties(index);
+ }
</#if>
bits.getMutator().setSafe(index, value.isSet);
values.getMutator().setSafe(index, value);
@@ -611,7 +619,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
public void setSafe(int index, ${minor.class}Holder value) {
<#if type.major == "VarLen">
- fillEmpties(index);
+ if (index > lastSet + 1) {
+ fillEmpties(index);
+ }
</#if>
bits.getMutator().setSafe(index, 1);
values.getMutator().setSafe(index, value);
@@ -622,7 +632,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")>
public void setSafe(int index, ${minor.javaType!type.javaType} value) {
<#if type.major == "VarLen">
- fillEmpties(index);
+ if (index > lastSet + 1) {
+ fillEmpties(index);
+ }
</#if>
bits.getMutator().setSafe(index, 1);
values.getMutator().setSafe(index, value);
http://git-wip-us.apache.org/repos/asf/drill/blob/152c87aa/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
index bb1d4fb..e970271 100644
--- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -507,11 +507,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
assert index >= 0;
final int currentOffset = offsetVector.getAccessor().get(index);
- while (data.capacity() < currentOffset + bytes.length) {
- reAlloc();
- }
offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length);
- data.setBytes(currentOffset, bytes, 0, bytes.length);
+ try {
+ data.setBytes(currentOffset, bytes, 0, bytes.length);
+ } catch (IndexOutOfBoundsException e) {
+ while (data.capacity() < currentOffset + bytes.length) {
+ reAlloc();
+ }
+ data.setBytes(currentOffset, bytes, 0, bytes.length);
+ }
}
/**
@@ -533,12 +537,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
assert index >= 0;
int currentOffset = offsetVector.getAccessor().get(index);
-
- while (data.capacity() < currentOffset + length) {
- reAlloc();
- }
offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
- data.setBytes(currentOffset, bytes, start, length);
+ try {
+ data.setBytes(currentOffset, bytes, start, length);
+ } catch (IndexOutOfBoundsException e) {
+ while (data.capacity() < currentOffset + length) {
+ reAlloc();
+ }
+ data.setBytes(currentOffset, bytes, start, length);
+ }
}
public void setSafe(int index, byte[] bytes, int start, int length) {
@@ -546,11 +553,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
final int currentOffset = offsetVector.getAccessor().get(index);
- while (data.capacity() < currentOffset + length) {
- reAlloc();
- }
offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
- data.setBytes(currentOffset, bytes, start, length);
+ try {
+ data.setBytes(currentOffset, bytes, start, length);
+ } catch (IndexOutOfBoundsException e) {
+ while (data.capacity() < currentOffset + length) {
+ reAlloc();
+ }
+ data.setBytes(currentOffset, bytes, start, length);
+ }
}
@Override
@@ -567,12 +578,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
final int len = end - start;
final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
- while(data.capacity() < outputStart + len) {
- reAlloc();
+ offsetVector.getMutator().setSafe( index+1, outputStart + len);
+ try{
+ buffer.getBytes(start, data, outputStart, len);
+ } catch (IndexOutOfBoundsException e) {
+ while (data.capacity() < outputStart + len) {
+ reAlloc();
+ }
+ buffer.getBytes(start, data, outputStart, len);
}
- offsetVector.getMutator().setSafe( index+1, outputStart + len);
- buffer.getBytes(start, data, outputStart, len);
}
public void setSafe(int index, Nullable${minor.class}Holder holder){
@@ -584,11 +599,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
- while(data.capacity() < outputStart + len) {
- reAlloc();
+ try {
+ holder.buffer.getBytes(start, data, outputStart, len);
+ } catch (IndexOutOfBoundsException e) {
+ while (data.capacity() < outputStart + len) {
+ reAlloc();
+ }
+ holder.buffer.getBytes(start, data, outputStart, len);
}
-
- holder.buffer.getBytes(start, data, outputStart, len);
offsetVector.getMutator().setSafe( index+1, outputStart + len);
}
@@ -598,11 +616,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
final int len = end - start;
final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
- while(data.capacity() < outputStart + len) {
- reAlloc();
- }
- holder.buffer.getBytes(start, data, outputStart, len);
+ try {
+ holder.buffer.getBytes(start, data, outputStart, len);
+ } catch (IndexOutOfBoundsException e) {
+ while(data.capacity() < outputStart + len) {
+ reAlloc();
+ }
+ holder.buffer.getBytes(start, data, outputStart, len);
+ }
offsetVector.getMutator().setSafe( index+1, outputStart + len);
}