You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2021/04/26 07:24:57 UTC
[GitHub] [spark] Peng-Lei commented on a change in pull request #32340: [SPARK-35139][SQL]Support ANSI intervals as Arrow Column vectors
Peng-Lei commented on a change in pull request #32340:
URL: https://github.com/apache/spark/pull/32340#discussion_r620035312
##########
File path: sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
##########
@@ -508,4 +516,39 @@ final ColumnarMap getMap(int rowId) {
super(vector);
}
}
+
+ private static class IntervalYearAccessor extends ArrowVectorAccessor {
+
+ private final IntervalYearVector accessor;
+
+ IntervalYearAccessor(IntervalYearVector vector) {
+ super(vector);
+ this.accessor = vector;
+ }
+
+ @Override
+ int getInt(int rowId) {
+ int months = accessor.get(rowId);
Review comment:
resolved
##########
File path: sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
##########
@@ -508,4 +516,39 @@ final ColumnarMap getMap(int rowId) {
super(vector);
}
}
+
+ private static class IntervalYearAccessor extends ArrowVectorAccessor {
+
+ private final IntervalYearVector accessor;
+
+ IntervalYearAccessor(IntervalYearVector vector) {
+ super(vector);
+ this.accessor = vector;
+ }
+
+ @Override
+ int getInt(int rowId) {
+ int months = accessor.get(rowId);
+ return months;
+ }
+ }
+
+ private static class IntervalDayAccessor extends ArrowVectorAccessor {
+
+ private final IntervalDayVector accessor;
+ private final NullableIntervalDayHolder intervalDayHolder = new NullableIntervalDayHolder();
+
+ IntervalDayAccessor(IntervalDayVector vector) {
+ super(vector);
+ this.accessor = vector;
+ }
+
+ @Override
+ long getLong(int rowId) {
+ accessor.get(rowId, intervalDayHolder);
+ final long microseconds = intervalDayHolder.days * MICROS_PER_DAY
+ + (long)intervalDayHolder.milliseconds * MICROS_PER_MILLIS;
Review comment:
resolved
##########
File path: sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowWriter.scala
##########
@@ -394,3 +397,29 @@ private[arrow] class NullWriter(val valueVector: NullVector) extends ArrowFieldW
override def setValue(input: SpecializedGetters, ordinal: Int): Unit = {
}
}
+
+private[arrow] class IntervalYearWriter(val valueVector: IntervalYearVector)
+ extends ArrowFieldWriter {
+ override def setNull(): Unit = {
+ valueVector.setNull(count)
+ }
+
+ override def setValue(input: SpecializedGetters, ordinal: Int): Unit = {
+ valueVector.setSafe(count, input.getInt(ordinal));
+ }
+}
+
+private[arrow] class IntervalDayWriter(val valueVector: IntervalDayVector)
+ extends ArrowFieldWriter {
+ override def setNull(): Unit = {
+ valueVector.setNull(count)
+ }
+
+ override def setValue(input: SpecializedGetters, ordinal: Int): Unit = {
+ val totalMicroseconds = input.getLong(ordinal)
+ val days = totalMicroseconds / MICROS_PER_DAY
+ val millis = (totalMicroseconds - days * MICROS_PER_DAY) / MICROS_PER_MILLIS
Review comment:
resolved
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org