You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2019/03/08 18:14:26 UTC
[spark] branch master updated: [SPARK-27079][MINOR][SQL] Fix typo &
Remove useless imports & Add missing `override` annotation
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d3d9c7b [SPARK-27079][MINOR][SQL] Fix typo & Remove useless imports & Add missing `override` annotation
d3d9c7b is described below
commit d3d9c7bb0a5397c88e601703ba2b69fc4b92b0a5
Author: wangguangxin.cn <wa...@gmail.com>
AuthorDate: Fri Mar 8 12:14:04 2019 -0600
[SPARK-27079][MINOR][SQL] Fix typo & Remove useless imports & Add missing `override` annotation
## What changes were proposed in this pull request?
1. Fix two typos
2. Remove useless imports in `CSVExprUtils.scala`
3. Add missing `override` annotation
## How was this patch tested?
test by existing uts
Closes #24000 from WangGuangxin/SPARK-27079.
Authored-by: wangguangxin.cn <wa...@gmail.com>
Signed-off-by: Sean Owen <se...@databricks.com>
---
.../spark/sql/catalyst/expressions/UnsafeArrayData.java | 8 ++++++++
.../sql/catalyst/expressions/codegen/UnsafeArrayWriter.java | 12 ++++++++++++
.../sql/catalyst/expressions/codegen/UnsafeRowWriter.java | 8 ++++++++
.../org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala | 4 ----
.../spark/sql/catalyst/encoders/ExpressionEncoder.scala | 2 +-
.../spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala | 2 +-
6 files changed, 30 insertions(+), 6 deletions(-)
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
index d5f679f..a07d6de 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
@@ -300,6 +300,7 @@ public final class UnsafeArrayData extends ArrayData {
@Override
public void update(int ordinal, Object value) { throw new UnsupportedOperationException(); }
+ @Override
public void setNullAt(int ordinal) {
assertIndexIsValid(ordinal);
BitSetMethods.set(baseObject, baseOffset + 8, ordinal);
@@ -308,36 +309,43 @@ public final class UnsafeArrayData extends ArrayData {
will be set to 0 later by the caller side */
}
+ @Override
public void setBoolean(int ordinal, boolean value) {
assertIndexIsValid(ordinal);
Platform.putBoolean(baseObject, getElementOffset(ordinal, 1), value);
}
+ @Override
public void setByte(int ordinal, byte value) {
assertIndexIsValid(ordinal);
Platform.putByte(baseObject, getElementOffset(ordinal, 1), value);
}
+ @Override
public void setShort(int ordinal, short value) {
assertIndexIsValid(ordinal);
Platform.putShort(baseObject, getElementOffset(ordinal, 2), value);
}
+ @Override
public void setInt(int ordinal, int value) {
assertIndexIsValid(ordinal);
Platform.putInt(baseObject, getElementOffset(ordinal, 4), value);
}
+ @Override
public void setLong(int ordinal, long value) {
assertIndexIsValid(ordinal);
Platform.putLong(baseObject, getElementOffset(ordinal, 8), value);
}
+ @Override
public void setFloat(int ordinal, float value) {
assertIndexIsValid(ordinal);
Platform.putFloat(baseObject, getElementOffset(ordinal, 4), value);
}
+ @Override
public void setDouble(int ordinal, double value) {
assertIndexIsValid(ordinal);
Platform.putDouble(baseObject, getElementOffset(ordinal, 8), value);
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java
index 997eecd..bf67923 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriter.java
@@ -82,24 +82,28 @@ public final class UnsafeArrayWriter extends UnsafeWriter {
BitSetMethods.set(getBuffer(), startingOffset + 8, ordinal);
}
+ @Override
public void setNull1Bytes(int ordinal) {
setNullBit(ordinal);
// put zero into the corresponding field when set null
writeByte(getElementOffset(ordinal), (byte)0);
}
+ @Override
public void setNull2Bytes(int ordinal) {
setNullBit(ordinal);
// put zero into the corresponding field when set null
writeShort(getElementOffset(ordinal), (short)0);
}
+ @Override
public void setNull4Bytes(int ordinal) {
setNullBit(ordinal);
// put zero into the corresponding field when set null
writeInt(getElementOffset(ordinal), 0);
}
+ @Override
public void setNull8Bytes(int ordinal) {
setNullBit(ordinal);
// put zero into the corresponding field when set null
@@ -108,41 +112,49 @@ public final class UnsafeArrayWriter extends UnsafeWriter {
public void setNull(int ordinal) { setNull8Bytes(ordinal); }
+ @Override
public void write(int ordinal, boolean value) {
assertIndexIsValid(ordinal);
writeBoolean(getElementOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, byte value) {
assertIndexIsValid(ordinal);
writeByte(getElementOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, short value) {
assertIndexIsValid(ordinal);
writeShort(getElementOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, int value) {
assertIndexIsValid(ordinal);
writeInt(getElementOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, long value) {
assertIndexIsValid(ordinal);
writeLong(getElementOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, float value) {
assertIndexIsValid(ordinal);
writeFloat(getElementOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, double value) {
assertIndexIsValid(ordinal);
writeDouble(getElementOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, Decimal input, int precision, int scale) {
// make sure Decimal object has the same scale as DecimalType
assertIndexIsValid(ordinal);
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
index d2298aa..5828823 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
@@ -135,44 +135,52 @@ public final class UnsafeRowWriter extends UnsafeWriter {
return startingOffset + nullBitsSize + 8L * ordinal;
}
+ @Override
public void write(int ordinal, boolean value) {
final long offset = getFieldOffset(ordinal);
writeLong(offset, 0L);
writeBoolean(offset, value);
}
+ @Override
public void write(int ordinal, byte value) {
final long offset = getFieldOffset(ordinal);
writeLong(offset, 0L);
writeByte(offset, value);
}
+ @Override
public void write(int ordinal, short value) {
final long offset = getFieldOffset(ordinal);
writeLong(offset, 0L);
writeShort(offset, value);
}
+ @Override
public void write(int ordinal, int value) {
final long offset = getFieldOffset(ordinal);
writeLong(offset, 0L);
writeInt(offset, value);
}
+ @Override
public void write(int ordinal, long value) {
writeLong(getFieldOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, float value) {
final long offset = getFieldOffset(ordinal);
writeLong(offset, 0);
writeFloat(offset, value);
}
+ @Override
public void write(int ordinal, double value) {
writeDouble(getFieldOffset(ordinal), value);
}
+ @Override
public void write(int ordinal, Decimal input, int precision, int scale) {
if (precision <= Decimal.MAX_LONG_DIGITS()) {
// make sure Decimal object has the same scale as DecimalType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
index 6c982a1..bbe2783 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
@@ -17,10 +17,6 @@
package org.apache.spark.sql.catalyst.csv
-import java.math.BigDecimal
-import java.text.{DecimalFormat, DecimalFormatSymbols, ParsePosition}
-import java.util.Locale
-
object CSVExprUtils {
/**
* Filter ignorable rows for CSV iterator (lines empty and starting with `comment`).
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
index abffda7..bcc4896 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
@@ -242,7 +242,7 @@ case class ExpressionEncoder[T](
}
// The schema after converting `T` to a Spark SQL row. This schema is dependent on the given
- // serialier.
+ // serializer.
val schema: StructType = StructType(serializer.map { s =>
StructField(s.name, s.dataType, s.nullable)
})
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala
index 6d1af12..9bf864f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala
@@ -240,7 +240,7 @@ class ReplaceOperatorSuite extends PlanTest {
comparePlans(result, correctAnswer)
}
- test("SPARK-26366: ReplaceExceptWithFilter should not transform non-detrministic") {
+ test("SPARK-26366: ReplaceExceptWithFilter should not transform non-deterministic") {
val basePlan = LocalRelation(Seq('a.int, 'b.int))
val otherPlan = basePlan.where('a > rand(1L))
val except = Except(basePlan, otherPlan, false)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org