You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ni...@apache.org on 2024/10/03 08:25:06 UTC
(hbase-connectors) branch master updated: HBASE-28835 Make connector support for Decimal type (#136)
This is an automated email from the ASF dual-hosted git repository.
nihaljain pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-connectors.git
The following commit(s) were added to refs/heads/master by this push:
new abf69b1 HBASE-28835 Make connector support for Decimal type (#136)
abf69b1 is described below
commit abf69b1950df5cf878dc85df5e84047fa40cda9a
Author: duanyyyyyyy <13...@users.noreply.github.com>
AuthorDate: Thu Oct 3 16:25:01 2024 +0800
HBASE-28835 Make connector support for Decimal type (#136)
---
.../scala/org/apache/hadoop/hbase/spark/datasources/Utils.scala | 2 ++
.../scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala | 8 ++++++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/spark/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Utils.scala b/spark/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Utils.scala
index de7eec9..b4e7e84 100644
--- a/spark/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Utils.scala
+++ b/spark/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Utils.scala
@@ -55,6 +55,7 @@ object Utils {
val newArray = new Array[Byte](length)
System.arraycopy(src, offset, newArray, 0, length)
newArray
+ case _: DecimalType => Bytes.toBigDecimal(src, offset, length)
// TODO: SparkSqlSerializer.deserialize[Any](src)
case _ => throw new Exception(s"unsupported data type ${f.dt}")
}
@@ -79,6 +80,7 @@ object Utils {
case DateType | TimestampType => Bytes.toBytes(input.asInstanceOf[java.util.Date].getTime)
case StringType => Bytes.toBytes(input.toString)
case BinaryType => input.asInstanceOf[Array[Byte]]
+ case _: DecimalType => Bytes.toBytes(input.asInstanceOf[java.math.BigDecimal])
case _ => throw new Exception(s"unsupported data type ${field.dt}")
}
}
diff --git a/spark/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala b/spark/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala
index cf92d08..0fab274 100644
--- a/spark/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala
+++ b/spark/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala
@@ -96,6 +96,7 @@ class DefaultSourceSuite
val columnFamily = "c"
val timestamp = 1234567890000L
+ val decimal = new java.math.BigDecimal("1234.56")
var sqlContext: SQLContext = null
var df: DataFrame = null
@@ -235,6 +236,7 @@ class DefaultSourceSuite
Bytes.toBytes("timestamp"),
Bytes.toBytes(timestamp))
put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes("string"), Bytes.toBytes("string"))
+ put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes("decimal"), Bytes.toBytes(decimal))
t3Table.put(put)
} finally {
t3Table.close()
@@ -990,7 +992,8 @@ class DefaultSourceSuite
|"DOUBLE_FIELD":{"cf":"c", "col":"double", "type":"double"},
|"DATE_FIELD":{"cf":"c", "col":"date", "type":"date"},
|"TIMESTAMP_FIELD":{"cf":"c", "col":"timestamp", "type":"timestamp"},
- |"STRING_FIELD":{"cf":"c", "col":"string", "type":"string"}
+ |"STRING_FIELD":{"cf":"c", "col":"string", "type":"string"},
+ |"DECIMAL_FIELD":{"cf":"c", "col":"decimal", "type":"decimal(10,2)"}
|}
|}""".stripMargin
df = sqlContext.load(
@@ -1004,7 +1007,7 @@ class DefaultSourceSuite
"SELECT binary_field, boolean_field, " +
"byte_field, short_field, int_field, long_field, " +
"float_field, double_field, date_field, timestamp_field, " +
- "string_field FROM hbaseTestMapping")
+ "string_field, decimal_field FROM hbaseTestMapping")
.collect()
assert(results.length == 1)
@@ -1030,6 +1033,7 @@ class DefaultSourceSuite
assert(Math.abs(result.get(8).asInstanceOf[Date].getTime - timestamp) <= 86400000)
assert(result.get(9).asInstanceOf[Timestamp].getTime == timestamp)
assert(result.get(10) == "string")
+ assert(result.get(11) == decimal)
}
def writeCatalog = s"""{