You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/25 14:31:44 UTC
[doris] branch master updated: [improvement](mutil-catalog) PaimonColumnValue support short and Decimal (#20723)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8f7a62c79b [improvement](mutil-catalog) PaimonColumnValue support short and Decimal (#20723)
8f7a62c79b is described below
commit 8f7a62c79bca69a7444780fcdb9e94a218b9f3a2
Author: yuxuan-luo <11...@users.noreply.github.com>
AuthorDate: Sun Jun 25 22:31:38 2023 +0800
[improvement](mutil-catalog) PaimonColumnValue support short and Decimal (#20723)
---
.../src/main/java/org/apache/doris/paimon/PaimonColumnValue.java | 7 +++++--
.../src/main/java/org/apache/doris/paimon/PaimonJniScanner.java | 8 ++++----
.../org/apache/doris/catalog/external/PaimonExternalTable.java | 2 ++
.../apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java | 2 +-
.../org/apache/doris/planner/external/paimon/PaimonScanNode.java | 7 ++++++-
5 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
index d10c876de0..e6ab3e76ac 100644
--- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
+++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
@@ -17,6 +17,7 @@
package org.apache.doris.paimon;
+import org.apache.doris.common.jni.vec.ColumnType;
import org.apache.doris.common.jni.vec.ColumnValue;
import org.apache.paimon.data.columnar.ColumnarRow;
@@ -32,12 +33,14 @@ import java.util.List;
public class PaimonColumnValue implements ColumnValue {
private int idx;
private ColumnarRow record;
+ ColumnType dorisType;
public PaimonColumnValue() {
}
- public void setIdx(int idx) {
+ public void setIdx(int idx, ColumnType dorisType) {
this.idx = idx;
+ this.dorisType = dorisType;
}
public void setOffsetRow(ColumnarRow record) {
@@ -91,7 +94,7 @@ public class PaimonColumnValue implements ColumnValue {
@Override
public BigDecimal getDecimal() {
- return BigDecimal.valueOf(getDouble());
+ return record.getDecimal(idx, dorisType.getPrecision(), dorisType.getScale()).toBigDecimal();
}
@Override
diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 190bdb829a..71965344a4 100644
--- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -70,12 +70,12 @@ public class PaimonJniScanner extends JniScanner {
warehouse = params.get("warehouse");
splitAddress = Long.parseLong(params.get("split_byte"));
lengthByte = Integer.parseInt(params.get("length_byte"));
- LOG.info("splitAddress:" + splitAddress);
- LOG.info("lengthByte:" + lengthByte);
dbName = params.get("db_name");
tblName = params.get("table_name");
String[] requiredFields = params.get("required_fields").split(",");
- String[] types = params.get("columns_types").split(",");
+ String[] types = Arrays.stream(params.get("columns_types").split("#"))
+ .map(s -> s.replaceAll("\\s+", ""))
+ .toArray(String[]::new);
ids = params.get("columns_id").split(",");
ColumnType[] columnTypes = new ColumnType[types.length];
for (int i = 0; i < types.length; i++) {
@@ -126,7 +126,7 @@ public class PaimonJniScanner extends JniScanner {
while ((record = batch.next()) != null) {
columnValue.setOffsetRow((ColumnarRow) record);
for (int i = 0; i < ids.length; i++) {
- columnValue.setIdx(i);
+ columnValue.setIdx(i, types[i]);
appendData(i, columnValue);
}
rows++;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
index c821160dd4..4d08d8883f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
@@ -92,6 +92,8 @@ public class PaimonExternalTable extends ExternalTable {
return Type.FLOAT;
case DOUBLE:
return Type.DOUBLE;
+ case SMALLINT:
+ return Type.SMALLINT;
case VARCHAR:
case BINARY:
case CHAR:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java
index 13775b0edf..50c92f8ebf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java
@@ -62,11 +62,11 @@ public class PaimonHMSExternalCatalog extends PaimonExternalCatalog {
super(catalogId, name, comment);
props = PropertyConverter.convertToMetaProperties(props);
catalogProperty = new CatalogProperty(resource, props);
- paimonCatalogType = PAIMON_HMS;
}
@Override
protected void initLocalObjectsImpl() {
+ paimonCatalogType = PAIMON_HMS;
String metastoreUris = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, "");
String warehouse = catalogProperty.getOrDefault(PaimonProperties.WAREHOUSE, "");
Options options = new Options();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
index 13da29b2bd..d8fcca48ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
@@ -93,9 +93,14 @@ public class PaimonScanNode extends FileQueryScanNode {
}
boolean isFirst = true;
for (SlotDescriptor slot : source.getDesc().getSlots()) {
+ // for example
+ // select a,b,c,d,e,f,g from paimon;
+ // columnNamesBuilder: a,b,c,d,e,f,g
+ // columnIdsBuilder: 0,1,2,3,4,5,6
+ // columnTypesBuilder: INT#STRING#BOOLEAN#BIGINT#FLOAT#DOUBLE#DECIMAL(10, 0)
if (!isFirst) {
columnNamesBuilder.append(",");
- columnTypesBuilder.append(",");
+ columnTypesBuilder.append("#");
columnIdsBuilder.append(",");
}
columnNamesBuilder.append(slot.getColumn().getName());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org