You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/11 12:26:47 UTC
[doris] branch branch-1.2-lts updated (1f0cb7dda3 -> 5d9dfe67c8)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from 1f0cb7dda3 [fix](decimalv2) fix loss of precision when cast to decimalv2 literal (#15629)
new e6b574e024 [Bug](schema scanner) Fix wrong type in schema scanner (#15768)
new 5d9dfe67c8 [fix](DOE): Fix query _id error and es properties error (#15792)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../exec/schema_scanner/schema_columns_scanner.cpp | 20 +++++++++++-
.../java/org/apache/doris/catalog/EsResource.java | 4 +++
.../java/org/apache/doris/catalog/EsTable.java | 36 +++++++++-------------
.../apache/doris/datasource/EsExternalCatalog.java | 28 ++++++-----------
.../doris/external/elasticsearch/MappingPhase.java | 22 +++++++------
5 files changed, 58 insertions(+), 52 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/02: [Bug](schema scanner) Fix wrong type in schema scanner (#15768)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e6b574e024fce11e37e43636c145fd0fa61df9da
Author: Gabriel <ga...@gmail.com>
AuthorDate: Wed Jan 11 08:37:39 2023 +0800
[Bug](schema scanner) Fix wrong type in schema scanner (#15768)
---
.../exec/schema_scanner/schema_columns_scanner.cpp | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp
index 3ccdfc42ab..fcf78542bf 100644
--- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp
@@ -149,7 +149,7 @@ std::string SchemaColumnsScanner::type_to_string(TColumnDesc& desc) {
case TPrimitiveType::BIGINT:
return "bigint(20)";
case TPrimitiveType::LARGEINT:
- return "bigint(20) unsigned";
+ return "largeint";
case TPrimitiveType::FLOAT:
return "float";
case TPrimitiveType::DOUBLE:
@@ -189,6 +189,24 @@ std::string SchemaColumnsScanner::type_to_string(TColumnDesc& desc) {
stream << ")";
return stream.str();
}
+ case TPrimitiveType::DECIMAL32:
+ case TPrimitiveType::DECIMAL64:
+ case TPrimitiveType::DECIMAL128I: {
+ fmt::memory_buffer debug_string_buffer;
+ fmt::format_to(
+ debug_string_buffer, "decimalv3({}, {})",
+ desc.__isset.columnPrecision ? std::to_string(desc.columnPrecision) : "UNKNOWN",
+ desc.__isset.columnScale ? std::to_string(desc.columnScale) : "UNKNOWN");
+ return fmt::to_string(debug_string_buffer);
+ }
+ case TPrimitiveType::DATEV2:
+ return "datev2";
+ case TPrimitiveType::DATETIMEV2: {
+ fmt::memory_buffer debug_string_buffer;
+ fmt::format_to(debug_string_buffer, "datetimev2({})",
+ desc.__isset.columnScale ? std::to_string(desc.columnScale) : "UNKNOWN");
+ return fmt::to_string(debug_string_buffer);
+ }
case TPrimitiveType::HLL: {
return "hll";
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/02: [fix](DOE): Fix query _id error and es properties error (#15792)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5d9dfe67c8b97be8e4775b47536d8ccf73a44744
Author: Stalary <st...@163.com>
AuthorDate: Wed Jan 11 17:00:59 2023 +0800
[fix](DOE): Fix query _id error and es properties error (#15792)
Fix query _id error
_id not exist mapping, but be can query it, we need skip check it exist mapping.
---
.../java/org/apache/doris/catalog/EsResource.java | 4 +++
.../java/org/apache/doris/catalog/EsTable.java | 36 +++++++++-------------
.../apache/doris/datasource/EsExternalCatalog.java | 28 ++++++-----------
.../doris/external/elasticsearch/MappingPhase.java | 22 +++++++------
4 files changed, 39 insertions(+), 51 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
index 32c44671d0..131ca70e09 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
@@ -61,6 +61,10 @@ public class EsResource extends Resource {
public static final String HTTP_SSL_ENABLED = "http_ssl_enabled";
public static final String QUERY_DSL = "query_dsl";
+ public static final String DOC_VALUE_SCAN_DEFAULT_VALUE = "true";
+ public static final String KEYWORD_SNIFF_DEFAULT_VALUE = "true";
+ public static final String HTTP_SSL_ENABLED_DEFAULT_VALUE = "false";
+ public static final String NODES_DISCOVERY_DEFAULT_VALUE = "true";
@SerializedName(value = "properties")
private Map<String, String> properties;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
index 54a294e1aa..0c89a3b0d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
@@ -50,9 +50,9 @@ import java.util.Set;
@Getter
@Setter
public class EsTable extends Table {
- private static final Logger LOG = LogManager.getLogger(EsTable.class);
-
public static final Set<String> DEFAULT_DOCVALUE_DISABLED_FIELDS = new HashSet<>(Arrays.asList("text"));
+
+ private static final Logger LOG = LogManager.getLogger(EsTable.class);
// Solr doc_values vs stored_fields performance-smackdown indicate:
// It is possible to notice that retrieving an high number of fields leads
// to a sensible worsening of performance if DocValues are used.
@@ -80,16 +80,16 @@ public class EsTable extends Table {
private EsTablePartitions esTablePartitions;
// Whether to enable docvalues scan optimization for fetching fields more fast, default to true
- private boolean enableDocValueScan = true;
+ private boolean enableDocValueScan = Boolean.parseBoolean(EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE);
// Whether to enable sniffing keyword for filtering more reasonable, default to true
- private boolean enableKeywordSniff = true;
+ private boolean enableKeywordSniff = Boolean.parseBoolean(EsResource.KEYWORD_SNIFF_DEFAULT_VALUE);
// if the number of fields which value extracted from `doc_value` exceeding this max limitation
// would downgrade to extract value from `stored_fields`
private int maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS;
- private boolean nodesDiscovery = true;
+ private boolean nodesDiscovery = Boolean.parseBoolean(EsResource.NODES_DISCOVERY_DEFAULT_VALUE);
- private boolean httpSslEnabled = false;
+ private boolean httpSslEnabled = Boolean.parseBoolean(EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE);
// tableContext is used for being convenient to persist some configuration parameters uniformly
private Map<String, String> tableContext = new HashMap<>();
@@ -260,12 +260,10 @@ public class EsTable extends Table {
indexName = tableContext.get("indexName");
mappingType = tableContext.get("mappingType");
- enableDocValueScan = Boolean.parseBoolean(tableContext.get("enableDocValueScan"));
- if (tableContext.containsKey("enableKeywordSniff")) {
- enableKeywordSniff = Boolean.parseBoolean(tableContext.get("enableKeywordSniff"));
- } else {
- enableKeywordSniff = true;
- }
+ enableDocValueScan = Boolean.parseBoolean(tableContext.getOrDefault("enableDocValueScan",
+ EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE));
+ enableKeywordSniff = Boolean.parseBoolean(tableContext.getOrDefault("enableKeywordSniff",
+ EsResource.KEYWORD_SNIFF_DEFAULT_VALUE));
if (tableContext.containsKey("maxDocValueFields")) {
try {
maxDocValueFields = Integer.parseInt(tableContext.get("maxDocValueFields"));
@@ -273,16 +271,10 @@ public class EsTable extends Table {
maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS;
}
}
- if (tableContext.containsKey(EsResource.NODES_DISCOVERY)) {
- nodesDiscovery = Boolean.parseBoolean(tableContext.get(EsResource.NODES_DISCOVERY));
- } else {
- nodesDiscovery = true;
- }
- if (tableContext.containsKey(EsResource.HTTP_SSL_ENABLED)) {
- httpSslEnabled = Boolean.parseBoolean(tableContext.get(EsResource.HTTP_SSL_ENABLED));
- } else {
- httpSslEnabled = false;
- }
+ nodesDiscovery = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.NODES_DISCOVERY,
+ EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
+ httpSslEnabled = Boolean.parseBoolean(tableContext.getOrDefault(EsResource.HTTP_SSL_ENABLED,
+ EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
PartitionType partType = PartitionType.valueOf(Text.readString(in));
if (partType == PartitionType.UNPARTITIONED) {
partitionInfo = SinglePartitionInfo.read(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
index 455ea0cb5c..f3441e0c9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
@@ -17,7 +17,6 @@
package org.apache.doris.datasource;
-
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EsResource;
@@ -41,10 +40,9 @@ import java.util.Map;
*/
@Getter
public class EsExternalCatalog extends ExternalCatalog {
- private static final Logger LOG = LogManager.getLogger(EsExternalCatalog.class);
-
public static final String DEFAULT_DB = "default_db";
+ private static final Logger LOG = LogManager.getLogger(EsExternalCatalog.class);
private EsRestClient esRestClient;
/**
@@ -66,27 +64,15 @@ public class EsExternalCatalog extends ExternalCatalog {
if (properties.containsKey("ssl")) {
properties.put(EsResource.HTTP_SSL_ENABLED, properties.remove("ssl"));
}
- if (!properties.containsKey(EsResource.HTTP_SSL_ENABLED)) {
- properties.put(EsResource.HTTP_SSL_ENABLED, String.valueOf(false));
- }
if (properties.containsKey("username")) {
properties.put(EsResource.USER, properties.remove("username"));
}
if (properties.containsKey("doc_value_scan")) {
properties.put(EsResource.DOC_VALUE_SCAN, properties.remove("doc_value_scan"));
}
- if (!properties.containsKey(EsResource.DOC_VALUE_SCAN)) {
- properties.put(EsResource.DOC_VALUE_SCAN, "true");
- }
if (properties.containsKey("keyword_sniff")) {
properties.put(EsResource.KEYWORD_SNIFF, properties.remove("keyword_sniff"));
}
- if (!properties.containsKey(EsResource.KEYWORD_SNIFF)) {
- properties.put(EsResource.KEYWORD_SNIFF, "true");
- }
- if (!properties.containsKey(EsResource.NODES_DISCOVERY)) {
- properties.put(EsResource.NODES_DISCOVERY, "true");
- }
return properties;
}
@@ -104,19 +90,23 @@ public class EsExternalCatalog extends ExternalCatalog {
}
public boolean enableDocValueScan() {
- return Boolean.valueOf(catalogProperty.getOrDefault(EsResource.DOC_VALUE_SCAN, "true"));
+ return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.DOC_VALUE_SCAN,
+ EsResource.DOC_VALUE_SCAN_DEFAULT_VALUE));
}
public boolean enableKeywordSniff() {
- return Boolean.valueOf(catalogProperty.getOrDefault(EsResource.KEYWORD_SNIFF, "true"));
+ return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.KEYWORD_SNIFF,
+ EsResource.KEYWORD_SNIFF_DEFAULT_VALUE));
}
public boolean enableSsl() {
- return Boolean.valueOf(catalogProperty.getOrDefault(EsResource.HTTP_SSL_ENABLED, "false"));
+ return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.HTTP_SSL_ENABLED,
+ EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE));
}
public boolean enableNodesDiscovery() {
- return Boolean.valueOf(catalogProperty.getOrDefault(EsResource.NODES_DISCOVERY, "true"));
+ return Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.NODES_DISCOVERY,
+ EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
index 493fa413a3..42500d7f96 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
@@ -58,15 +58,17 @@ public class MappingPhase implements SearchPhase {
JSONObject properties = EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping, searchContext.type());
for (Column col : searchContext.columns()) {
String colName = col.getName();
- // if column exists in Doris Table but no found in ES's mapping, we choose to ignore this situation?
- if (!properties.containsKey(colName)) {
- throw new DorisEsException(
- "index[" + searchContext.sourceIndex() + "] type[" + indexMapping + "] mapping not found column"
- + colName + " for the ES Cluster");
+ // _id not exist mapping, but be can query it.
+ if (!"_id".equals(colName)) {
+ if (!properties.containsKey(colName)) {
+ throw new DorisEsException(
+ "index[" + searchContext.sourceIndex() + "] mapping[" + indexMapping + "] not found "
+ + "column " + colName + " for the ES Cluster");
+ }
+ JSONObject fieldObject = (JSONObject) properties.get(colName);
+ resolveKeywordFields(searchContext, fieldObject, colName);
+ resolveDocValuesFields(searchContext, fieldObject, colName);
}
- JSONObject fieldObject = (JSONObject) properties.get(colName);
- resolveKeywordFields(searchContext, fieldObject, colName);
- resolveDocValuesFields(searchContext, fieldObject, colName);
}
}
@@ -80,9 +82,9 @@ public class MappingPhase implements SearchPhase {
JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
if (fieldsObject != null) {
for (Object key : fieldsObject.keySet()) {
- JSONObject innerTypeObject = (JSONObject) fieldsObject.get((String) key);
+ JSONObject innerTypeObject = (JSONObject) fieldsObject.get(key);
// just for text type
- if ("keyword".equals((String) innerTypeObject.get("type"))) {
+ if ("keyword".equals(innerTypeObject.get("type"))) {
searchContext.fetchFieldsContext().put(colName, colName + "." + key);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org