You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/04/26 15:52:07 UTC
[incubator-doris] 04/15: [fix](cache) Generate md5 value using utf8 encoding for sqlkey string (#9121)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit f0cac16b31b5a76bf96ebc1335675bafada0c2b2
Author: ZenoYang <co...@qq.com>
AuthorDate: Sat Apr 23 21:37:34 2022 +0800
[fix](cache) Generate md5 value using utf8 encoding for sqlkey string (#9121)
---
.../java/org/apache/doris/qe/cache/CacheProxy.java | 5 ++++-
.../java/org/apache/doris/qe/cache/SqlCache.java | 6 ++++++
.../org/apache/doris/qe/PartitionCacheTest.java | 25 ++++++++++++++++++++--
3 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java
index f9664d2664..54438a9db7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheProxy.java
@@ -25,6 +25,7 @@ import org.apache.doris.proto.Types;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
/**
@@ -67,13 +68,15 @@ public abstract class CacheProxy {
public static Types.PUniqueId getMd5(String str) {
MessageDigest msgDigest;
+ final byte[] digest;
try {
//128 bit
msgDigest = MessageDigest.getInstance("MD5");
+ digest = msgDigest.digest(str.getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
return null;
}
- final byte[] digest = msgDigest.digest(str.getBytes());
+
Types.PUniqueId key = Types.PUniqueId.newBuilder()
.setLo(getLongFromByte(digest, 0))
.setHi(getLongFromByte(digest, 8))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
index a9ba2912a0..64b4d95f6f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.Status;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.proto.InternalService;
+import org.apache.doris.proto.Types;
import org.apache.doris.qe.RowBatch;
import org.apache.doris.thrift.TUniqueId;
@@ -44,6 +45,11 @@ public class SqlCache extends Cache {
return selectStmt.toSql() + "|" + allViewExpandStmtListStr;
}
+ // only used for UT
+ public Types.PUniqueId getSqlKey() {
+ return CacheProxy.getMd5(getSqlWithViewStmt());
+ }
+
public InternalService.PFetchCacheResult getCacheData(Status status) {
InternalService.PFetchCacheRequest request = InternalService.PFetchCacheRequest.newBuilder()
.setSqlKey(CacheProxy.getMd5(getSqlWithViewStmt()))
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
index 342c876a2f..414dc28bb1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
@@ -60,6 +60,7 @@ import org.apache.doris.planner.ScanNode;
import org.apache.doris.proto.Types;
import org.apache.doris.qe.cache.Cache;
import org.apache.doris.qe.cache.CacheAnalyzer;
+import org.apache.doris.qe.cache.CacheProxy;
import org.apache.doris.qe.cache.CacheAnalyzer.CacheMode;
import org.apache.doris.qe.cache.CacheCoordinator;
import org.apache.doris.qe.cache.PartitionCache;
@@ -393,14 +394,15 @@ public class PartitionCacheTest {
}
/**
- * table appevent(date(pk), userid, eventid, eventtime), stream load every 5 miniutes
+ * table appevent(date(pk), userid, eventid, eventtime, city), stream load every 5 miniutes
*/
private OlapTable createEventTable() {
Column column1 = new Column("eventdate", ScalarType.DATE);
Column column2 = new Column("userid", ScalarType.INT);
Column column3 = new Column("eventid", ScalarType.INT);
Column column4 = new Column("eventtime", ScalarType.DATETIME);
- List<Column> columns = Lists.newArrayList(column1, column2, column3, column4);
+ Column column5 = new Column("city", ScalarType.VARCHAR);
+ List<Column> columns = Lists.newArrayList(column1, column2, column3, column4, column5);
PartitionInfo partInfo = new RangePartitionInfo(Lists.newArrayList(column1));
MaterializedIndex baseIndex = new MaterializedIndex(30001, IndexState.NORMAL);
RandomDistributionInfo distInfo = new RandomDistributionInfo(10);
@@ -1022,6 +1024,25 @@ public class PartitionCacheTest {
">= '2020-01-12 00:00:00' AND `eventdate` <= '2020-01-14 00:00:00' GROUP BY `eventdate`|");
}
+ @Test
+ public void testSqlCacheKeyWithChineseChar() {
+ Catalog.getCurrentSystemInfo();
+ StatementBase parseStmt = parseSql(
+ "SELECT eventdate, COUNT(userid) FROM appevent WHERE eventdate>=\"2020-01-12\" and " +
+ "eventdate<=\"2020-01-14\" and city=\"北京\" GROUP BY eventdate"
+ );
+ ArrayList<Long> selectedPartitionIds
+ = Lists.newArrayList(20200112L, 20200113L, 20200114L);
+ List<ScanNode> scanNodes = Lists.newArrayList(createEventScanNode(selectedPartitionIds));
+ CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
+ ca.checkCacheMode(1579053661000L); //2020-1-15 10:01:01
+ Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
+ SqlCache sqlCache = (SqlCache) ca.getCache();
+ String cacheKey = sqlCache.getSqlWithViewStmt();
+ Types.PUniqueId sqlKey2 = CacheProxy.getMd5(cacheKey.replace("北京", "上海"));
+ Assert.assertNotEquals(sqlCache.getSqlKey(), sqlKey2);
+ }
+
@Test
public void testSqlCacheKeyWithView() {
Catalog.getCurrentSystemInfo();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org