You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/10/26 10:02:21 UTC
[doris] 01/01: [improvement](regression-test) avoid query empty result after loading finished
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-unstable
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e23c7eb85e8387f129a40b7bba43085e99ebb28a
Author: morningman <mo...@163.com>
AuthorDate: Wed Oct 26 15:17:34 2022 +0800
[improvement](regression-test) avoid query empty result after loading finished
---
.../doris/transaction/DatabaseTransactionMgr.java | 5 +++++
.../org/apache/doris/transaction/TransactionState.java | 17 +++++++++++------
.../ssb_unique_load_zstd_p0/load_one_step/load.groovy | 2 ++
3 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 843df334ee..321a1c3d72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -944,6 +944,11 @@ public class DatabaseTransactionMgr {
} finally {
MetaLockUtils.writeUnlockTables(tableList);
}
+ // The visible latch should only be counted down after all things are done
+ // (finish transaction, write edit log, etc).
+ // Otherwise, there is no way for stream load to query the result right after loading finished,
+ // even if we call "sync" before querying.
+ transactionState.countdownVisibleLatch();
LOG.info("finish transaction {} successfully", transactionState);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java
index 3a59fac9d9..c6a22c2a24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java
@@ -188,9 +188,10 @@ public class TransactionState implements Writable {
private String reason = "";
// error replica ids
private Set<Long> errorReplicas;
- private CountDownLatch latch;
+ // this latch will be counted down when txn status change to VISIBLE
+ private CountDownLatch visibleLatch;
- // this state need not to be serialized
+ // this state need not be serialized
private Map<Long, PublishVersionTask> publishVersionTasks;
private boolean hasSendTask;
private long publishVersionTime = -1;
@@ -246,7 +247,7 @@ public class TransactionState implements Writable {
this.errorReplicas = Sets.newHashSet();
this.publishVersionTasks = Maps.newHashMap();
this.hasSendTask = false;
- this.latch = new CountDownLatch(1);
+ this.visibleLatch = new CountDownLatch(1);
this.authCode = UUID.randomUUID().toString();
}
@@ -269,7 +270,7 @@ public class TransactionState implements Writable {
this.errorReplicas = Sets.newHashSet();
this.publishVersionTasks = Maps.newHashMap();
this.hasSendTask = false;
- this.latch = new CountDownLatch(1);
+ this.visibleLatch = new CountDownLatch(1);
this.callbackId = callbackId;
this.timeoutMs = timeoutMs;
this.authCode = UUID.randomUUID().toString();
@@ -379,7 +380,7 @@ public class TransactionState implements Writable {
// after status changed
if (transactionStatus == TransactionStatus.VISIBLE) {
- this.latch.countDown();
+ this.visibleLatch.countDown();
if (MetricRepo.isInit) {
MetricRepo.COUNTER_TXN_SUCCESS.increase(1L);
}
@@ -457,8 +458,12 @@ public class TransactionState implements Writable {
}
}
+ public void countdownVisibleLatch() {
+ this.visibleLatch.countDown();
+ }
+
public void waitTransactionVisible(long timeoutMillis) throws InterruptedException {
- this.latch.await(timeoutMillis, TimeUnit.MILLISECONDS);
+ this.visibleLatch.await(timeoutMillis, TimeUnit.MILLISECONDS);
}
public void setPrepareTime(long prepareTime) {
diff --git a/regression-test/suites/ssb_unique_load_zstd_p0/load_one_step/load.groovy b/regression-test/suites/ssb_unique_load_zstd_p0/load_one_step/load.groovy
index dfe107a0bf..621315c9b6 100644
--- a/regression-test/suites/ssb_unique_load_zstd_p0/load_one_step/load.groovy
+++ b/regression-test/suites/ssb_unique_load_zstd_p0/load_one_step/load.groovy
@@ -28,6 +28,8 @@ suite("load_one_step") {
sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_create.sql""").text
streamLoad {
table tableName
+ def label = "load_one_step_${tableName}_" + UUID.randomUUID().toString()
+ set 'label', label
set 'column_separator', '|'
set 'compress_type', 'GZ'
set 'columns', rows[0]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org