(doris-streamloader) branch master updated: disable concurrency when the format is not CSV (#15)

[li...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

liaoxin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-streamloader.git


The following commit(s) were added to refs/heads/master by this push:
     new 5cd581b  disable concurrency when the format is not CSV (#15)
5cd581b is described below

commit 5cd581b1409303c6a35989dd6a5a44a9b11412d8
Author: hui lai <13...@qq.com>
AuthorDate: Tue Jun 11 15:03:14 2024 +0800

    disable concurrency when the format is not CSV (#15)
---
 main.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/main.go b/main.go
index 7a7737c..1677b4f 100644
--- a/main.go
+++ b/main.go
@@ -70,6 +70,7 @@ var (
 	maxBytesPerTask      int
 	debug                bool
 	workers              int
+	enableConcurrency    bool
 	diskThroughput       int
 	streamLoadThroughput int
 	checkUTF8            bool
@@ -245,6 +246,9 @@ func paramCheck() {
 				continue
 			}
 			kv := strings.Split(v, ":")
+			if strings.ToLower(kv[0]) == "format" && strings.ToLower(kv[1]) == "csv" {
+				enableConcurrency = true
+			}
 			if len(kv) > 2 {
 				headers[kv[0]] = strings.Join(kv[1:], ":")
 			} else {
@@ -294,6 +298,11 @@ func calculateAndCheckWorkers(reader *file.FileReader, size int64) {
 		return
 	}
 
+	if !enableConcurrency {
+		loadInfo.Workers = 1
+		return
+	}
+
 	ratio := float64(size) / float64(maxBytesPerTask)
 	tmpWorkers := 0
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org