You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by fa...@apache.org on 2022/07/26 04:02:07 UTC
[incubator-seatunnel] branch dev updated: [Feature][connector] clickhousefile sink connector support non-root username for fileTransfer (#2263)
This is an automated email from the ASF dual-hosted git repository.
fanjia pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 704661f1f [Feature][connector] clickhousefile sink connector support non-root username for fileTransfer (#2263)
704661f1f is described below
commit 704661f1fdea391276726040e7b14c0604214ef7
Author: stormrise <li...@gmail.com>
AuthorDate: Tue Jul 26 12:02:02 2022 +0800
[Feature][connector] clickhousefile sink connector support non-root username for fileTransfer (#2263)
* clickhousefile fileTransfer support other username
* ClickhouseFile support non-root user
* fix bug
* fix code style
* fix code style
Co-authored-by: Lingxiao Li (李凌霄) <li...@trip.com>
---
docs/en/connector-v2/sink/ClickhouseFile.md | 9 +++-
.../clickhouse/config/FileReaderOption.java | 11 ++++
.../clickhouse/sink/file/ClickhouseFileSink.java | 5 +-
.../sink/file/ClickhouseFileSinkWriter.java | 3 +-
.../clickhouse/sink/file/FileTransferFactory.java | 6 +--
.../clickhouse/sink/file/RsyncFileTransfer.java | 8 +--
.../clickhouse/sink/file/ScpFileTransfer.java | 8 +--
.../spark/clickhouse/sink/ClickhouseFile.scala | 10 +++-
.../sink/filetransfer/RsyncFileTransfer.scala | 8 +--
.../sink/filetransfer/ScpFileTransfer.scala | 6 ++-
.../src/test/resources/sea.conf | 61 ++++++++++++++++++++++
.../spark/clickhouse/sink/ClickhouseFileTest.scala | 28 ++++++++--
12 files changed, 140 insertions(+), 23 deletions(-)
diff --git a/docs/en/connector-v2/sink/ClickhouseFile.md b/docs/en/connector-v2/sink/ClickhouseFile.md
index 3e8a8d567..12bdf69a7 100644
--- a/docs/en/connector-v2/sink/ClickhouseFile.md
+++ b/docs/en/connector-v2/sink/ClickhouseFile.md
@@ -27,6 +27,7 @@ Write data to Clickhouse can also be done using JDBC
| node_free_password | boolean | no | false |
| node_pass | list | no | - |
| node_pass.node_address | string | no | - |
+| node_pass.username | string | no | "root" |
| node_pass.password | string | no | - |
| common-options | string | no | - |
@@ -78,9 +79,13 @@ Used to save the addresses and corresponding passwords of all clickhouse servers
The address corresponding to the clickhouse server
-### node_pass.node_password [string]
+### node_pass.username [string]
-The password corresponding to the clickhouse server, only support root user yet.
+The username corresponding to the clickhouse server, default root user.
+
+### node_pass.password [string]
+
+The password corresponding to the clickhouse server.
### common options [string]
diff --git a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/config/FileReaderOption.java b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/config/FileReaderOption.java
index 720810789..283c1c4f9 100644
--- a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/config/FileReaderOption.java
+++ b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/config/FileReaderOption.java
@@ -32,18 +32,21 @@ public class FileReaderOption implements Serializable {
private String clickhouseLocalPath;
private ClickhouseFileCopyMethod copyMethod;
private boolean nodeFreePass;
+ private Map<String, String> nodeUser;
private Map<String, String> nodePassword;
private SeaTunnelRowType seaTunnelRowType;
public FileReaderOption(ShardMetadata shardMetadata, Map<String, String> tableSchema,
List<String> fields, String clickhouseLocalPath,
ClickhouseFileCopyMethod copyMethod,
+ Map<String, String> nodeUser,
Map<String, String> nodePassword) {
this.shardMetadata = shardMetadata;
this.tableSchema = tableSchema;
this.fields = fields;
this.clickhouseLocalPath = clickhouseLocalPath;
this.copyMethod = copyMethod;
+ this.nodeUser = nodeUser;
this.nodePassword = nodePassword;
}
@@ -79,6 +82,14 @@ public class FileReaderOption implements Serializable {
this.copyMethod = copyMethod;
}
+ public Map<String, String> getNodeUser() {
+ return nodeUser;
+ }
+
+ public void setNodeUser(Map<String, String> nodeUser) {
+ this.nodeUser = nodeUser;
+ }
+
public Map<String, String> getNodePassword() {
return nodePassword;
}
diff --git a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ClickhouseFileSink.java b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ClickhouseFileSink.java
index 05c511292..62694c3fe 100644
--- a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ClickhouseFileSink.java
+++ b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ClickhouseFileSink.java
@@ -114,13 +114,16 @@ public class ClickhouseFileSink implements SeaTunnelSink<SeaTunnelRow, Clickhous
} else {
fields = new ArrayList<>(tableSchema.keySet());
}
+ Map<String, String> nodeUser = config.getObjectList(NODE_PASS).stream()
+ .collect(Collectors.toMap(configObject -> configObject.toConfig().getString(NODE_ADDRESS),
+ configObject -> configObject.toConfig().hasPath(USERNAME) ? configObject.toConfig().getString(USERNAME) : "root"));
Map<String, String> nodePassword = config.getObjectList(NODE_PASS).stream()
.collect(Collectors.toMap(configObject -> configObject.toConfig().getString(NODE_ADDRESS),
configObject -> configObject.toConfig().getString(PASSWORD)));
proxy.close();
this.readerOption = new FileReaderOption(shardMetadata, tableSchema, fields, config.getString(CLICKHOUSE_LOCAL_PATH),
- ClickhouseFileCopyMethod.from(config.getString(COPY_METHOD)), nodePassword);
+ ClickhouseFileCopyMethod.from(config.getString(COPY_METHOD)), nodeUser, nodePassword);
}
@Override
diff --git a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ClickhouseFileSinkWriter.java b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ClickhouseFileSinkWriter.java
index a23645d12..c19fdae34 100644
--- a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ClickhouseFileSinkWriter.java
+++ b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ClickhouseFileSinkWriter.java
@@ -207,8 +207,9 @@ public class ClickhouseFileSinkWriter implements SinkWriter<SeaTunnelRow, CKComm
private void attachClickhouseLocalFileToServer(Shard shard, List<String> clickhouseLocalFiles) throws ClickHouseException {
String hostAddress = shard.getNode().getAddress().getHostName();
+ String user = readerOption.getNodeUser().getOrDefault(hostAddress, "root");
String password = readerOption.getNodePassword().getOrDefault(hostAddress, null);
- FileTransfer fileTransfer = FileTransferFactory.createFileTransfer(this.readerOption.getCopyMethod(), hostAddress, password);
+ FileTransfer fileTransfer = FileTransferFactory.createFileTransfer(this.readerOption.getCopyMethod(), hostAddress, user, password);
fileTransfer.init();
fileTransfer.transferAndChown(clickhouseLocalFiles, shardLocalDataPaths.get(shard).get(0) + "detached/");
fileTransfer.close();
diff --git a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/FileTransferFactory.java b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/FileTransferFactory.java
index 73c996473..14455b45f 100644
--- a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/FileTransferFactory.java
+++ b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/FileTransferFactory.java
@@ -20,12 +20,12 @@ package org.apache.seatunnel.connectors.seatunnel.clickhouse.sink.file;
import org.apache.seatunnel.connectors.seatunnel.clickhouse.config.ClickhouseFileCopyMethod;
public class FileTransferFactory {
- public static FileTransfer createFileTransfer(ClickhouseFileCopyMethod type, String host, String password) {
+ public static FileTransfer createFileTransfer(ClickhouseFileCopyMethod type, String host, String user, String password) {
switch (type) {
case SCP:
- return new ScpFileTransfer(host, password);
+ return new ScpFileTransfer(host, user, password);
case RSYNC:
- return new RsyncFileTransfer(host, password);
+ return new RsyncFileTransfer(host, user, password);
default:
throw new RuntimeException("unsupported clickhouse file copy method:" + type);
}
diff --git a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/RsyncFileTransfer.java b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/RsyncFileTransfer.java
index a19f2ef5f..9d1a888c2 100644
--- a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/RsyncFileTransfer.java
+++ b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/RsyncFileTransfer.java
@@ -37,13 +37,15 @@ public class RsyncFileTransfer implements FileTransfer {
private static final int SSH_PORT = 22;
private final String host;
+ private final String user;
private final String password;
private ClientSession clientSession;
private SshClient sshClient;
- public RsyncFileTransfer(String host, String password) {
+ public RsyncFileTransfer(String host, String user, String password) {
this.host = host;
+ this.user = user;
this.password = password;
}
@@ -52,7 +54,7 @@ public class RsyncFileTransfer implements FileTransfer {
try {
sshClient = SshClient.setUpDefaultClient();
sshClient.start();
- clientSession = sshClient.connect("root", host, SSH_PORT).verify().getSession();
+ clientSession = sshClient.connect(user, host, SSH_PORT).verify().getSession();
if (password != null) {
clientSession.addPasswordIdentity(password);
}
@@ -61,7 +63,7 @@ public class RsyncFileTransfer implements FileTransfer {
throw new IOException("ssh host " + host + "authentication failed");
}
} catch (IOException e) {
- throw new RuntimeException("Failed to connect to host: " + host + " by user: root on port 22", e);
+ throw new RuntimeException("Failed to connect to host: " + host + " by user: " + user + " on port 22", e);
}
}
diff --git a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ScpFileTransfer.java b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ScpFileTransfer.java
index 6fa83794c..53347457d 100644
--- a/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ScpFileTransfer.java
+++ b/seatunnel-connectors-v2/connector-clickhouse/src/main/java/org/apache/seatunnel/connectors/seatunnel/clickhouse/sink/file/ScpFileTransfer.java
@@ -36,14 +36,16 @@ public class ScpFileTransfer implements FileTransfer {
private static final int SCP_PORT = 22;
private final String host;
+ private final String user;
private final String password;
private ScpClient scpClient;
private ClientSession clientSession;
private SshClient sshClient;
- public ScpFileTransfer(String host, String password) {
+ public ScpFileTransfer(String host, String user, String password) {
this.host = host;
+ this.user = user;
this.password = password;
}
@@ -52,7 +54,7 @@ public class ScpFileTransfer implements FileTransfer {
try {
sshClient = SshClient.setUpDefaultClient();
sshClient.start();
- clientSession = sshClient.connect("root", host, SCP_PORT).verify().getSession();
+ clientSession = sshClient.connect(user, host, SCP_PORT).verify().getSession();
if (password != null) {
clientSession.addPasswordIdentity(password);
}
@@ -62,7 +64,7 @@ public class ScpFileTransfer implements FileTransfer {
}
scpClient = ScpClientCreator.instance().createScpClient(clientSession);
} catch (IOException e) {
- throw new RuntimeException("Failed to connect to host: " + host + " by user: root on port 22", e);
+ throw new RuntimeException("Failed to connect to host: " + host + " by user: " + user + " on port 22", e);
}
}
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/ClickhouseFile.scala b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/ClickhouseFile.scala
index c7f37bffc..637733fe9 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/ClickhouseFile.scala
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/ClickhouseFile.scala
@@ -58,6 +58,7 @@ class ClickhouseFile extends SparkBatchSink {
private var clickhouseLocalPath: String = _
private var table: Table = _
private var fields: List[String] = _
+ private var nodeUser: Map[String, String] = _
private var nodePass: Map[String, String] = _
private val random = ThreadLocalRandom.current()
private var freePass: Boolean = false
@@ -169,14 +170,14 @@ class ClickhouseFile extends SparkBatchSink {
if (freePass || !nodePass.contains(shard.hostAddress)) {
fileTransfer = new ScpFileTransfer(shard.hostAddress)
} else {
- fileTransfer = new ScpFileTransfer(shard.hostAddress, nodePass(shard.hostAddress))
+ fileTransfer = new ScpFileTransfer(shard.hostAddress, nodeUser(shard.hostAddress), nodePass(shard.hostAddress))
}
}
case RSYNC => {
if (freePass || !nodePass.contains(shard.hostAddress)) {
fileTransfer = new RsyncFileTransfer(shard.hostAddress)
} else {
- fileTransfer = new RsyncFileTransfer(shard.hostAddress, nodePass(shard.hostAddress))
+ fileTransfer = new RsyncFileTransfer(shard.hostAddress, nodeUser(shard.hostAddress), nodePass(shard.hostAddress))
}
}
case _ => throw new UnsupportedOperationException(s"unknown copy file method: '$copyFileMethod', please use " +
@@ -243,12 +244,17 @@ class ClickhouseFile extends SparkBatchSink {
this.freePass = true
} else if (config.hasPath(NODE_PASS)) {
val nodePass = config.getObjectList(NODE_PASS)
+ val nodeUserMap = mutable.Map[String, String]()
val nodePassMap = mutable.Map[String, String]()
nodePass.foreach(np => {
val address = np.toConfig.getString(NODE_ADDRESS)
+ // default user "root"
+ val username = if (np.toConfig.hasPath(USERNAME)) np.toConfig.getString(USERNAME) else "root"
val password = np.toConfig.getString(PASSWORD)
+ nodeUserMap(address) = username
nodePassMap(address) = password
})
+ this.nodeUser = nodePassMap.toMap
this.nodePass = nodePassMap.toMap
checkResult = checkNodePass(this.nodePass, tableInfo.shards.values().toList)
} else {
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/RsyncFileTransfer.scala b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/RsyncFileTransfer.scala
index 1618683e8..ce2e07758 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/RsyncFileTransfer.scala
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/RsyncFileTransfer.scala
@@ -27,10 +27,12 @@ import scala.sys.process.Process
class RsyncFileTransfer(host: String) extends FileTransfer {
private val LOGGER = LoggerFactory.getLogger(classOf[RsyncFileTransfer])
+ var username: String = _
var password: String = _
- def this(host: String, password: String) {
+ def this(host: String, username: String, password: String) {
this(host)
+ this.username = username
this.password = password
}
@@ -52,7 +54,7 @@ class RsyncFileTransfer(host: String) extends FileTransfer {
exec.append("-e")
exec.append(sshParameter)
exec.append(sourcePath)
- exec.append(s"root@$host:$targetPath")
+ exec.append(s"$username@$host:$targetPath")
val command = Process(exec)
LOGGER.info(command.lineStream.mkString("\n"))
// remote exec command to change file owner. Only file owner equal with server's clickhouse user can
@@ -68,7 +70,7 @@ class RsyncFileTransfer(host: String) extends FileTransfer {
override def init(): Unit = {
client = SshClient.setUpDefaultClient()
client.start()
- session = client.connect("root", this.host, 22).verify().getSession
+ session = client.connect(this.username, this.host, 22).verify().getSession
if (password != null) {
session.addPasswordIdentity(this.password)
}
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/ScpFileTransfer.scala b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/ScpFileTransfer.scala
index adddd5628..ffece6682 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/ScpFileTransfer.scala
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/main/scala/org/apache/seatunnel/spark/clickhouse/sink/filetransfer/ScpFileTransfer.scala
@@ -23,10 +23,12 @@ import org.apache.sshd.scp.client.{ScpClient, ScpClientCreator}
class ScpFileTransfer(host: String) extends FileTransfer {
+ var username: String = _
var password: String = _
- def this(host: String, password: String) {
+ def this(host: String, username: String, password: String) {
this(host)
+ this.username = username
this.password = password
}
@@ -60,7 +62,7 @@ class ScpFileTransfer(host: String) extends FileTransfer {
override def init(): Unit = {
client = SshClient.setUpDefaultClient()
client.start()
- session = client.connect("root", this.host, 22).verify().getSession
+ session = client.connect(username, this.host, 22).verify().getSession
if (password != null) {
session.addPasswordIdentity(this.password)
}
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/test/resources/sea.conf b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/test/resources/sea.conf
new file mode 100644
index 000000000..b11c75d4e
--- /dev/null
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/test/resources/sea.conf
@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ execution.parallelism = 1
+ spark.sql.catalogImplementation = "hive"
+}
+
+source {
+ hive {
+ pre_sql = "select * from mydb.mytb"
+ }
+}
+
+transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+}
+
+sink {
+ ClickhouseFile {
+ host = "10.60.94.1:8123,10.60.94.2:8123,10.60.94.3:8123"
+ database = "mydb"
+ table = "access_msg"
+ fields = ["date", "datetime", "hostname", "http_code", "data_size", "ua", "request_time"]
+ username = "username"
+ password = "password"
+ sharding_key = "age"
+ clickhouse_local_path = "/usr/bin/clickhouse-local"
+ node_pass = [
+ {
+ node_address = "10.60.94.1"
+ username = "clickhouse"
+ password = "password"
+ }
+ {
+ node_address = "10.60.94.2"
+ password = "password"
+ }
+ {
+ node_address = "10.60.94.3"
+ password = "password"
+ }
+ ]
+}
+}
\ No newline at end of file
diff --git a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/test/scala/org/apache/seatunnel/spark/clickhouse/sink/ClickhouseFileTest.scala b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/test/scala/org/apache/seatunnel/spark/clickhouse/sink/ClickhouseFileTest.scala
index 8037dc574..98147766f 100644
--- a/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/test/scala/org/apache/seatunnel/spark/clickhouse/sink/ClickhouseFileTest.scala
+++ b/seatunnel-connectors/seatunnel-connectors-spark/seatunnel-connector-spark-clickhouse/src/test/scala/org/apache/seatunnel/spark/clickhouse/sink/ClickhouseFileTest.scala
@@ -17,11 +17,13 @@
package org.apache.seatunnel.spark.clickhouse.sink
-import org.apache.seatunnel.shade.com.typesafe.config.Config
-import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory
-
+import org.apache.seatunnel.shade.com.typesafe.config.{ConfigFactory, ConfigResolveOptions}
+import org.apache.seatunnel.spark.clickhouse.Config.{NODE_ADDRESS, NODE_PASS, PASSWORD, USERNAME}
import org.scalatest.funsuite.AnyFunSuite
+import scala.collection.JavaConversions.collectionAsScalaIterable
+import scala.collection.mutable
+
class ClickhouseFileTest extends AnyFunSuite {
test("test for localizationEngine") {
@@ -39,6 +41,26 @@ class ClickhouseFileTest extends AnyFunSuite {
engine = originalEngine, createTableDDL = replicatedMergeTreeDDL,
engineFull = "replicatedMergeTree", dataPaths = List[String]())
assert(localizationDDL.equals(table.localizationEngine(originalEngine, replicatedMergeTreeDDL)))
+
+ val url = Thread.currentThread().getContextClassLoader.getResource("sea.conf")
+ val config = ConfigFactory.parseURL(url)
+ .resolve(ConfigResolveOptions.defaults.setAllowUnresolved(true))
+ .resolveWith(ConfigFactory.systemProperties(),
+ ConfigResolveOptions.defaults().setAllowUnresolved(true));
+ val ckf = config.getObjectList("sink").get(0).toConfig
+ val nodePass = ckf.getObjectList(NODE_PASS)
+ val nodeUserMap = mutable.Map[String, String]()
+ val nodePassMap = mutable.Map[String, String]()
+ nodePass.foreach(np => {
+ val address = np.toConfig.getString(NODE_ADDRESS)
+ // default user "root"
+ val username = if (np.toConfig.hasPath(USERNAME)) np.toConfig.getString(USERNAME) else "root"
+ val password = np.toConfig.getString(PASSWORD)
+ nodeUserMap(address) = username
+ nodePassMap(address) = password
+ })
+ println(nodeUserMap.toMap.values)
+ assert(nodeUserMap.toMap.values.count(s => s.equals("root")).equals(2))
}
}