You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/14 21:39:19 UTC
(hudi) 12/13: [HUDI-7516] Put jdbc-h2 creds into static variables for hudi-utilities tests (#10889)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 219f1811e565568175eb261e6888d6f7169916ce
Author: wombatu-kun <wo...@gmail.com>
AuthorDate: Wed Mar 20 14:29:54 2024 +0700
[HUDI-7516] Put jdbc-h2 creds into static variables for hudi-utilities tests (#10889)
Co-authored-by: Vova Kolmakov <vl...@nsk.beeline.ru>
---
.../utilities/deltastreamer/TestHoodieDeltaStreamer.java | 14 +++++++++-----
.../functional/TestJdbcbasedSchemaProvider.java | 14 +++++++++-----
.../apache/hudi/utilities/sources/TestJdbcSource.java | 16 ++++++++++------
.../apache/hudi/utilities/testutils/JdbcTestUtils.java | 5 +++++
4 files changed, 33 insertions(+), 16 deletions(-)
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index 3628f2477b4..7604bce856b 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -167,6 +167,10 @@ import static org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamerTestBas
import static org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor.KAFKA_SOURCE_OFFSET_COLUMN;
import static org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor.KAFKA_SOURCE_PARTITION_COLUMN;
import static org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor.KAFKA_SOURCE_TIMESTAMP_COLUMN;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_DRIVER;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_PASS;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_URL;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_USER;
import static org.apache.hudi.utilities.testutils.UtilitiesTestBase.Helpers.jsonifyRecordsByPartitions;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -2428,12 +2432,12 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
@Test
public void testJdbcSourceIncrementalFetchInContinuousMode() {
- try (Connection connection = DriverManager.getConnection("jdbc:h2:mem:test_mem", "sa", "")) {
+ try (Connection connection = DriverManager.getConnection(JDBC_URL, JDBC_USER, JDBC_PASS)) {
TypedProperties props = new TypedProperties();
- props.setProperty("hoodie.streamer.jdbc.url", "jdbc:h2:mem:test_mem");
- props.setProperty("hoodie.streamer.jdbc.driver.class", "org.h2.Driver");
- props.setProperty("hoodie.streamer.jdbc.user", "sa");
- props.setProperty("hoodie.streamer.jdbc.password", "");
+ props.setProperty("hoodie.streamer.jdbc.url", JDBC_URL);
+ props.setProperty("hoodie.streamer.jdbc.driver.class", JDBC_DRIVER);
+ props.setProperty("hoodie.streamer.jdbc.user", JDBC_USER);
+ props.setProperty("hoodie.streamer.jdbc.password", JDBC_PASS);
props.setProperty("hoodie.streamer.jdbc.table.name", "triprec");
props.setProperty("hoodie.streamer.jdbc.incr.pull", "true");
props.setProperty("hoodie.streamer.jdbc.table.incr.column.name", "id");
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestJdbcbasedSchemaProvider.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestJdbcbasedSchemaProvider.java
index 46400dda48d..05a623f0e09 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestJdbcbasedSchemaProvider.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestJdbcbasedSchemaProvider.java
@@ -37,6 +37,10 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_DRIVER;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_PASS;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_URL;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_USER;
import static org.junit.jupiter.api.Assertions.assertEquals;
@Tag("functional")
@@ -47,10 +51,10 @@ public class TestJdbcbasedSchemaProvider extends SparkClientFunctionalTestHarnes
@BeforeAll
public static void init() {
- PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.connection.url", "jdbc:h2:mem:test_mem");
- PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.driver.type", "org.h2.Driver");
- PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.username", "sa");
- PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.password", "");
+ PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.connection.url", JDBC_URL);
+ PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.driver.type", JDBC_DRIVER);
+ PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.username", JDBC_USER);
+ PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.password", JDBC_PASS);
PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.dbtable", "triprec");
PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.timeout", "0");
PROPS.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.jdbc.nullable", "false");
@@ -73,7 +77,7 @@ public class TestJdbcbasedSchemaProvider extends SparkClientFunctionalTestHarnes
* @throws SQLException
*/
private void initH2Database() throws SQLException {
- try (Connection conn = DriverManager.getConnection("jdbc:h2:mem:test_mem", "sa", "")) {
+ try (Connection conn = DriverManager.getConnection(JDBC_URL, JDBC_USER, JDBC_PASS)) {
PreparedStatement ps = conn.prepareStatement(UtilitiesTestBase.Helpers.readFile("streamer-config/triprec.sql"));
ps.executeUpdate();
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJdbcSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJdbcSource.java
index 4c8b264fe16..dcd12ac7c8e 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJdbcSource.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJdbcSource.java
@@ -46,6 +46,10 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.stream.Collectors;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_DRIVER;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_PASS;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_URL;
+import static org.apache.hudi.utilities.testutils.JdbcTestUtils.JDBC_USER;
import static org.apache.hudi.utilities.testutils.JdbcTestUtils.clearAndInsert;
import static org.apache.hudi.utilities.testutils.JdbcTestUtils.close;
import static org.apache.hudi.utilities.testutils.JdbcTestUtils.count;
@@ -73,12 +77,12 @@ public class TestJdbcSource extends UtilitiesTestBase {
@BeforeEach
public void setup() throws Exception {
super.setup();
- PROPS.setProperty("hoodie.deltastreamer.jdbc.url", "jdbc:h2:mem:test_mem");
- PROPS.setProperty("hoodie.deltastreamer.jdbc.driver.class", "org.h2.Driver");
- PROPS.setProperty("hoodie.deltastreamer.jdbc.user", "test");
- PROPS.setProperty("hoodie.deltastreamer.jdbc.password", "jdbc");
+ PROPS.setProperty("hoodie.deltastreamer.jdbc.url", JDBC_URL);
+ PROPS.setProperty("hoodie.deltastreamer.jdbc.driver.class", JDBC_DRIVER);
+ PROPS.setProperty("hoodie.deltastreamer.jdbc.user", JDBC_USER);
+ PROPS.setProperty("hoodie.deltastreamer.jdbc.password", JDBC_PASS);
PROPS.setProperty("hoodie.deltastreamer.jdbc.table.name", "triprec");
- connection = DriverManager.getConnection("jdbc:h2:mem:test_mem", "test", "jdbc");
+ connection = DriverManager.getConnection(JDBC_URL, JDBC_USER, JDBC_PASS);
}
@AfterEach
@@ -438,7 +442,7 @@ public class TestJdbcSource extends UtilitiesTestBase {
private void writeSecretToFs() throws IOException {
FileSystem fs = FileSystem.get(new Configuration());
FSDataOutputStream outputStream = fs.create(new Path("file:///tmp/hudi/config/secret"));
- outputStream.writeBytes("jdbc");
+ outputStream.writeBytes(JDBC_PASS);
outputStream.close();
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/JdbcTestUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/JdbcTestUtils.java
index 79047794f97..227013b0548 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/JdbcTestUtils.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/JdbcTestUtils.java
@@ -44,6 +44,11 @@ public class JdbcTestUtils {
private static final Logger LOG = LoggerFactory.getLogger(JdbcTestUtils.class);
+ public static final String JDBC_URL = "jdbc:h2:mem:test_mem";
+ public static final String JDBC_DRIVER = "org.h2.Driver";
+ public static final String JDBC_USER = "test";
+ public static final String JDBC_PASS = "jdbc";
+
public static List<HoodieRecord> clearAndInsert(String commitTime, int numRecords, Connection connection, HoodieTestDataGenerator dataGenerator, TypedProperties props)
throws SQLException {
execute(connection, "DROP TABLE triprec", "Table does not exists");