You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2022/01/04 21:42:50 UTC
[hudi] branch master updated: [HUDI-3147] Add endpoint_url to dynamodb lock provider (#4500)
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 37b15ff [HUDI-3147] Add endpoint_url to dynamodb lock provider (#4500)
37b15ff is described below
commit 37b15ff458d8d332cbc884de07a43d78a8ebe5ce
Author: Nicolas Paris <ni...@riseup.net>
AuthorDate: Tue Jan 4 22:42:28 2022 +0100
[HUDI-3147] Add endpoint_url to dynamodb lock provider (#4500)
Co-authored-by: Nicolas Paris <ni...@adevinta.com>
---
.../transaction/lock/DynamoDBBasedLockProvider.java | 4 +++-
.../apache/hudi/config/DynamoDbBasedLockConfig.java | 21 ++++++++++++++-------
2 files changed, 17 insertions(+), 8 deletions(-)
diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java
index 140e8ea..57348be 100644
--- a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java
+++ b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java
@@ -156,7 +156,9 @@ public class DynamoDBBasedLockProvider implements LockProvider<LockItem> {
private AmazonDynamoDB getDynamoDBClient() {
String region = this.lockConfiguration.getConfig().getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_REGION.key());
- String endpointURL = RegionUtils.getRegion(region).getServiceEndpoint(AmazonDynamoDB.ENDPOINT_PREFIX);
+ String endpointURL = this.lockConfiguration.getConfig().getString(DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL.key()) == null
+ ? RegionUtils.getRegion(region).getServiceEndpoint(AmazonDynamoDB.ENDPOINT_PREFIX)
+ : this.lockConfiguration.getConfig().getString(DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL.key());
AwsClientBuilder.EndpointConfiguration dynamodbEndpoint =
new AwsClientBuilder.EndpointConfiguration(endpointURL, region);
return AmazonDynamoDBClientBuilder.standard()
diff --git a/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java b/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java
index 3731533..19c63ea 100644
--- a/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java
+++ b/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java
@@ -34,10 +34,10 @@ import static org.apache.hudi.common.config.LockConfiguration.LOCK_PREFIX;
* Hoodie Configs for Locks.
*/
@ConfigClassProperty(name = "DynamoDB based Locks Configurations",
- groupName = ConfigGroups.Names.WRITE_CLIENT,
- description = "Configs that control DynamoDB based locking mechanisms required for concurrency control "
- + " between writers to a Hudi table. Concurrency between Hudi's own table services "
- + " are auto managed internally.")
+ groupName = ConfigGroups.Names.WRITE_CLIENT,
+ description = "Configs that control DynamoDB based locking mechanisms required for concurrency control "
+ + " between writers to a Hudi table. Concurrency between Hudi's own table services "
+ + " are auto managed internally.")
public class DynamoDbBasedLockConfig extends HoodieConfig {
// configs for DynamoDb based locks
@@ -60,8 +60,8 @@ public class DynamoDbBasedLockConfig extends HoodieConfig {
return Option.empty();
})
.withDocumentation("For DynamoDB based lock provider, the partition key for the DynamoDB lock table. "
- + "Each Hudi dataset should has it's unique key so concurrent writers could refer to the same partition key."
- + " By default we use the Hudi table name specified to be the partition key");
+ + "Each Hudi dataset should has it's unique key so concurrent writers could refer to the same partition key."
+ + " By default we use the Hudi table name specified to be the partition key");
public static final ConfigProperty<String> DYNAMODB_LOCK_REGION = ConfigProperty
.key(DYNAMODB_BASED_LOCK_PROPERTY_PREFIX + "region")
@@ -75,7 +75,7 @@ public class DynamoDbBasedLockConfig extends HoodieConfig {
return Option.empty();
})
.withDocumentation("For DynamoDB based lock provider, the region used in endpoint for Amazon DynamoDB service."
- + " Would try to first get it from AWS_REGION environment variable. If not find, by default use us-east-1");
+ + " Would try to first get it from AWS_REGION environment variable. If not find, by default use us-east-1");
public static final ConfigProperty<String> DYNAMODB_LOCK_BILLING_MODE = ConfigProperty
.key(DYNAMODB_BASED_LOCK_PROPERTY_PREFIX + "billing_mode")
@@ -100,4 +100,11 @@ public class DynamoDbBasedLockConfig extends HoodieConfig {
.defaultValue(String.valueOf(10 * 60 * 1000))
.sinceVersion("0.10.0")
.withDocumentation("For DynamoDB based lock provider, the maximum number of milliseconds to wait for creating DynamoDB table");
+
+ public static final ConfigProperty<String> DYNAMODB_ENDPOINT_URL = ConfigProperty
+ .key(DYNAMODB_BASED_LOCK_PROPERTY_PREFIX + "endpoint_url")
+ .defaultValue("us-east-1")
+ .sinceVersion("0.11.0")
+ .withDocumentation("For DynamoDB based lock provider, the url endpoint used for Amazon DynamoDB service."
+ + " Useful for development with a local dynamodb instance.");
}