You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2018/08/01 03:30:51 UTC
incubator-gobblin git commit: [GOBBLIN-549] Add configurability to
Couchbase Writer (failure percentage, retries)
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 807eb46e1 -> 1fb2bb732
[GOBBLIN-549] Add configurability to Couchbase Writer (failure percentage, retries)
Closes #2410 from shirshanka/couchbase-config
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/1fb2bb73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/1fb2bb73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/1fb2bb73
Branch: refs/heads/master
Commit: 1fb2bb732a36994174aaab29455755c4ded15357
Parents: 807eb46
Author: Shirshanka Das <sd...@linkedin.com>
Authored: Tue Jul 31 20:30:43 2018 -0700
Committer: Abhishek Tiwari <ab...@gmail.com>
Committed: Tue Jul 31 20:30:43 2018 -0700
----------------------------------------------------------------------
.../couchbase/writer/CouchbaseWriterBuilder.java | 15 +++++++++++++--
.../writer/CouchbaseWriterConfigurationKeys.java | 11 ++++++++++-
2 files changed, 23 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/1fb2bb73/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java
index 58f1a7a..7e8625b 100644
--- a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java
+++ b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterBuilder.java
@@ -42,12 +42,23 @@ public class CouchbaseWriterBuilder extends DataWriterBuilder {
Config config = ConfigUtils.propertiesToConfig(taskProps);
CouchbaseEnvironment couchbaseEnvironment = CouchbaseEnvironmentFactory.getInstance(config);
//TODO: Read config to decide whether to build a blocking writer or an async writer
+
+ double failureAllowance = ConfigUtils.getDouble(config, CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_CONFIG,
+ CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0;
+
+ boolean retriesEnabled = ConfigUtils.getBoolean(config, CouchbaseWriterConfigurationKeys.RETRIES_ENABLED,
+ CouchbaseWriterConfigurationKeys.RETRIES_ENABLED_DEFAULT);
+
+ int maxRetries = ConfigUtils.getInt(config, CouchbaseWriterConfigurationKeys.MAX_RETRIES,
+ CouchbaseWriterConfigurationKeys.MAX_RETRIES_DEFAULT);
+
// build an async couchbase writer
AsyncDataWriter couchbaseWriter = new CouchbaseWriter(couchbaseEnvironment, config);
return AsyncWriterManager.builder()
.asyncDataWriter(couchbaseWriter)
- .failureAllowanceRatio(0.0)
- .retriesEnabled(false)
+ .failureAllowanceRatio(failureAllowance)
+ .retriesEnabled(retriesEnabled)
+ .numRetries(maxRetries)
.config(config)
.build();
}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/1fb2bb73/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java
index 7e29ed8..9d4400b 100644
--- a/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java
+++ b/gobblin-modules/gobblin-couchbase/src/main/java/org/apache/gobblin/couchbase/writer/CouchbaseWriterConfigurationKeys.java
@@ -41,7 +41,16 @@ public class CouchbaseWriterConfigurationKeys {
public static final String SSL_TRUSTSTORE_PASSWORD = prefix("sslTruststorePassword");
public static final String CERT_AUTH_ENABLED = prefix("certAuthEnabled");
- public static final String OPERATION_TIMEOUT_MILLIS = "operationTimeoutMillis";
+ public static final String OPERATION_TIMEOUT_MILLIS = prefix("operationTimeoutMillis");
public static final long OPERATION_TIMEOUT_DEFAULT = 10000; // 10 second default timeout
+ public static final String RETRIES_ENABLED = prefix("retriesEnabled");
+ public static final boolean RETRIES_ENABLED_DEFAULT = false;
+
+ public static final String MAX_RETRIES = prefix("maxRetries");
+ public static final int MAX_RETRIES_DEFAULT = 5;
+
+ static final String FAILURE_ALLOWANCE_PCT_CONFIG = prefix("failureAllowancePercentage");
+ static final double FAILURE_ALLOWANCE_PCT_DEFAULT = 0.0;
+
}