You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2020/10/05 18:37:06 UTC
[hudi] branch master updated: [HUDI-1203] add port configuration
for EmbeddedTimelineService (#2142)
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new fdae388 [HUDI-1203] add port configuration for EmbeddedTimelineService (#2142)
fdae388 is described below
commit fdae388626b8d97acc01191aa0e7075c36a41132
Author: lw0090 <lw...@gmail.com>
AuthorDate: Tue Oct 6 02:36:54 2020 +0800
[HUDI-1203] add port configuration for EmbeddedTimelineService (#2142)
---
.../apache/hudi/client/AbstractHoodieClient.java | 2 +-
.../client/embedded/EmbeddedTimelineService.java | 6 ++--
.../org/apache/hudi/config/HoodieWriteConfig.java | 11 +++++++
.../hudi/timeline/service/TimelineService.java | 37 +++++++++++++++++++---
4 files changed, 48 insertions(+), 8 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
index 9628e41..7a59ebc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
@@ -103,7 +103,7 @@ public abstract class AbstractHoodieClient implements Serializable, AutoCloseabl
LOG.info("Starting Timeline service !!");
Option<String> hostAddr = context.getProperty(EngineProperty.EMBEDDED_SERVER_HOST);
timelineServer = Option.of(new EmbeddedTimelineService(context, hostAddr.orElse(null),
- config.getClientSpecifiedViewStorageConfig()));
+ config.getEmbeddedTimelineServerPort(), config.getClientSpecifiedViewStorageConfig()));
try {
timelineServer.get().startServer();
// Allow executor to find this newly instantiated timeline service
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
index ca0c713..738119c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
@@ -39,17 +39,19 @@ public class EmbeddedTimelineService {
private static final Logger LOG = LogManager.getLogger(EmbeddedTimelineService.class);
private int serverPort;
+ private int preferredPort;
private String hostAddr;
private final SerializableConfiguration hadoopConf;
private final FileSystemViewStorageConfig config;
private transient FileSystemViewManager viewManager;
private transient TimelineService server;
- public EmbeddedTimelineService(HoodieEngineContext context, String embeddedTimelineServiceHostAddr, FileSystemViewStorageConfig config) {
+ public EmbeddedTimelineService(HoodieEngineContext context, String embeddedTimelineServiceHostAddr, int embeddedTimelineServerPort, FileSystemViewStorageConfig config) {
setHostAddr(embeddedTimelineServiceHostAddr);
this.config = config;
this.hadoopConf = context.getHadoopConf();
this.viewManager = createViewManager();
+ this.preferredPort = embeddedTimelineServerPort;
}
private FileSystemViewManager createViewManager() {
@@ -66,7 +68,7 @@ public class EmbeddedTimelineService {
}
public void startServer() throws IOException {
- server = new TimelineService(0, viewManager, hadoopConf.newCopy());
+ server = new TimelineService(preferredPort, viewManager, hadoopConf.newCopy());
serverPort = server.startService();
LOG.info("Started embedded timeline server at " + hostAddr + ":" + serverPort);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 60ca3c2..bb65600 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -99,6 +99,8 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
public static final String EMBEDDED_TIMELINE_SERVER_ENABLED = "hoodie.embed.timeline.server";
public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_ENABLED = "true";
+ public static final String EMBEDDED_TIMELINE_SERVER_PORT = "hoodie.embed.timeline.server.port";
+ public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_PORT = "0";
public static final String FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP = "hoodie.fail.on.timeline.archiving";
public static final String DEFAULT_FAIL_ON_TIMELINE_ARCHIVING_ENABLED = "true";
@@ -255,6 +257,10 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return Boolean.parseBoolean(props.getProperty(EMBEDDED_TIMELINE_SERVER_ENABLED));
}
+ public int getEmbeddedTimelineServerPort() {
+ return Integer.parseInt(props.getProperty(EMBEDDED_TIMELINE_SERVER_PORT, DEFAULT_EMBEDDED_TIMELINE_SERVER_PORT));
+ }
+
public boolean isFailOnTimelineArchivingEnabled() {
return Boolean.parseBoolean(props.getProperty(FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP));
}
@@ -953,6 +959,11 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return this;
}
+ public Builder withEmbeddedTimelineServerPort(int port) {
+ props.setProperty(EMBEDDED_TIMELINE_SERVER_PORT, String.valueOf(port));
+ return this;
+ }
+
public Builder withBulkInsertSortMode(String mode) {
props.setProperty(BULKINSERT_SORT_MODE, mode);
return this;
diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
index f38909f..969f824 100644
--- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
+++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java
@@ -41,6 +41,7 @@ import java.io.Serializable;
public class TimelineService {
private static final Logger LOG = LogManager.getLogger(TimelineService.class);
+ private static final int START_SERVICE_MAX_RETRIES = 16;
private int serverPort;
private Configuration conf;
@@ -98,16 +99,42 @@ public class TimelineService {
public Boolean help = false;
}
+ private int startServiceOnPort(int port) throws IOException {
+ if (!(port == 0 || (1024 <= port && port < 65536))) {
+ throw new IllegalArgumentException(String.format("startPort should be between 1024 and 65535 (inclusive), "
+ + "or 0 for a random free port. but now is %s.", port));
+ }
+ for (int attempt = 0; attempt < START_SERVICE_MAX_RETRIES; attempt++) {
+ // Returns port to try when trying to bind a service. Handles wrapping and skipping privileged ports.
+ int tryPort = port == 0 ? port : (port + attempt - 1024) % (65536 - 1024) + 1024;
+ try {
+ app.start(tryPort);
+ return app.port();
+ } catch (Exception e) {
+ if (e.getMessage() != null && e.getMessage().contains("Failed to bind to")) {
+ if (tryPort == 0) {
+ LOG.warn("Timeline server could not bind on a random free port.");
+ } else {
+ LOG.warn(String.format("Timeline server could not bind on port %d. "
+ + "Attempting port %d + 1.",tryPort, tryPort));
+ }
+ } else {
+ LOG.warn(String.format("Timeline server start failed on port %d. Attempting port %d + 1.",tryPort, tryPort), e);
+ }
+ }
+ }
+ throw new IOException(String.format("Timeline server start failed on port %d, after retry %d times", port, START_SERVICE_MAX_RETRIES));
+ }
+
public int startService() throws IOException {
app = Javalin.create();
FileSystemViewHandler router = new FileSystemViewHandler(app, conf, fsViewsManager);
app.get("/", ctx -> ctx.result("Hello World"));
router.register();
- app.start(serverPort);
- // If port = 0, a dynamic port is assigned. Store it.
- serverPort = app.port();
- LOG.info("Starting Timeline server on port :" + serverPort);
- return serverPort;
+ int realServerPort = startServiceOnPort(serverPort);
+ LOG.info("Starting Timeline server on port :" + realServerPort);
+ this.serverPort = realServerPort;
+ return realServerPort;
}
public void run() throws IOException {