You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2023/02/06 18:21:42 UTC
[airavata-mft] branch master updated (4ed5113 -> 8a38bc8)
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
from 4ed5113 Fixing the bug in passing wrong child id for failed transfers
new 4c4e402 Avoding database resets in service restart
new 4fcb762 Latest cli release
new 8a38bc8 Distributing transfer load uniformly across qualified agents
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/airavata/mft/admin/MFTConsulClient.java | 2 +-
.../mft/controller/TransferDispatcher.java | 38 +++++++++++++++++++++-
python-cli/mft_cli/pyproject.toml | 2 +-
.../distribution/conf/application.properties | 6 ++--
4 files changed, 43 insertions(+), 5 deletions(-)
[airavata-mft] 01/03: Avoding database resets in service restart
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 4c4e402b79f88cc1b3755ad7bed1e3f3013f1704
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Mon Feb 6 13:19:22 2023 -0500
Avoding database resets in service restart
---
.../src/main/resources/distribution/conf/application.properties | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/standalone-service/src/main/resources/distribution/conf/application.properties b/standalone-service/src/main/resources/distribution/conf/application.properties
index 95a221f..e22d45d 100644
--- a/standalone-service/src/main/resources/distribution/conf/application.properties
+++ b/standalone-service/src/main/resources/distribution/conf/application.properties
@@ -16,11 +16,13 @@
#
spring.main.allow-bean-definition-overriding=true
-spring.datasource.url=jdbc:h2:~/mft_db;DB_CLOSE_ON_EXIT=FALSE
+spring.datasource.url=jdbc:h2:~/mft_db;DB_CLOSE_ON_EXIT=FALSE;IFEXISTS=TRUE;DB_CLOSE_DELAY=-1;
+spring.jpa.hibernate.ddl-auto=update
consul.host=localhost
consul.port=8500
resource.service.host=localhost
resource.service.port=7003
secret.service.host=localhost
-secret.service.port=7003
\ No newline at end of file
+secret.service.port=7003
+agent.id=local-agent
\ No newline at end of file
[airavata-mft] 02/03: Latest cli release
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 4fcb7629cc329971194c394ca2c0d3f20cc29415
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Mon Feb 6 13:20:25 2023 -0500
Latest cli release
---
python-cli/mft_cli/pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python-cli/mft_cli/pyproject.toml b/python-cli/mft_cli/pyproject.toml
index 83f46f0..d630c00 100644
--- a/python-cli/mft_cli/pyproject.toml
+++ b/python-cli/mft_cli/pyproject.toml
@@ -18,7 +18,7 @@
[tool.poetry]
name = "airavata-mft-cli"
-version = "0.1.6"
+version = "0.1.7"
description = "Command Line Client for Airavata MFT data transfer software"
authors = ["Apache Airavata <de...@apache.airavata.org>"]
readme = "README.md"
[airavata-mft] 03/03: Distributing transfer load uniformly across qualified agents
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 8a38bc8b901bbcf12ae7d8066e12eb0b200e639c
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Mon Feb 6 13:21:33 2023 -0500
Distributing transfer load uniformly across qualified agents
---
.../apache/airavata/mft/admin/MFTConsulClient.java | 2 +-
.../mft/controller/TransferDispatcher.java | 38 +++++++++++++++++++++-
2 files changed, 38 insertions(+), 2 deletions(-)
diff --git a/common/common-clients/src/main/java/org/apache/airavata/mft/admin/MFTConsulClient.java b/common/common-clients/src/main/java/org/apache/airavata/mft/admin/MFTConsulClient.java
index 35cb4c3..273e6c4 100644
--- a/common/common-clients/src/main/java/org/apache/airavata/mft/admin/MFTConsulClient.java
+++ b/common/common-clients/src/main/java/org/apache/airavata/mft/admin/MFTConsulClient.java
@@ -257,7 +257,7 @@ public class MFTConsulClient {
* @return
* @throws MFTConsulClientException
*/
- public List<String> getAgentActiveTransferIds(AgentInfo agentInfo) throws MFTConsulClientException {
+ public List<String> getAgentActiveTransfers(AgentInfo agentInfo) throws MFTConsulClientException {
try {
List<String> keys = kvClient.getKeys(MFTConsulClient.AGENTS_SCHEDULED_PATH + agentInfo.getId() + "/" + agentInfo.getSessionId());
return keys.stream().map(key -> key.substring(key.lastIndexOf("/") + 1)).collect(Collectors.toList());
diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/TransferDispatcher.java b/controller/src/main/java/org/apache/airavata/mft/controller/TransferDispatcher.java
index bf3b153..3e3b04d 100644
--- a/controller/src/main/java/org/apache/airavata/mft/controller/TransferDispatcher.java
+++ b/controller/src/main/java/org/apache/airavata/mft/controller/TransferDispatcher.java
@@ -18,6 +18,8 @@
package org.apache.airavata.mft.controller;
import org.apache.airavata.mft.admin.MFTConsulClient;
+import org.apache.airavata.mft.admin.MFTConsulClientException;
+import org.apache.airavata.mft.admin.models.AgentInfo;
import org.apache.airavata.mft.admin.models.TransferState;
import org.apache.airavata.mft.agent.stub.AgentTransferRequest;
import org.apache.airavata.mft.api.service.EndpointPaths;
@@ -70,7 +72,7 @@ public class TransferDispatcher {
.setDescription("Initializing the transfer"));
// TODO use a better way to select the right agent
- mftConsulClient.commandTransferToAgent(filteredAgents.get(0), transferId, agentTransferRequest);
+ mftConsulClient.commandTransferToAgent(selectTargetAgent(filteredAgents), transferId, agentTransferRequest);
mftConsulClient.markTransferAsProcessed(transferId, transferRequest);
logger.info("Marked transfer {} as processed", transferId);
@@ -93,6 +95,40 @@ public class TransferDispatcher {
}
}
+
+ private String selectTargetAgent(List<String> liveAgentIds) throws MFTConsulClientException {
+ String selectedAgent = null;
+ List<Optional<AgentInfo>> agentInfos = liveAgentIds.stream().map(
+ id -> mftConsulClient.getAgentInfo(id)).collect(Collectors.toList());
+ int transferCount = -1;
+ List<String> candidates = new ArrayList<>();
+
+ for (Optional<AgentInfo> agentInfo : agentInfos) {
+ if (agentInfo.isPresent()) {
+ List<String> agentActiveTransfers = mftConsulClient.getAgentActiveTransfers(agentInfo.get());
+ logger.info("Agent {} has transfers assigned {}", agentInfo.get().getId(), agentActiveTransfers.size());
+ if (transferCount == -1) {
+ transferCount = agentActiveTransfers.size();
+ candidates.add(agentInfo.get().getId());
+ } else if (transferCount == agentActiveTransfers.size()) {
+ candidates.add(agentInfo.get().getId());
+ } else if (transferCount > agentActiveTransfers.size()) {
+ candidates = new ArrayList<>();
+ transferCount = agentActiveTransfers.size();
+ candidates.add(agentInfo.get().getId());
+ }
+ }
+ }
+
+ if (candidates.size() > 0) {
+ Random rand = new Random();
+ selectedAgent = candidates.get(rand.nextInt(candidates.size()));
+ logger.info("Selecting agent {}", selectedAgent);
+ }
+
+ return selectedAgent;
+ }
+
public void handleTransferRequest(String transferId,
TransferApiRequest transferRequest,
AgentTransferRequest.Builder agentTransferRequestTemplate,