You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2021/08/18 14:16:40 UTC
[airavata] branch develop updated (8878d9c -> af0b722)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git.
from 8878d9c Merge branch 'airavata-3496' into develop
new 247d63f AIRAVATA-3322 Update airavata-python-sdk to 1.0.1
new ef0b2f9 Merge branch 'airavata-3322-old' into airavata-3302
new b800474 AIRAVATA-3302 Push experiment search filters into sharing registry query
new af0b722 Merge branch 'airavata-3302' into develop
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../api/server/handler/AiravataServerHandler.java | 86 +++++++++++++++---
.../airavata-python-sdk/setup.py | 2 +-
.../core/repositories/AbstractRepository.java | 15 ++++
.../expcatalog/ExperimentSummaryRepository.java | 100 +++++++++++++++++----
.../ExperimentSummaryRepositoryTest.java | 62 ++++++++++---
5 files changed, 225 insertions(+), 40 deletions(-)
[airavata] 02/04: Merge branch 'airavata-3322-old' into
airavata-3302
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit ef0b2f99f1765ea4551e5dbc03b03525d6947c8a
Merge: ad3ce1c 247d63f
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Mon Aug 16 15:04:26 2021 -0400
Merge branch 'airavata-3322-old' into airavata-3302
.../api/server/handler/AiravataServerHandler.java | 4 +-
.../java/org/apache/airavata/api/Airavata.java | 240 ++++++++++++++++++--
.../airavata/api/Airavata-remote | 8 +-
.../airavata-python-sdk/airavata/api/Airavata.py | 54 ++++-
.../airavata/model/workspace/ttypes.py | 90 ++++++++
.../airavata-python-sdk/setup.py | 2 +-
.../expcatalog/ExperimentSummaryRepository.java | 132 +++++++----
.../ExperimentSummaryRepositoryTest.java | 28 ++-
.../api/service/handler/RegistryServerHandler.java | 7 +-
.../airavata/registry/api/RegistryService.java | 241 +++++++++++++++++++--
.../experimentCatalog_schema_delta.sql | 3 +
.../airavata-apis/airavata_api.thrift | 10 +-
.../component-cpis/registry-api.thrift | 11 +-
13 files changed, 742 insertions(+), 88 deletions(-)
[airavata] 01/04: AIRAVATA-3322 Update airavata-python-sdk to 1.0.1
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 247d63f6079cdf6827d5b0f589a7d622c15540ce
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue Jul 27 16:31:07 2021 -0400
AIRAVATA-3322 Update airavata-python-sdk to 1.0.1
---
airavata-api/airavata-client-sdks/airavata-python-sdk/setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airavata-api/airavata-client-sdks/airavata-python-sdk/setup.py b/airavata-api/airavata-client-sdks/airavata-python-sdk/setup.py
index d190881..e5af942 100644
--- a/airavata-api/airavata-client-sdks/airavata-python-sdk/setup.py
+++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/setup.py
@@ -10,7 +10,7 @@ def read(fname):
setup(
name='airavata-python-sdk',
- version='1.0.0',
+ version='1.0.1',
packages=find_packages(),
package_data={'transport': ['*.ini'], 'sample': ['*.pem']},
url='http://airavata.com',
[airavata] 03/04: AIRAVATA-3302 Push experiment search filters into
sharing registry query
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit b8004747428f969eda9345cab84ad73d70e068fd
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Aug 18 10:06:40 2021 -0400
AIRAVATA-3302 Push experiment search filters into sharing registry query
This fixes the issue where only the most recent 1000 experiments are ever
searched in the searchExperiments API method. As much filtering as possible
occurs in the sharing API call and when there are experiment filters that
don't map to the sharing API, all accessible experiment ids are retrieved
and further filtered by the Registry.
---
.../api/server/handler/AiravataServerHandler.java | 86 +++++++++++++++---
.../core/repositories/AbstractRepository.java | 15 ++++
.../expcatalog/ExperimentSummaryRepository.java | 100 +++++++++++++++++----
.../ExperimentSummaryRepositoryTest.java | 62 ++++++++++---
4 files changed, 224 insertions(+), 39 deletions(-)
diff --git a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
index 6b86d01..34a5f35 100644
--- a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
+++ b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java
@@ -1160,7 +1160,7 @@ public class AiravataServerHandler implements Airavata.Iface {
searchCriteria.setValue(gatewayId + ":PROJECT");
sharingFilters.add(searchCriteria);
sharingClient.searchEntities(authzToken.getClaimsMap().get(Constants.GATEWAY_ID),
- userName + "@" + gatewayId, sharingFilters, 0, -1).stream().forEach(e -> accessibleProjIds.add(e.getEntityId()));
+ userName + "@" + gatewayId, sharingFilters, 0, Integer.MAX_VALUE).stream().forEach(e -> accessibleProjIds.add(e.getEntityId()));
if (accessibleProjIds.isEmpty()) {
result = Collections.emptyList();
} else {
@@ -1208,17 +1208,81 @@ public class AiravataServerHandler implements Airavata.Iface {
SharingRegistryService.Client sharingClient = sharingClientPool.getResource();
try {
List<String> accessibleExpIds = new ArrayList<>();
- if (ServerSettings.isEnableSharing()) {
- List<SearchCriteria> sharingFilters = new ArrayList<>();
- SearchCriteria searchCriteria = new SearchCriteria();
- searchCriteria.setSearchField(EntitySearchField.ENTITY_TYPE_ID);
- searchCriteria.setSearchCondition(SearchCondition.EQUAL);
- searchCriteria.setValue(gatewayId + ":EXPERIMENT");
- sharingFilters.add(searchCriteria);
- sharingClient.searchEntities(authzToken.getClaimsMap().get(Constants.GATEWAY_ID),
- userName + "@" + gatewayId, sharingFilters, 0, -1).forEach(e -> accessibleExpIds.add(e.getEntityId()));
+ Map<ExperimentSearchFields, String> filtersCopy = new HashMap<>(filters);
+ List<SearchCriteria> sharingFilters = new ArrayList<>();
+ SearchCriteria searchCriteria = new SearchCriteria();
+ searchCriteria.setSearchField(EntitySearchField.ENTITY_TYPE_ID);
+ searchCriteria.setSearchCondition(SearchCondition.EQUAL);
+ searchCriteria.setValue(gatewayId + ":EXPERIMENT");
+ sharingFilters.add(searchCriteria);
+
+ // Apply as much of the filters in the sharing API as possible,
+ // removing each filter that can be filtered via the sharing API
+ if (filtersCopy.containsKey(ExperimentSearchFields.FROM_DATE)) {
+ String fromTime = filtersCopy.remove(ExperimentSearchFields.FROM_DATE);
+ SearchCriteria fromCreatedTimeCriteria = new SearchCriteria();
+ fromCreatedTimeCriteria.setSearchField(EntitySearchField.CREATED_TIME);
+ fromCreatedTimeCriteria.setSearchCondition(SearchCondition.GTE);
+ fromCreatedTimeCriteria.setValue(fromTime);
+ sharingFilters.add(fromCreatedTimeCriteria);
+ }
+ if (filtersCopy.containsKey(ExperimentSearchFields.TO_DATE)) {
+ String toTime = filtersCopy.remove(ExperimentSearchFields.TO_DATE);
+ SearchCriteria toCreatedTimeCriteria = new SearchCriteria();
+ toCreatedTimeCriteria.setSearchField(EntitySearchField.CREATED_TIME);
+ toCreatedTimeCriteria.setSearchCondition(SearchCondition.LTE);
+ toCreatedTimeCriteria.setValue(toTime);
+ sharingFilters.add(toCreatedTimeCriteria);
+ }
+ if (filtersCopy.containsKey(ExperimentSearchFields.PROJECT_ID)) {
+ String projectId = filtersCopy.remove(ExperimentSearchFields.PROJECT_ID);
+ SearchCriteria projectParentEntityCriteria = new SearchCriteria();
+ projectParentEntityCriteria.setSearchField(EntitySearchField.PARRENT_ENTITY_ID);
+ projectParentEntityCriteria.setSearchCondition(SearchCondition.EQUAL);
+ projectParentEntityCriteria.setValue(projectId);
+ sharingFilters.add(projectParentEntityCriteria);
+ }
+ if (filtersCopy.containsKey(ExperimentSearchFields.USER_NAME)) {
+ String username = filtersCopy.remove(ExperimentSearchFields.USER_NAME);
+ SearchCriteria usernameOwnerCriteria = new SearchCriteria();
+ usernameOwnerCriteria.setSearchField(EntitySearchField.OWNER_ID);
+ usernameOwnerCriteria.setSearchCondition(SearchCondition.EQUAL);
+ usernameOwnerCriteria.setValue(username + "@" + gatewayId);
+ sharingFilters.add(usernameOwnerCriteria);
+ }
+ if (filtersCopy.containsKey(ExperimentSearchFields.EXPERIMENT_NAME)) {
+ String experimentName = filtersCopy.remove(ExperimentSearchFields.EXPERIMENT_NAME);
+ SearchCriteria experimentNameCriteria = new SearchCriteria();
+ experimentNameCriteria.setSearchField(EntitySearchField.NAME);
+ experimentNameCriteria.setSearchCondition(SearchCondition.LIKE);
+ experimentNameCriteria.setValue(experimentName);
+ sharingFilters.add(experimentNameCriteria);
+ }
+ if (filtersCopy.containsKey(ExperimentSearchFields.EXPERIMENT_DESC)) {
+ String experimentDescription = filtersCopy.remove(ExperimentSearchFields.EXPERIMENT_DESC);
+ SearchCriteria experimentDescriptionCriteria = new SearchCriteria();
+ experimentDescriptionCriteria.setSearchField(EntitySearchField.DESCRIPTION);
+ experimentDescriptionCriteria.setSearchCondition(SearchCondition.LIKE);
+ experimentDescriptionCriteria.setValue(experimentDescription);
+ sharingFilters.add(experimentDescriptionCriteria);
+ }
+ // Grab all of the matching experiments in the sharing registry
+ // unless all of the filtering can be done through the sharing API
+ int searchOffset = 0;
+ int searchLimit = Integer.MAX_VALUE;
+ boolean filteredInSharing = filtersCopy.isEmpty();
+ if (filteredInSharing) {
+ searchOffset = offset;
+ searchLimit = limit;
+ }
+ sharingClient.searchEntities(authzToken.getClaimsMap().get(Constants.GATEWAY_ID),
+ userName + "@" + gatewayId, sharingFilters, searchOffset, searchLimit).forEach(e -> accessibleExpIds.add(e.getEntityId()));
+ int finalOffset = offset;
+ // If no more filtering to be done (either empty or all done through sharing API), set the offset to 0
+ if (filteredInSharing) {
+ finalOffset = 0;
}
- List<ExperimentSummaryModel> result = regClient.searchExperiments(gatewayId, userName, accessibleExpIds, filters, limit, offset);
+ List<ExperimentSummaryModel> result = regClient.searchExperiments(gatewayId, userName, accessibleExpIds, filtersCopy, limit, finalOffset);
registryClientPool.returnResource(regClient);
sharingClientPool.returnResource(sharingClient);
return result;
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/repositories/AbstractRepository.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/repositories/AbstractRepository.java
index acbd3fb..95062c3 100644
--- a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/repositories/AbstractRepository.java
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/repositories/AbstractRepository.java
@@ -113,6 +113,21 @@ public abstract class AbstractRepository<T, E, Id> {
return get(id) != null;
}
+ public int scalarInt(String query, Map<String, Object> queryParams) {
+
+ int scalarInt = execute(entityManager -> {
+ Query jpaQuery = entityManager.createQuery(query);
+
+ for (Map.Entry<String, Object> entry : queryParams.entrySet()) {
+
+ jpaQuery.setParameter(entry.getKey(), entry.getValue());
+ }
+
+ return ((Number)jpaQuery.getSingleResult()).intValue();
+ });
+ return scalarInt;
+ }
+
public <R> R execute(Committer<EntityManager, R> committer){
EntityManager entityManager = null;
try {
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/repositories/expcatalog/ExperimentSummaryRepository.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/repositories/expcatalog/ExperimentSummaryRepository.java
index f6ab852..ee887fd 100644
--- a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/repositories/expcatalog/ExperimentSummaryRepository.java
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/repositories/expcatalog/ExperimentSummaryRepository.java
@@ -25,13 +25,9 @@ import org.apache.airavata.model.experiment.ExperimentSummaryModel;
import org.apache.airavata.model.status.ExperimentState;
import org.apache.airavata.registry.core.entities.expcatalog.ExperimentSummaryEntity;
import org.apache.airavata.registry.core.entities.expcatalog.JobEntity;
-import org.apache.airavata.registry.core.entities.expcatalog.ProcessEntity;
-import org.apache.airavata.registry.core.entities.expcatalog.TaskEntity;
import org.apache.airavata.registry.core.utils.DBConstants;
import org.apache.airavata.registry.cpi.RegistryException;
import org.apache.airavata.registry.cpi.ResultOrderType;
-import org.apache.airavata.registry.cpi.utils.Constants;
-import org.apache.derby.vti.Restriction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,12 +43,14 @@ import javax.persistence.Query;
public class ExperimentSummaryRepository extends ExpCatAbstractRepository<ExperimentSummaryModel, ExperimentSummaryEntity, String> {
private final static Logger logger = LoggerFactory.getLogger(ExperimentSummaryRepository.class);
+ private final static int ACCESSIBLE_EXPERIMENT_IDS_BATCH_SIZE = 10000;
public ExperimentSummaryRepository() { super(ExperimentSummaryModel.class, ExperimentSummaryEntity.class); }
public List<ExperimentSummaryModel> searchAllAccessibleExperiments(List<String> accessibleExperimentIds, Map<String, String> filters, int limit,
int offset, Object orderByIdentifier, ResultOrderType resultOrderType) throws RegistryException, IllegalArgumentException {
String query = "SELECT ES FROM " + ExperimentSummaryEntity.class.getSimpleName() + " ES WHERE ";
+ String whereClause = "";
Map<String, Object> queryParameters = new HashMap<>();
if (filters == null || !filters.containsKey(DBConstants.Experiment.GATEWAY_ID)) {
@@ -68,50 +66,50 @@ public class ExperimentSummaryRepository extends ExpCatAbstractRepository<Experi
+ " JOIN J.task T"
+ " JOIN T.process P"
+ " WHERE J.jobId = : " + DBConstants.Job.JOB_ID;
- query += "ES.experimentId IN ( "+ query_jobId + " ) AND ";
+ whereClause += "ES.experimentId IN ( " + query_jobId + " ) AND ";
}
if (filters.get(DBConstants.Experiment.USER_NAME) != null) {
logger.debug("Filter Experiments by User");
queryParameters.put(DBConstants.Experiment.USER_NAME, filters.get(DBConstants.Experiment.USER_NAME));
- query += "ES.userName LIKE :" + DBConstants.Experiment.USER_NAME + " AND ";
+ whereClause += "ES.userName LIKE :" + DBConstants.Experiment.USER_NAME + " AND ";
}
if (filters.get(DBConstants.Experiment.GATEWAY_ID) != null) {
logger.debug("Filter Experiments by Gateway ID");
queryParameters.put(DBConstants.Experiment.GATEWAY_ID, filters.get(DBConstants.Experiment.GATEWAY_ID));
- query += "ES.gatewayId LIKE :" + DBConstants.Experiment.GATEWAY_ID + " AND ";
+ whereClause += "ES.gatewayId LIKE :" + DBConstants.Experiment.GATEWAY_ID + " AND ";
}
if (filters.get(DBConstants.Experiment.PROJECT_ID) != null) {
logger.debug("Filter Experiments by Project ID");
queryParameters.put(DBConstants.Experiment.PROJECT_ID, filters.get(DBConstants.Experiment.PROJECT_ID));
- query += "ES.projectId LIKE :" + DBConstants.Experiment.PROJECT_ID + " AND ";
+ whereClause += "ES.projectId LIKE :" + DBConstants.Experiment.PROJECT_ID + " AND ";
}
if (filters.get(DBConstants.Experiment.EXPERIMENT_NAME) != null) {
logger.debug("Filter Experiments by Name");
queryParameters.put(DBConstants.Experiment.EXPERIMENT_NAME, filters.get(DBConstants.Experiment.EXPERIMENT_NAME));
- query += "ES.name LIKE :" + DBConstants.Experiment.EXPERIMENT_NAME + " AND ";
+ whereClause += "ES.name LIKE :" + DBConstants.Experiment.EXPERIMENT_NAME + " AND ";
}
if (filters.get(DBConstants.Experiment.DESCRIPTION) != null) {
logger.debug("Filter Experiments by Description");
queryParameters.put(DBConstants.Experiment.DESCRIPTION, filters.get(DBConstants.Experiment.DESCRIPTION));
- query += "ES.description LIKE :" + DBConstants.Experiment.DESCRIPTION + " AND ";
+ whereClause += "ES.description LIKE :" + DBConstants.Experiment.DESCRIPTION + " AND ";
}
if (filters.get(DBConstants.Experiment.EXECUTION_ID) != null) {
logger.debug("Filter Experiments by Execution ID");
queryParameters.put(DBConstants.Experiment.EXECUTION_ID, filters.get(DBConstants.Experiment.EXECUTION_ID));
- query += "ES.executionId LIKE :" + DBConstants.Experiment.EXECUTION_ID + " AND ";
+ whereClause += "ES.executionId LIKE :" + DBConstants.Experiment.EXECUTION_ID + " AND ";
}
if (filters.get(DBConstants.ExperimentSummary.EXPERIMENT_STATUS) != null) {
logger.debug("Filter Experiments by State");
String state = ExperimentState.valueOf(filters.get(DBConstants.ExperimentSummary.EXPERIMENT_STATUS)).toString();
queryParameters.put(DBConstants.ExperimentSummary.EXPERIMENT_STATUS, state);
- query += "ES.experimentStatus LIKE :" + DBConstants.ExperimentSummary.EXPERIMENT_STATUS + " AND ";
+ whereClause += "ES.experimentStatus LIKE :" + DBConstants.ExperimentSummary.EXPERIMENT_STATUS + " AND ";
}
if (filters.get(DBConstants.ExperimentSummary.FROM_DATE) != null
@@ -124,7 +122,8 @@ public class ExperimentSummaryRepository extends ExpCatAbstractRepository<Experi
logger.debug("Filter Experiments by CreationTime");
queryParameters.put(DBConstants.ExperimentSummary.FROM_DATE, fromDate);
queryParameters.put(DBConstants.ExperimentSummary.TO_DATE, toDate);
- query += "ES.creationTime BETWEEN :" + DBConstants.ExperimentSummary.FROM_DATE + " AND :" + DBConstants.ExperimentSummary.TO_DATE + " AND ";
+ whereClause += "ES.creationTime BETWEEN :" + DBConstants.ExperimentSummary.FROM_DATE + " AND :"
+ + DBConstants.ExperimentSummary.TO_DATE + " AND ";
}
}
@@ -132,13 +131,13 @@ public class ExperimentSummaryRepository extends ExpCatAbstractRepository<Experi
if (filters.get(DBConstants.Experiment.USER_NAME) != null) {
logger.debug("Filter Experiments by Username");
queryParameters.put(DBConstants.Experiment.USER_NAME, filters.get(DBConstants.Experiment.USER_NAME));
- query += "ES.userName = :" + DBConstants.Experiment.USER_NAME + " AND ";
+ whereClause += "ES.userName = :" + DBConstants.Experiment.USER_NAME + " AND ";
}
if (!accessibleExperimentIds.isEmpty()) {
logger.debug("Filter Experiments by Accessible Experiment IDs");
queryParameters.put(DBConstants.Experiment.ACCESSIBLE_EXPERIMENT_IDS, accessibleExperimentIds);
- query += " ES.experimentId IN :" + DBConstants.Experiment.ACCESSIBLE_EXPERIMENT_IDS;
+ whereClause += " ES.experimentId IN :" + DBConstants.Experiment.ACCESSIBLE_EXPERIMENT_IDS;
}
else {
@@ -146,13 +145,80 @@ public class ExperimentSummaryRepository extends ExpCatAbstractRepository<Experi
return new ArrayList<ExperimentSummaryModel>();
}
+ int queryLimit = limit;
+ int queryOffset = offset;
+ int accessibleExperimentIdsBatchNum = 0;
+
+ // Figure out the initial batch of accessible experiment ids and the
+ // offset into it by counting the matching experiments in each batch
+ if (queryOffset > 0) {
+ String countQuery = "SELECT COUNT(ES) FROM " + ExperimentSummaryEntity.class.getSimpleName() + " ES WHERE ";
+ countQuery += whereClause;
+ BatchOffset batchOffset = findInitialAccessibleExperimentsBatchOffset(countQuery, queryOffset, queryParameters, accessibleExperimentIds);
+ queryOffset = batchOffset.offset;
+ accessibleExperimentIdsBatchNum = batchOffset.batchNum;
+ }
+
+ query += whereClause;
if (orderByIdentifier != null && resultOrderType != null && orderByIdentifier.equals(DBConstants.Experiment.CREATION_TIME)) {
String order = (resultOrderType == ResultOrderType.ASC) ? "ASC" : "DESC";
query += " ORDER BY ES." + DBConstants.Experiment.CREATION_TIME + " " + order;
}
- List<ExperimentSummaryModel> experimentSummaryModelList = select(query, limit, offset, queryParameters);
- return experimentSummaryModelList;
+ List<ExperimentSummaryModel> allExperimentSummaryModels = new ArrayList<>();
+
+ // Break up the query in batches over accessibleExperimentIds
+ // NOTE: this assumes that the accessibleExperimentIds are sorted in the
+ // same order as the expected experiment summary results
+ double totalBatches = Math.ceil(
+ Integer.valueOf(accessibleExperimentIds.size()).floatValue() / ACCESSIBLE_EXPERIMENT_IDS_BATCH_SIZE);
+ for (int batchNum = accessibleExperimentIdsBatchNum; batchNum < totalBatches; batchNum++) {
+ List<String> accessibleExperimentIdsBatch = accessibleExperimentIds.subList(
+ batchNum * ACCESSIBLE_EXPERIMENT_IDS_BATCH_SIZE,
+ Math.min(accessibleExperimentIds.size(), (batchNum + 1) * ACCESSIBLE_EXPERIMENT_IDS_BATCH_SIZE));
+ queryParameters.put(DBConstants.Experiment.ACCESSIBLE_EXPERIMENT_IDS, accessibleExperimentIdsBatch);
+ List<ExperimentSummaryModel> experimentSummaryModelList = select(query, queryLimit, queryOffset, queryParameters);
+ allExperimentSummaryModels.addAll(experimentSummaryModelList);
+ if (allExperimentSummaryModels.size() == limit) {
+ return allExperimentSummaryModels;
+ } else if (limit > 0 && allExperimentSummaryModels.size() < limit) {
+ queryLimit -= experimentSummaryModelList.size();
+ // In the next and subsequent batches, start from offset 0
+ queryOffset = 0;
+ }
+ }
+ return allExperimentSummaryModels;
+ }
+
+ class BatchOffset {
+ final int batchNum;
+ final int offset;
+
+ BatchOffset(int batchNum, int offset) {
+ this.batchNum = batchNum;
+ this.offset = offset;
+ }
+ }
+
+ private BatchOffset findInitialAccessibleExperimentsBatchOffset(String query, int queryOffset,
+ Map<String, Object> queryParameters, List<String> accessibleExperimentIds) {
+
+ int accumulator = 0;
+
+ double totalBatches = Math.ceil(
+ Integer.valueOf(accessibleExperimentIds.size()).floatValue() / ACCESSIBLE_EXPERIMENT_IDS_BATCH_SIZE);
+ for (int batchNum = 0; batchNum < totalBatches; batchNum++) {
+ List<String> accessibleExperimentIdsBatch = accessibleExperimentIds.subList(
+ batchNum * ACCESSIBLE_EXPERIMENT_IDS_BATCH_SIZE,
+ Math.min(accessibleExperimentIds.size(), (batchNum + 1) * ACCESSIBLE_EXPERIMENT_IDS_BATCH_SIZE));
+ queryParameters.put(DBConstants.Experiment.ACCESSIBLE_EXPERIMENT_IDS, accessibleExperimentIdsBatch);
+ int count = scalarInt(query, queryParameters);
+ if (accumulator + count > queryOffset ) {
+ return new BatchOffset(batchNum, queryOffset - accumulator);
+ }
+ accumulator += count;
+ }
+ return new BatchOffset(0, 0);
}
public ExperimentStatistics getAccessibleExperimentStatistics(List<String> accessibleExperimentIds, Map<String,String> filters, int limit, int offset) throws RegistryException {
diff --git a/modules/registry/registry-core/src/test/java/org/apache/airavata/registry/core/repositories/expcatalog/ExperimentSummaryRepositoryTest.java b/modules/registry/registry-core/src/test/java/org/apache/airavata/registry/core/repositories/expcatalog/ExperimentSummaryRepositoryTest.java
index 248b12e..bbc9ad4 100644
--- a/modules/registry/registry-core/src/test/java/org/apache/airavata/registry/core/repositories/expcatalog/ExperimentSummaryRepositoryTest.java
+++ b/modules/registry/registry-core/src/test/java/org/apache/airavata/registry/core/repositories/expcatalog/ExperimentSummaryRepositoryTest.java
@@ -93,6 +93,15 @@ public class ExperimentSummaryRepositoryTest extends TestBase{
experimentModelTwo.setDescription("descriptionTwo");
experimentModelTwo.setExecutionId("executionIdTwo");
+ ExperimentModel experimentModelThree = new ExperimentModel();
+ experimentModelThree.setProjectId(projectId);
+ experimentModelThree.setGatewayId(gatewayId);
+ experimentModelThree.setExperimentType(ExperimentType.SINGLE_APPLICATION);
+ experimentModelThree.setUserName("userThree");
+ experimentModelThree.setExperimentName("nameThree");
+ experimentModelThree.setDescription("descriptionThree");
+ experimentModelThree.setExecutionId("executionIdThree");
+
String experimentIdOne = experimentRepository.addExperiment(experimentModelOne);
assertTrue(experimentIdOne != null);
// Reload experiment to get its status' identifier
@@ -103,6 +112,11 @@ public class ExperimentSummaryRepositoryTest extends TestBase{
// Reload experiment to get its status' identifier
experimentModelTwo = experimentRepository.getExperiment(experimentIdTwo);
+ String experimentIdThree = experimentRepository.addExperiment(experimentModelThree);
+ assertTrue(experimentIdThree != null);
+ // Reload experiment to get its status' identifier
+ experimentModelThree = experimentRepository.getExperiment(experimentIdThree);
+
Timestamp timeOne = Timestamp.valueOf("2010-01-01 09:00:00");
experimentModelOne.setCreationTime(timeOne.getTime());
experimentRepository.updateExperiment(experimentModelOne, experimentIdOne);
@@ -111,14 +125,18 @@ public class ExperimentSummaryRepositoryTest extends TestBase{
experimentModelTwo.setCreationTime(timeTwo.getTime());
experimentRepository.updateExperiment(experimentModelTwo, experimentIdTwo);
+ Timestamp timeThree = Timestamp.valueOf("2020-01-01 09:00:00");
+ experimentModelThree.setCreationTime(timeThree.getTime());
+ experimentRepository.updateExperiment(experimentModelThree, experimentIdThree);
+
Map<String, String> filters = new HashMap<>();
filters.put(DBConstants.Experiment.GATEWAY_ID, gatewayId);
filters.put(DBConstants.Experiment.PROJECT_ID, projectId);
- List<String> allExperimentIds = Arrays.asList( experimentIdOne, experimentIdTwo);
+ List<String> allExperimentIds = Arrays.asList( experimentIdOne, experimentIdTwo, experimentIdThree);
List<ExperimentSummaryModel> experimentSummaryModelList = experimentSummaryRepository.
searchAllAccessibleExperiments(allExperimentIds, filters, -1, 0, null, null);
- assertEquals(2, experimentSummaryModelList.size());
+ assertEquals(3, experimentSummaryModelList.size());
filters.put(DBConstants.Experiment.EXECUTION_ID, "executionIdTwo");
@@ -166,6 +184,22 @@ public class ExperimentSummaryRepositoryTest extends TestBase{
assertEquals("should return only userOne's exp", 1, experimentSummaryModelList.size());
assertEquals("userOne", experimentSummaryModelList.get(0).getUserName());
+ // Test with pagination
+ filters.clear();
+ filters.put(DBConstants.Experiment.GATEWAY_ID, gatewayId);
+ experimentSummaryModelList = experimentSummaryRepository.searchAllAccessibleExperiments(
+ allExperimentIds, filters, 2, 0,
+ DBConstants.Experiment.CREATION_TIME, ResultOrderType.ASC);
+ assertEquals("should only return 2 experiments since limit=2", 2, experimentSummaryModelList.size());
+ assertEquals(experimentIdOne, experimentSummaryModelList.get(0).getExperimentId());
+ assertEquals(experimentIdTwo, experimentSummaryModelList.get(1).getExperimentId());
+ // page 2
+ experimentSummaryModelList = experimentSummaryRepository.searchAllAccessibleExperiments(
+ allExperimentIds, filters, 2, 2,
+ DBConstants.Experiment.CREATION_TIME, ResultOrderType.ASC);
+ assertEquals("should only return 1 experiment since limit=2 but partial last page", 1, experimentSummaryModelList.size());
+ assertEquals(experimentIdThree, experimentSummaryModelList.get(0).getExperimentId());
+
filters = new HashMap<>();
filters.put(DBConstants.Experiment.GATEWAY_ID, gatewayId);
filters.put(DBConstants.Experiment.USER_NAME, "userTwo");
@@ -194,16 +228,21 @@ public class ExperimentSummaryRepositoryTest extends TestBase{
String statusIdTwo = experimentStatusRepository.addExperimentStatus(experimentStatusTwo, experimentIdTwo);
assertTrue(statusIdTwo != null);
+ ExperimentStatus experimentStatusThree = new ExperimentStatus(ExperimentState.CANCELED);
+ String statusIdThree = experimentStatusRepository.addExperimentStatus(experimentStatusThree, experimentIdThree);
+ assertTrue(statusIdThree != null);
+
experimentStatistics = experimentSummaryRepository.getAccessibleExperimentStatistics(allExperimentIds, filters, 10, 0);
- assertTrue(experimentStatistics.getAllExperimentCount() == 1);
+ assertEquals(2, experimentStatistics.getAllExperimentCount());
assertTrue(experimentStatistics.getRunningExperimentCount() == 1);
- assertEquals(experimentIdTwo, experimentStatistics.getAllExperiments().get(0).getExperimentId());
+ // Experiment 3 is most recent
+ assertEquals(experimentIdThree, experimentStatistics.getAllExperiments().get(0).getExperimentId());
filters.remove(DBConstants.ExperimentSummary.FROM_DATE);
filters.remove(DBConstants.ExperimentSummary.TO_DATE);
experimentStatistics = experimentSummaryRepository.getAccessibleExperimentStatistics(allExperimentIds, filters, 10, 0);
- assertTrue(experimentStatistics.getAllExperimentCount() == 2);
+ assertTrue(experimentStatistics.getAllExperimentCount() == 3);
assertTrue(experimentStatistics.getCreatedExperimentCount() == 1);
assertTrue(experimentStatistics.getRunningExperimentCount() == 1);
@@ -237,20 +276,21 @@ public class ExperimentSummaryRepositoryTest extends TestBase{
// First page
experimentStatistics = experimentSummaryRepository.getAccessibleExperimentStatistics(allExperimentIds, filters, 1, 0);
// Should still return total count even when only returning the first page of experiment summaries
- assertEquals(2, experimentStatistics.getAllExperimentCount());
- // experiment 2 is more recent
+ assertEquals(3, experimentStatistics.getAllExperimentCount());
+ // experiment 3 is most recent
assertEquals(1, experimentStatistics.getAllExperimentsSize());
- assertEquals(experimentIdTwo, experimentStatistics.getAllExperiments().get(0).getExperimentId());
+ assertEquals(experimentIdThree, experimentStatistics.getAllExperiments().get(0).getExperimentId());
// Second page
experimentStatistics = experimentSummaryRepository.getAccessibleExperimentStatistics(allExperimentIds, filters, 1, 1);
// Should still return total count even when only returning the first page of experiment summaries
- assertEquals(2, experimentStatistics.getAllExperimentCount());
- // experiment 1 is less recent
+ assertEquals(3, experimentStatistics.getAllExperimentCount());
+ // experiment 2 is less recent
assertEquals(1, experimentStatistics.getAllExperimentsSize());
- assertEquals(experimentIdOne, experimentStatistics.getAllExperiments().get(0).getExperimentId());
+ assertEquals(experimentIdTwo, experimentStatistics.getAllExperiments().get(0).getExperimentId());
experimentRepository.removeExperiment(experimentIdOne);
experimentRepository.removeExperiment(experimentIdTwo);
+ experimentRepository.removeExperiment(experimentIdThree);
gatewayRepository.removeGateway(gatewayId);
projectRepository.removeProject(projectId);
[airavata] 04/04: Merge branch 'airavata-3302' into develop
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit af0b722f8556cd25f047fecc749cd83d205f1fac
Merge: 8878d9c b800474
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Aug 18 10:07:13 2021 -0400
Merge branch 'airavata-3302' into develop
.../api/server/handler/AiravataServerHandler.java | 86 +++++++++++++++---
.../airavata-python-sdk/setup.py | 2 +-
.../core/repositories/AbstractRepository.java | 15 ++++
.../expcatalog/ExperimentSummaryRepository.java | 100 +++++++++++++++++----
.../ExperimentSummaryRepositoryTest.java | 62 ++++++++++---
5 files changed, 225 insertions(+), 40 deletions(-)
diff --cc airavata-api/airavata-client-sdks/airavata-python-sdk/setup.py
index f57883c,e5af942..c0762d7
--- a/airavata-api/airavata-client-sdks/airavata-python-sdk/setup.py
+++ b/airavata-api/airavata-client-sdks/airavata-python-sdk/setup.py
@@@ -10,9 -10,9 +10,9 @@@ def read(fname)
setup(
name='airavata-python-sdk',
- version='1.0.0',
+ version='1.0.1',
packages=find_packages(),
- package_data={'transport': ['*.ini'], 'sample': ['*.pem']},
+ package_data={'airavata_sdk.transport': ['*.ini'], 'airavata_sdk.samples.resources': ['*.pem']},
url='http://airavata.com',
license='Apache License 2.0',
author='Airavata Developers',